xref: /linux/include/linux/bpf.h (revision d0d106a2bd21499901299160744e5fe9f4c83ddb)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  */
4 #ifndef _LINUX_BPF_H
5 #define _LINUX_BPF_H 1
6 
7 #include <uapi/linux/bpf.h>
8 #include <uapi/linux/filter.h>
9 
10 #include <linux/workqueue.h>
11 #include <linux/file.h>
12 #include <linux/percpu.h>
13 #include <linux/err.h>
14 #include <linux/rbtree_latch.h>
15 #include <linux/numa.h>
16 #include <linux/mm_types.h>
17 #include <linux/wait.h>
18 #include <linux/refcount.h>
19 #include <linux/mutex.h>
20 #include <linux/module.h>
21 #include <linux/kallsyms.h>
22 #include <linux/capability.h>
23 #include <linux/sched/mm.h>
24 #include <linux/slab.h>
25 #include <linux/percpu-refcount.h>
26 #include <linux/stddef.h>
27 #include <linux/bpfptr.h>
28 #include <linux/btf.h>
29 #include <linux/rcupdate_trace.h>
30 #include <linux/static_call.h>
31 #include <linux/memcontrol.h>
32 #include <linux/cfi.h>
33 
34 struct bpf_verifier_env;
35 struct bpf_verifier_log;
36 struct perf_event;
37 struct bpf_prog;
38 struct bpf_prog_aux;
39 struct bpf_map;
40 struct bpf_arena;
41 struct sock;
42 struct seq_file;
43 struct btf;
44 struct btf_type;
45 struct exception_table_entry;
46 struct seq_operations;
47 struct bpf_iter_aux_info;
48 struct bpf_local_storage;
49 struct bpf_local_storage_map;
50 struct kobject;
51 struct mem_cgroup;
52 struct module;
53 struct bpf_func_state;
54 struct ftrace_ops;
55 struct cgroup;
56 struct bpf_token;
57 struct user_namespace;
58 struct super_block;
59 struct inode;
60 
61 extern struct idr btf_idr;
62 extern spinlock_t btf_idr_lock;
63 extern struct kobject *btf_kobj;
64 extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
65 extern bool bpf_global_ma_set;
66 
67 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
68 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
69 					struct bpf_iter_aux_info *aux);
70 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
71 typedef unsigned int (*bpf_func_t)(const void *,
72 				   const struct bpf_insn *);
73 struct bpf_iter_seq_info {
74 	const struct seq_operations *seq_ops;
75 	bpf_iter_init_seq_priv_t init_seq_private;
76 	bpf_iter_fini_seq_priv_t fini_seq_private;
77 	u32 seq_priv_size;
78 };
79 
80 /* map is generic key/value storage optionally accessible by eBPF programs */
81 struct bpf_map_ops {
82 	/* funcs callable from userspace (via syscall) */
83 	int (*map_alloc_check)(union bpf_attr *attr);
84 	struct bpf_map *(*map_alloc)(union bpf_attr *attr);
85 	void (*map_release)(struct bpf_map *map, struct file *map_file);
86 	void (*map_free)(struct bpf_map *map);
87 	int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
88 	void (*map_release_uref)(struct bpf_map *map);
89 	void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
90 	int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
91 				union bpf_attr __user *uattr);
92 	int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
93 					  void *value, u64 flags);
94 	int (*map_lookup_and_delete_batch)(struct bpf_map *map,
95 					   const union bpf_attr *attr,
96 					   union bpf_attr __user *uattr);
97 	int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
98 				const union bpf_attr *attr,
99 				union bpf_attr __user *uattr);
100 	int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
101 				union bpf_attr __user *uattr);
102 
103 	/* funcs callable from userspace and from eBPF programs */
104 	void *(*map_lookup_elem)(struct bpf_map *map, void *key);
105 	long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
106 	long (*map_delete_elem)(struct bpf_map *map, void *key);
107 	long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
108 	long (*map_pop_elem)(struct bpf_map *map, void *value);
109 	long (*map_peek_elem)(struct bpf_map *map, void *value);
110 	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
111 
112 	/* funcs called by prog_array and perf_event_array map */
113 	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
114 				int fd);
115 	/* If need_defer is true, the implementation should guarantee that
116 	 * the to-be-put element is still alive before the bpf program, which
117 	 * may manipulate it, exists.
118 	 */
119 	void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);
120 	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
121 	u32 (*map_fd_sys_lookup_elem)(void *ptr);
122 	void (*map_seq_show_elem)(struct bpf_map *map, void *key,
123 				  struct seq_file *m);
124 	int (*map_check_btf)(const struct bpf_map *map,
125 			     const struct btf *btf,
126 			     const struct btf_type *key_type,
127 			     const struct btf_type *value_type);
128 
129 	/* Prog poke tracking helpers. */
130 	int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
131 	void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
132 	void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
133 			     struct bpf_prog *new);
134 
135 	/* Direct value access helpers. */
136 	int (*map_direct_value_addr)(const struct bpf_map *map,
137 				     u64 *imm, u32 off);
138 	int (*map_direct_value_meta)(const struct bpf_map *map,
139 				     u64 imm, u32 *off);
140 	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
141 	__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
142 			     struct poll_table_struct *pts);
143 	unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
144 					       unsigned long len, unsigned long pgoff,
145 					       unsigned long flags);
146 
147 	/* Functions called by bpf_local_storage maps */
148 	int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
149 					void *owner, u32 size);
150 	void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
151 					   void *owner, u32 size);
152 	struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
153 
154 	/* Misc helpers.*/
155 	long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);
156 
157 	/* map_meta_equal must be implemented for maps that can be
158 	 * used as an inner map.  It is a runtime check to ensure
159 	 * an inner map can be inserted to an outer map.
160 	 *
161 	 * Some properties of the inner map has been used during the
162 	 * verification time.  When inserting an inner map at the runtime,
163 	 * map_meta_equal has to ensure the inserting map has the same
164 	 * properties that the verifier has used earlier.
165 	 */
166 	bool (*map_meta_equal)(const struct bpf_map *meta0,
167 			       const struct bpf_map *meta1);
168 
169 
170 	int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
171 					      struct bpf_func_state *caller,
172 					      struct bpf_func_state *callee);
173 	long (*map_for_each_callback)(struct bpf_map *map,
174 				     bpf_callback_t callback_fn,
175 				     void *callback_ctx, u64 flags);
176 
177 	u64 (*map_mem_usage)(const struct bpf_map *map);
178 
179 	/* BTF id of struct allocated by map_alloc */
180 	int *map_btf_id;
181 
182 	/* bpf_iter info used to open a seq_file */
183 	const struct bpf_iter_seq_info *iter_seq_info;
184 };
185 
186 enum {
187 	/* Support at most 11 fields in a BTF type */
188 	BTF_FIELDS_MAX	   = 11,
189 };
190 
191 enum btf_field_type {
192 	BPF_SPIN_LOCK  = (1 << 0),
193 	BPF_TIMER      = (1 << 1),
194 	BPF_KPTR_UNREF = (1 << 2),
195 	BPF_KPTR_REF   = (1 << 3),
196 	BPF_KPTR_PERCPU = (1 << 4),
197 	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
198 	BPF_LIST_HEAD  = (1 << 5),
199 	BPF_LIST_NODE  = (1 << 6),
200 	BPF_RB_ROOT    = (1 << 7),
201 	BPF_RB_NODE    = (1 << 8),
202 	BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
203 	BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
204 	BPF_REFCOUNT   = (1 << 9),
205 	BPF_WORKQUEUE  = (1 << 10),
206 	BPF_UPTR       = (1 << 11),
207 };
208 
209 typedef void (*btf_dtor_kfunc_t)(void *);
210 
211 struct btf_field_kptr {
212 	struct btf *btf;
213 	struct module *module;
214 	/* dtor used if btf_is_kernel(btf), otherwise the type is
215 	 * program-allocated, dtor is NULL,  and __bpf_obj_drop_impl is used
216 	 */
217 	btf_dtor_kfunc_t dtor;
218 	u32 btf_id;
219 };
220 
221 struct btf_field_graph_root {
222 	struct btf *btf;
223 	u32 value_btf_id;
224 	u32 node_offset;
225 	struct btf_record *value_rec;
226 };
227 
228 struct btf_field {
229 	u32 offset;
230 	u32 size;
231 	enum btf_field_type type;
232 	union {
233 		struct btf_field_kptr kptr;
234 		struct btf_field_graph_root graph_root;
235 	};
236 };
237 
238 struct btf_record {
239 	u32 cnt;
240 	u32 field_mask;
241 	int spin_lock_off;
242 	int timer_off;
243 	int wq_off;
244 	int refcount_off;
245 	struct btf_field fields[];
246 };
247 
248 /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
249 struct bpf_rb_node_kern {
250 	struct rb_node rb_node;
251 	void *owner;
252 } __attribute__((aligned(8)));
253 
254 /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
255 struct bpf_list_node_kern {
256 	struct list_head list_head;
257 	void *owner;
258 } __attribute__((aligned(8)));
259 
260 struct bpf_map {
261 	const struct bpf_map_ops *ops;
262 	struct bpf_map *inner_map_meta;
263 #ifdef CONFIG_SECURITY
264 	void *security;
265 #endif
266 	enum bpf_map_type map_type;
267 	u32 key_size;
268 	u32 value_size;
269 	u32 max_entries;
270 	u64 map_extra; /* any per-map-type extra fields */
271 	u32 map_flags;
272 	u32 id;
273 	struct btf_record *record;
274 	int numa_node;
275 	u32 btf_key_type_id;
276 	u32 btf_value_type_id;
277 	u32 btf_vmlinux_value_type_id;
278 	struct btf *btf;
279 #ifdef CONFIG_MEMCG
280 	struct obj_cgroup *objcg;
281 #endif
282 	char name[BPF_OBJ_NAME_LEN];
283 	struct mutex freeze_mutex;
284 	atomic64_t refcnt;
285 	atomic64_t usercnt;
286 	/* rcu is used before freeing and work is only used during freeing */
287 	union {
288 		struct work_struct work;
289 		struct rcu_head rcu;
290 	};
291 	atomic64_t writecnt;
292 	/* 'Ownership' of program-containing map is claimed by the first program
293 	 * that is going to use this map or by the first program which FD is
294 	 * stored in the map to make sure that all callers and callees have the
295 	 * same prog type, JITed flag and xdp_has_frags flag.
296 	 */
297 	struct {
298 		const struct btf_type *attach_func_proto;
299 		spinlock_t lock;
300 		enum bpf_prog_type type;
301 		bool jited;
302 		bool xdp_has_frags;
303 	} owner;
304 	bool bypass_spec_v1;
305 	bool frozen; /* write-once; write-protected by freeze_mutex */
306 	bool free_after_mult_rcu_gp;
307 	bool free_after_rcu_gp;
308 	atomic64_t sleepable_refcnt;
309 	s64 __percpu *elem_count;
310 };
311 
btf_field_type_name(enum btf_field_type type)312 static inline const char *btf_field_type_name(enum btf_field_type type)
313 {
314 	switch (type) {
315 	case BPF_SPIN_LOCK:
316 		return "bpf_spin_lock";
317 	case BPF_TIMER:
318 		return "bpf_timer";
319 	case BPF_WORKQUEUE:
320 		return "bpf_wq";
321 	case BPF_KPTR_UNREF:
322 	case BPF_KPTR_REF:
323 		return "kptr";
324 	case BPF_KPTR_PERCPU:
325 		return "percpu_kptr";
326 	case BPF_UPTR:
327 		return "uptr";
328 	case BPF_LIST_HEAD:
329 		return "bpf_list_head";
330 	case BPF_LIST_NODE:
331 		return "bpf_list_node";
332 	case BPF_RB_ROOT:
333 		return "bpf_rb_root";
334 	case BPF_RB_NODE:
335 		return "bpf_rb_node";
336 	case BPF_REFCOUNT:
337 		return "bpf_refcount";
338 	default:
339 		WARN_ON_ONCE(1);
340 		return "unknown";
341 	}
342 }
343 
btf_field_type_size(enum btf_field_type type)344 static inline u32 btf_field_type_size(enum btf_field_type type)
345 {
346 	switch (type) {
347 	case BPF_SPIN_LOCK:
348 		return sizeof(struct bpf_spin_lock);
349 	case BPF_TIMER:
350 		return sizeof(struct bpf_timer);
351 	case BPF_WORKQUEUE:
352 		return sizeof(struct bpf_wq);
353 	case BPF_KPTR_UNREF:
354 	case BPF_KPTR_REF:
355 	case BPF_KPTR_PERCPU:
356 	case BPF_UPTR:
357 		return sizeof(u64);
358 	case BPF_LIST_HEAD:
359 		return sizeof(struct bpf_list_head);
360 	case BPF_LIST_NODE:
361 		return sizeof(struct bpf_list_node);
362 	case BPF_RB_ROOT:
363 		return sizeof(struct bpf_rb_root);
364 	case BPF_RB_NODE:
365 		return sizeof(struct bpf_rb_node);
366 	case BPF_REFCOUNT:
367 		return sizeof(struct bpf_refcount);
368 	default:
369 		WARN_ON_ONCE(1);
370 		return 0;
371 	}
372 }
373 
btf_field_type_align(enum btf_field_type type)374 static inline u32 btf_field_type_align(enum btf_field_type type)
375 {
376 	switch (type) {
377 	case BPF_SPIN_LOCK:
378 		return __alignof__(struct bpf_spin_lock);
379 	case BPF_TIMER:
380 		return __alignof__(struct bpf_timer);
381 	case BPF_WORKQUEUE:
382 		return __alignof__(struct bpf_wq);
383 	case BPF_KPTR_UNREF:
384 	case BPF_KPTR_REF:
385 	case BPF_KPTR_PERCPU:
386 	case BPF_UPTR:
387 		return __alignof__(u64);
388 	case BPF_LIST_HEAD:
389 		return __alignof__(struct bpf_list_head);
390 	case BPF_LIST_NODE:
391 		return __alignof__(struct bpf_list_node);
392 	case BPF_RB_ROOT:
393 		return __alignof__(struct bpf_rb_root);
394 	case BPF_RB_NODE:
395 		return __alignof__(struct bpf_rb_node);
396 	case BPF_REFCOUNT:
397 		return __alignof__(struct bpf_refcount);
398 	default:
399 		WARN_ON_ONCE(1);
400 		return 0;
401 	}
402 }
403 
bpf_obj_init_field(const struct btf_field * field,void * addr)404 static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
405 {
406 	memset(addr, 0, field->size);
407 
408 	switch (field->type) {
409 	case BPF_REFCOUNT:
410 		refcount_set((refcount_t *)addr, 1);
411 		break;
412 	case BPF_RB_NODE:
413 		RB_CLEAR_NODE((struct rb_node *)addr);
414 		break;
415 	case BPF_LIST_HEAD:
416 	case BPF_LIST_NODE:
417 		INIT_LIST_HEAD((struct list_head *)addr);
418 		break;
419 	case BPF_RB_ROOT:
420 		/* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
421 	case BPF_SPIN_LOCK:
422 	case BPF_TIMER:
423 	case BPF_WORKQUEUE:
424 	case BPF_KPTR_UNREF:
425 	case BPF_KPTR_REF:
426 	case BPF_KPTR_PERCPU:
427 	case BPF_UPTR:
428 		break;
429 	default:
430 		WARN_ON_ONCE(1);
431 		return;
432 	}
433 }
434 
btf_record_has_field(const struct btf_record * rec,enum btf_field_type type)435 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
436 {
437 	if (IS_ERR_OR_NULL(rec))
438 		return false;
439 	return rec->field_mask & type;
440 }
441 
bpf_obj_init(const struct btf_record * rec,void * obj)442 static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
443 {
444 	int i;
445 
446 	if (IS_ERR_OR_NULL(rec))
447 		return;
448 	for (i = 0; i < rec->cnt; i++)
449 		bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
450 }
451 
452 /* 'dst' must be a temporary buffer and should not point to memory that is being
453  * used in parallel by a bpf program or bpf syscall, otherwise the access from
454  * the bpf program or bpf syscall may be corrupted by the reinitialization,
455  * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
456  * allocator, it is still possible for 'dst' to be used in parallel by a bpf
457  * program or bpf syscall.
458  */
check_and_init_map_value(struct bpf_map * map,void * dst)459 static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
460 {
461 	bpf_obj_init(map->record, dst);
462 }
463 
464 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
465  * forced to use 'long' read/writes to try to atomically copy long counters.
466  * Best-effort only.  No barriers here, since it _will_ race with concurrent
467  * updates from BPF programs. Called from bpf syscall and mostly used with
468  * size 8 or 16 bytes, so ask compiler to inline it.
469  */
bpf_long_memcpy(void * dst,const void * src,u32 size)470 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
471 {
472 	const long *lsrc = src;
473 	long *ldst = dst;
474 
475 	size /= sizeof(long);
476 	while (size--)
477 		data_race(*ldst++ = *lsrc++);
478 }
479 
480 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
bpf_obj_memcpy(struct btf_record * rec,void * dst,void * src,u32 size,bool long_memcpy)481 static inline void bpf_obj_memcpy(struct btf_record *rec,
482 				  void *dst, void *src, u32 size,
483 				  bool long_memcpy)
484 {
485 	u32 curr_off = 0;
486 	int i;
487 
488 	if (IS_ERR_OR_NULL(rec)) {
489 		if (long_memcpy)
490 			bpf_long_memcpy(dst, src, round_up(size, 8));
491 		else
492 			memcpy(dst, src, size);
493 		return;
494 	}
495 
496 	for (i = 0; i < rec->cnt; i++) {
497 		u32 next_off = rec->fields[i].offset;
498 		u32 sz = next_off - curr_off;
499 
500 		memcpy(dst + curr_off, src + curr_off, sz);
501 		curr_off += rec->fields[i].size + sz;
502 	}
503 	memcpy(dst + curr_off, src + curr_off, size - curr_off);
504 }
505 
copy_map_value(struct bpf_map * map,void * dst,void * src)506 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
507 {
508 	bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
509 }
510 
copy_map_value_long(struct bpf_map * map,void * dst,void * src)511 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
512 {
513 	bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
514 }
515 
bpf_obj_swap_uptrs(const struct btf_record * rec,void * dst,void * src)516 static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
517 {
518 	unsigned long *src_uptr, *dst_uptr;
519 	const struct btf_field *field;
520 	int i;
521 
522 	if (!btf_record_has_field(rec, BPF_UPTR))
523 		return;
524 
525 	for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
526 		if (field->type != BPF_UPTR)
527 			continue;
528 
529 		src_uptr = src + field->offset;
530 		dst_uptr = dst + field->offset;
531 		swap(*src_uptr, *dst_uptr);
532 	}
533 }
534 
bpf_obj_memzero(struct btf_record * rec,void * dst,u32 size)535 static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
536 {
537 	u32 curr_off = 0;
538 	int i;
539 
540 	if (IS_ERR_OR_NULL(rec)) {
541 		memset(dst, 0, size);
542 		return;
543 	}
544 
545 	for (i = 0; i < rec->cnt; i++) {
546 		u32 next_off = rec->fields[i].offset;
547 		u32 sz = next_off - curr_off;
548 
549 		memset(dst + curr_off, 0, sz);
550 		curr_off += rec->fields[i].size + sz;
551 	}
552 	memset(dst + curr_off, 0, size - curr_off);
553 }
554 
zero_map_value(struct bpf_map * map,void * dst)555 static inline void zero_map_value(struct bpf_map *map, void *dst)
556 {
557 	bpf_obj_memzero(map->record, dst, map->value_size);
558 }
559 
560 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
561 			   bool lock_src);
562 void bpf_timer_cancel_and_free(void *timer);
563 void bpf_wq_cancel_and_free(void *timer);
564 void bpf_list_head_free(const struct btf_field *field, void *list_head,
565 			struct bpf_spin_lock *spin_lock);
566 void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
567 		      struct bpf_spin_lock *spin_lock);
568 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
569 u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
570 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
571 
572 struct bpf_offload_dev;
573 struct bpf_offloaded_map;
574 
575 struct bpf_map_dev_ops {
576 	int (*map_get_next_key)(struct bpf_offloaded_map *map,
577 				void *key, void *next_key);
578 	int (*map_lookup_elem)(struct bpf_offloaded_map *map,
579 			       void *key, void *value);
580 	int (*map_update_elem)(struct bpf_offloaded_map *map,
581 			       void *key, void *value, u64 flags);
582 	int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
583 };
584 
585 struct bpf_offloaded_map {
586 	struct bpf_map map;
587 	struct net_device *netdev;
588 	const struct bpf_map_dev_ops *dev_ops;
589 	void *dev_priv;
590 	struct list_head offloads;
591 };
592 
map_to_offmap(struct bpf_map * map)593 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
594 {
595 	return container_of(map, struct bpf_offloaded_map, map);
596 }
597 
bpf_map_offload_neutral(const struct bpf_map * map)598 static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
599 {
600 	return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
601 }
602 
bpf_map_support_seq_show(const struct bpf_map * map)603 static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
604 {
605 	return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
606 		map->ops->map_seq_show_elem;
607 }
608 
609 int map_check_no_btf(const struct bpf_map *map,
610 		     const struct btf *btf,
611 		     const struct btf_type *key_type,
612 		     const struct btf_type *value_type);
613 
614 bool bpf_map_meta_equal(const struct bpf_map *meta0,
615 			const struct bpf_map *meta1);
616 
617 extern const struct bpf_map_ops bpf_map_offload_ops;
618 
619 /* bpf_type_flag contains a set of flags that are applicable to the values of
620  * arg_type, ret_type and reg_type. For example, a pointer value may be null,
621  * or a memory is read-only. We classify types into two categories: base types
622  * and extended types. Extended types are base types combined with a type flag.
623  *
624  * Currently there are no more than 32 base types in arg_type, ret_type and
625  * reg_types.
626  */
627 #define BPF_BASE_TYPE_BITS	8
628 
629 enum bpf_type_flag {
630 	/* PTR may be NULL. */
631 	PTR_MAYBE_NULL		= BIT(0 + BPF_BASE_TYPE_BITS),
632 
633 	/* MEM is read-only. When applied on bpf_arg, it indicates the arg is
634 	 * compatible with both mutable and immutable memory.
635 	 */
636 	MEM_RDONLY		= BIT(1 + BPF_BASE_TYPE_BITS),
637 
638 	/* MEM points to BPF ring buffer reservation. */
639 	MEM_RINGBUF		= BIT(2 + BPF_BASE_TYPE_BITS),
640 
641 	/* MEM is in user address space. */
642 	MEM_USER		= BIT(3 + BPF_BASE_TYPE_BITS),
643 
644 	/* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
645 	 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
646 	 * order to drop this tag, it must be passed into bpf_per_cpu_ptr()
647 	 * or bpf_this_cpu_ptr(), which will return the pointer corresponding
648 	 * to the specified cpu.
649 	 */
650 	MEM_PERCPU		= BIT(4 + BPF_BASE_TYPE_BITS),
651 
652 	/* Indicates that the argument will be released. */
653 	OBJ_RELEASE		= BIT(5 + BPF_BASE_TYPE_BITS),
654 
655 	/* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
656 	 * unreferenced and referenced kptr loaded from map value using a load
657 	 * instruction, so that they can only be dereferenced but not escape the
658 	 * BPF program into the kernel (i.e. cannot be passed as arguments to
659 	 * kfunc or bpf helpers).
660 	 */
661 	PTR_UNTRUSTED		= BIT(6 + BPF_BASE_TYPE_BITS),
662 
663 	/* MEM can be uninitialized. */
664 	MEM_UNINIT		= BIT(7 + BPF_BASE_TYPE_BITS),
665 
666 	/* DYNPTR points to memory local to the bpf program. */
667 	DYNPTR_TYPE_LOCAL	= BIT(8 + BPF_BASE_TYPE_BITS),
668 
669 	/* DYNPTR points to a kernel-produced ringbuf record. */
670 	DYNPTR_TYPE_RINGBUF	= BIT(9 + BPF_BASE_TYPE_BITS),
671 
672 	/* Size is known at compile time. */
673 	MEM_FIXED_SIZE		= BIT(10 + BPF_BASE_TYPE_BITS),
674 
675 	/* MEM is of an allocated object of type in program BTF. This is used to
676 	 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
677 	 */
678 	MEM_ALLOC		= BIT(11 + BPF_BASE_TYPE_BITS),
679 
680 	/* PTR was passed from the kernel in a trusted context, and may be
681 	 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
682 	 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
683 	 * PTR_UNTRUSTED refers to a kptr that was read directly from a map
684 	 * without invoking bpf_kptr_xchg(). What we really need to know is
685 	 * whether a pointer is safe to pass to a kfunc or BPF helper function.
686 	 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
687 	 * helpers, they do not cover all possible instances of unsafe
688 	 * pointers. For example, a pointer that was obtained from walking a
689 	 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
690 	 * fact that it may be NULL, invalid, etc. This is due to backwards
691 	 * compatibility requirements, as this was the behavior that was first
692 	 * introduced when kptrs were added. The behavior is now considered
693 	 * deprecated, and PTR_UNTRUSTED will eventually be removed.
694 	 *
695 	 * PTR_TRUSTED, on the other hand, is a pointer that the kernel
696 	 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
697 	 * For example, pointers passed to tracepoint arguments are considered
698 	 * PTR_TRUSTED, as are pointers that are passed to struct_ops
699 	 * callbacks. As alluded to above, pointers that are obtained from
700 	 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
701 	 * struct task_struct *task is PTR_TRUSTED, then accessing
702 	 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
703 	 * in a BPF register. Similarly, pointers passed to certain programs
704 	 * types such as kretprobes are not guaranteed to be valid, as they may
705 	 * for example contain an object that was recently freed.
706 	 */
707 	PTR_TRUSTED		= BIT(12 + BPF_BASE_TYPE_BITS),
708 
709 	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
710 	MEM_RCU			= BIT(13 + BPF_BASE_TYPE_BITS),
711 
712 	/* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
713 	 * Currently only valid for linked-list and rbtree nodes. If the nodes
714 	 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
715 	 */
716 	NON_OWN_REF		= BIT(14 + BPF_BASE_TYPE_BITS),
717 
718 	/* DYNPTR points to sk_buff */
719 	DYNPTR_TYPE_SKB		= BIT(15 + BPF_BASE_TYPE_BITS),
720 
721 	/* DYNPTR points to xdp_buff */
722 	DYNPTR_TYPE_XDP		= BIT(16 + BPF_BASE_TYPE_BITS),
723 
724 	/* Memory must be aligned on some architectures, used in combination with
725 	 * MEM_FIXED_SIZE.
726 	 */
727 	MEM_ALIGNED		= BIT(17 + BPF_BASE_TYPE_BITS),
728 
729 	/* MEM is being written to, often combined with MEM_UNINIT. Non-presence
730 	 * of MEM_WRITE means that MEM is only being read. MEM_WRITE without the
731 	 * MEM_UNINIT means that memory needs to be initialized since it is also
732 	 * read.
733 	 */
734 	MEM_WRITE		= BIT(18 + BPF_BASE_TYPE_BITS),
735 
736 	__BPF_TYPE_FLAG_MAX,
737 	__BPF_TYPE_LAST_FLAG	= __BPF_TYPE_FLAG_MAX - 1,
738 };
739 
740 #define DYNPTR_TYPE_FLAG_MASK	(DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
741 				 | DYNPTR_TYPE_XDP)
742 
743 /* Max number of base types. */
744 #define BPF_BASE_TYPE_LIMIT	(1UL << BPF_BASE_TYPE_BITS)
745 
746 /* Max number of all types. */
747 #define BPF_TYPE_LIMIT		(__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
748 
749 /* function argument constraints */
750 enum bpf_arg_type {
751 	ARG_DONTCARE = 0,	/* unused argument in helper function */
752 
753 	/* the following constraints used to prototype
754 	 * bpf_map_lookup/update/delete_elem() functions
755 	 */
756 	ARG_CONST_MAP_PTR,	/* const argument used as pointer to bpf_map */
757 	ARG_PTR_TO_MAP_KEY,	/* pointer to stack used as map key */
758 	ARG_PTR_TO_MAP_VALUE,	/* pointer to stack used as map value */
759 
760 	/* Used to prototype bpf_memcmp() and other functions that access data
761 	 * on eBPF program stack
762 	 */
763 	ARG_PTR_TO_MEM,		/* pointer to valid memory (stack, packet, map value) */
764 	ARG_PTR_TO_ARENA,
765 
766 	ARG_CONST_SIZE,		/* number of bytes accessed from memory */
767 	ARG_CONST_SIZE_OR_ZERO,	/* number of bytes accessed from memory or 0 */
768 
769 	ARG_PTR_TO_CTX,		/* pointer to context */
770 	ARG_ANYTHING,		/* any (initialized) argument is ok */
771 	ARG_PTR_TO_SPIN_LOCK,	/* pointer to bpf_spin_lock */
772 	ARG_PTR_TO_SOCK_COMMON,	/* pointer to sock_common */
773 	ARG_PTR_TO_SOCKET,	/* pointer to bpf_sock (fullsock) */
774 	ARG_PTR_TO_BTF_ID,	/* pointer to in-kernel struct */
775 	ARG_PTR_TO_RINGBUF_MEM,	/* pointer to dynamically reserved ringbuf memory */
776 	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */
777 	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
778 	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */
779 	ARG_PTR_TO_FUNC,	/* pointer to a bpf program function */
780 	ARG_PTR_TO_STACK,	/* pointer to stack */
781 	ARG_PTR_TO_CONST_STR,	/* pointer to a null terminated read-only string */
782 	ARG_PTR_TO_TIMER,	/* pointer to bpf_timer */
783 	ARG_KPTR_XCHG_DEST,	/* pointer to destination that kptrs are bpf_kptr_xchg'd into */
784 	ARG_PTR_TO_DYNPTR,      /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
785 	__BPF_ARG_TYPE_MAX,
786 
787 	/* Extended arg_types. */
788 	ARG_PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
789 	ARG_PTR_TO_MEM_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
790 	ARG_PTR_TO_CTX_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
791 	ARG_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
792 	ARG_PTR_TO_STACK_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
793 	ARG_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
794 	/* Pointer to memory does not need to be initialized, since helper function
795 	 * fills all bytes or clears them in error case.
796 	 */
797 	ARG_PTR_TO_UNINIT_MEM		= MEM_UNINIT | MEM_WRITE | ARG_PTR_TO_MEM,
798 	/* Pointer to valid memory of size known at compile time. */
799 	ARG_PTR_TO_FIXED_SIZE_MEM	= MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
800 
801 	/* This must be the last entry. Its purpose is to ensure the enum is
802 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
803 	 */
804 	__BPF_ARG_TYPE_LIMIT	= BPF_TYPE_LIMIT,
805 };
806 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
807 
808 /* type of values returned from helper functions */
809 enum bpf_return_type {
810 	RET_INTEGER,			/* function returns integer */
811 	RET_VOID,			/* function doesn't return anything */
812 	RET_PTR_TO_MAP_VALUE,		/* returns a pointer to map elem value */
813 	RET_PTR_TO_SOCKET,		/* returns a pointer to a socket */
814 	RET_PTR_TO_TCP_SOCK,		/* returns a pointer to a tcp_sock */
815 	RET_PTR_TO_SOCK_COMMON,		/* returns a pointer to a sock_common */
816 	RET_PTR_TO_MEM,			/* returns a pointer to memory */
817 	RET_PTR_TO_MEM_OR_BTF_ID,	/* returns a pointer to a valid memory or a btf_id */
818 	RET_PTR_TO_BTF_ID,		/* returns a pointer to a btf_id */
819 	__BPF_RET_TYPE_MAX,
820 
821 	/* Extended ret_types. */
822 	RET_PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
823 	RET_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
824 	RET_PTR_TO_TCP_SOCK_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
825 	RET_PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
826 	RET_PTR_TO_RINGBUF_MEM_OR_NULL	= PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
827 	RET_PTR_TO_DYNPTR_MEM_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MEM,
828 	RET_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
829 	RET_PTR_TO_BTF_ID_TRUSTED	= PTR_TRUSTED	 | RET_PTR_TO_BTF_ID,
830 
831 	/* This must be the last entry. Its purpose is to ensure the enum is
832 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
833 	 */
834 	__BPF_RET_TYPE_LIMIT	= BPF_TYPE_LIMIT,
835 };
836 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
837 
838 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
839  * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
840  * instructions after verifying
841  */
842 struct bpf_func_proto {
843 	u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
844 	bool gpl_only;
845 	bool pkt_access;
846 	bool might_sleep;
847 	/* set to true if helper follows contract for llvm
848 	 * attribute bpf_fastcall:
849 	 * - void functions do not scratch r0
850 	 * - functions taking N arguments scratch only registers r1-rN
851 	 */
852 	bool allow_fastcall;
853 	enum bpf_return_type ret_type;
854 	union {
855 		struct {
856 			enum bpf_arg_type arg1_type;
857 			enum bpf_arg_type arg2_type;
858 			enum bpf_arg_type arg3_type;
859 			enum bpf_arg_type arg4_type;
860 			enum bpf_arg_type arg5_type;
861 		};
862 		enum bpf_arg_type arg_type[5];
863 	};
864 	union {
865 		struct {
866 			u32 *arg1_btf_id;
867 			u32 *arg2_btf_id;
868 			u32 *arg3_btf_id;
869 			u32 *arg4_btf_id;
870 			u32 *arg5_btf_id;
871 		};
872 		u32 *arg_btf_id[5];
873 		struct {
874 			size_t arg1_size;
875 			size_t arg2_size;
876 			size_t arg3_size;
877 			size_t arg4_size;
878 			size_t arg5_size;
879 		};
880 		size_t arg_size[5];
881 	};
882 	int *ret_btf_id; /* return value btf_id */
883 	bool (*allowed)(const struct bpf_prog *prog);
884 };
885 
886 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
887  * the first argument to eBPF programs.
888  * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
889  */
890 struct bpf_context;
891 
892 enum bpf_access_type {
893 	BPF_READ = 1,
894 	BPF_WRITE = 2
895 };
896 
897 /* types of values stored in eBPF registers */
898 /* Pointer types represent:
899  * pointer
900  * pointer + imm
901  * pointer + (u16) var
902  * pointer + (u16) var + imm
903  * if (range > 0) then [ptr, ptr + range - off) is safe to access
904  * if (id > 0) means that some 'var' was added
905  * if (off > 0) means that 'imm' was added
906  */
907 enum bpf_reg_type {
908 	NOT_INIT = 0,		 /* nothing was written into register */
909 	SCALAR_VALUE,		 /* reg doesn't contain a valid pointer */
910 	PTR_TO_CTX,		 /* reg points to bpf_context */
911 	CONST_PTR_TO_MAP,	 /* reg points to struct bpf_map */
912 	PTR_TO_MAP_VALUE,	 /* reg points to map element value */
913 	PTR_TO_MAP_KEY,		 /* reg points to a map element key */
914 	PTR_TO_STACK,		 /* reg == frame_pointer + offset */
915 	PTR_TO_PACKET_META,	 /* skb->data - meta_len */
916 	PTR_TO_PACKET,		 /* reg points to skb->data */
917 	PTR_TO_PACKET_END,	 /* skb->data + headlen */
918 	PTR_TO_FLOW_KEYS,	 /* reg points to bpf_flow_keys */
919 	PTR_TO_SOCKET,		 /* reg points to struct bpf_sock */
920 	PTR_TO_SOCK_COMMON,	 /* reg points to sock_common */
921 	PTR_TO_TCP_SOCK,	 /* reg points to struct tcp_sock */
922 	PTR_TO_TP_BUFFER,	 /* reg points to a writable raw tp's buffer */
923 	PTR_TO_XDP_SOCK,	 /* reg points to struct xdp_sock */
924 	/* PTR_TO_BTF_ID points to a kernel struct that does not need
925 	 * to be null checked by the BPF program. This does not imply the
926 	 * pointer is _not_ null and in practice this can easily be a null
927 	 * pointer when reading pointer chains. The assumption is program
928 	 * context will handle null pointer dereference typically via fault
929 	 * handling. The verifier must keep this in mind and can make no
930 	 * assumptions about null or non-null when doing branch analysis.
931 	 * Further, when passed into helpers the helpers can not, without
932 	 * additional context, assume the value is non-null.
933 	 */
934 	PTR_TO_BTF_ID,
935 	PTR_TO_MEM,		 /* reg points to valid memory region */
936 	PTR_TO_ARENA,
937 	PTR_TO_BUF,		 /* reg points to a read/write buffer */
938 	PTR_TO_FUNC,		 /* reg points to a bpf program function */
939 	CONST_PTR_TO_DYNPTR,	 /* reg points to a const struct bpf_dynptr */
940 	__BPF_REG_TYPE_MAX,
941 
942 	/* Extended reg_types. */
943 	PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
944 	PTR_TO_SOCKET_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_SOCKET,
945 	PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
946 	PTR_TO_TCP_SOCK_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
947 	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
948 	 * been checked for null. Used primarily to inform the verifier
949 	 * an explicit null check is required for this struct.
950 	 */
951 	PTR_TO_BTF_ID_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_BTF_ID,
952 
953 	/* This must be the last entry. Its purpose is to ensure the enum is
954 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
955 	 */
956 	__BPF_REG_TYPE_LIMIT	= BPF_TYPE_LIMIT,
957 };
958 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
959 
960 /* The information passed from prog-specific *_is_valid_access
961  * back to the verifier.
962  */
963 struct bpf_insn_access_aux {
964 	enum bpf_reg_type reg_type;
965 	bool is_ldsx;
966 	union {
967 		int ctx_field_size;
968 		struct {
969 			struct btf *btf;
970 			u32 btf_id;
971 		};
972 	};
973 	struct bpf_verifier_log *log; /* for verbose logs */
974 	bool is_retval; /* is accessing function return value ? */
975 };
976 
977 static inline void
bpf_ctx_record_field_size(struct bpf_insn_access_aux * aux,u32 size)978 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
979 {
980 	aux->ctx_field_size = size;
981 }
982 
bpf_is_ldimm64(const struct bpf_insn * insn)983 static bool bpf_is_ldimm64(const struct bpf_insn *insn)
984 {
985 	return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
986 }
987 
bpf_pseudo_func(const struct bpf_insn * insn)988 static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
989 {
990 	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
991 }
992 
993 struct bpf_prog_ops {
994 	int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
995 			union bpf_attr __user *uattr);
996 };
997 
998 struct bpf_reg_state;
999 struct bpf_verifier_ops {
1000 	/* return eBPF function prototype for verification */
1001 	const struct bpf_func_proto *
1002 	(*get_func_proto)(enum bpf_func_id func_id,
1003 			  const struct bpf_prog *prog);
1004 
1005 	/* return true if 'size' wide access at offset 'off' within bpf_context
1006 	 * with 'type' (read or write) is allowed
1007 	 */
1008 	bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
1009 				const struct bpf_prog *prog,
1010 				struct bpf_insn_access_aux *info);
1011 	int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
1012 			    const struct bpf_prog *prog);
1013 	int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,
1014 			    s16 ctx_stack_off);
1015 	int (*gen_ld_abs)(const struct bpf_insn *orig,
1016 			  struct bpf_insn *insn_buf);
1017 	u32 (*convert_ctx_access)(enum bpf_access_type type,
1018 				  const struct bpf_insn *src,
1019 				  struct bpf_insn *dst,
1020 				  struct bpf_prog *prog, u32 *target_size);
1021 	int (*btf_struct_access)(struct bpf_verifier_log *log,
1022 				 const struct bpf_reg_state *reg,
1023 				 int off, int size);
1024 };
1025 
1026 struct bpf_prog_offload_ops {
1027 	/* verifier basic callbacks */
1028 	int (*insn_hook)(struct bpf_verifier_env *env,
1029 			 int insn_idx, int prev_insn_idx);
1030 	int (*finalize)(struct bpf_verifier_env *env);
1031 	/* verifier optimization callbacks (called after .finalize) */
1032 	int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
1033 			    struct bpf_insn *insn);
1034 	int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
1035 	/* program management callbacks */
1036 	int (*prepare)(struct bpf_prog *prog);
1037 	int (*translate)(struct bpf_prog *prog);
1038 	void (*destroy)(struct bpf_prog *prog);
1039 };
1040 
1041 struct bpf_prog_offload {
1042 	struct bpf_prog		*prog;
1043 	struct net_device	*netdev;
1044 	struct bpf_offload_dev	*offdev;
1045 	void			*dev_priv;
1046 	struct list_head	offloads;
1047 	bool			dev_state;
1048 	bool			opt_failed;
1049 	void			*jited_image;
1050 	u32			jited_len;
1051 };
1052 
1053 enum bpf_cgroup_storage_type {
1054 	BPF_CGROUP_STORAGE_SHARED,
1055 	BPF_CGROUP_STORAGE_PERCPU,
1056 	__BPF_CGROUP_STORAGE_MAX
1057 };
1058 
1059 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
1060 
1061 /* The longest tracepoint has 12 args.
1062  * See include/trace/bpf_probe.h
1063  */
1064 #define MAX_BPF_FUNC_ARGS 12
1065 
1066 /* The maximum number of arguments passed through registers
1067  * a single function may have.
1068  */
1069 #define MAX_BPF_FUNC_REG_ARGS 5
1070 
1071 /* The argument is a structure. */
1072 #define BTF_FMODEL_STRUCT_ARG		BIT(0)
1073 
1074 /* The argument is signed. */
1075 #define BTF_FMODEL_SIGNED_ARG		BIT(1)
1076 
1077 struct btf_func_model {
1078 	u8 ret_size;
1079 	u8 ret_flags;
1080 	u8 nr_args;
1081 	u8 arg_size[MAX_BPF_FUNC_ARGS];
1082 	u8 arg_flags[MAX_BPF_FUNC_ARGS];
1083 };
1084 
1085 /* Restore arguments before returning from trampoline to let original function
1086  * continue executing. This flag is used for fentry progs when there are no
1087  * fexit progs.
1088  */
1089 #define BPF_TRAMP_F_RESTORE_REGS	BIT(0)
1090 /* Call original function after fentry progs, but before fexit progs.
1091  * Makes sense for fentry/fexit, normal calls and indirect calls.
1092  */
1093 #define BPF_TRAMP_F_CALL_ORIG		BIT(1)
1094 /* Skip current frame and return to parent.  Makes sense for fentry/fexit
1095  * programs only. Should not be used with normal calls and indirect calls.
1096  */
1097 #define BPF_TRAMP_F_SKIP_FRAME		BIT(2)
1098 /* Store IP address of the caller on the trampoline stack,
1099  * so it's available for trampoline's programs.
1100  */
1101 #define BPF_TRAMP_F_IP_ARG		BIT(3)
1102 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1103 #define BPF_TRAMP_F_RET_FENTRY_RET	BIT(4)
1104 
1105 /* Get original function from stack instead of from provided direct address.
1106  * Makes sense for trampolines with fexit or fmod_ret programs.
1107  */
1108 #define BPF_TRAMP_F_ORIG_STACK		BIT(5)
1109 
1110 /* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1111  * e.g., a live patch. This flag is set and cleared by ftrace call backs,
1112  */
1113 #define BPF_TRAMP_F_SHARE_IPMODIFY	BIT(6)
1114 
1115 /* Indicate that current trampoline is in a tail call context. Then, it has to
1116  * cache and restore tail_call_cnt to avoid infinite tail call loop.
1117  */
1118 #define BPF_TRAMP_F_TAIL_CALL_CTX	BIT(7)
1119 
1120 /*
1121  * Indicate the trampoline should be suitable to receive indirect calls;
1122  * without this indirectly calling the generated code can result in #UD/#CP,
1123  * depending on the CFI options.
1124  *
1125  * Used by bpf_struct_ops.
1126  *
1127  * Incompatible with FENTRY usage, overloads @func_addr argument.
1128  */
1129 #define BPF_TRAMP_F_INDIRECT		BIT(8)
1130 
1131 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1132  * bytes on x86.
1133  */
1134 enum {
1135 #if defined(__s390x__)
1136 	BPF_MAX_TRAMP_LINKS = 27,
1137 #else
1138 	BPF_MAX_TRAMP_LINKS = 38,
1139 #endif
1140 };
1141 
1142 struct bpf_tramp_links {
1143 	struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
1144 	int nr_links;
1145 };
1146 
1147 struct bpf_tramp_run_ctx;
1148 
1149 /* Different use cases for BPF trampoline:
1150  * 1. replace nop at the function entry (kprobe equivalent)
1151  *    flags = BPF_TRAMP_F_RESTORE_REGS
1152  *    fentry = a set of programs to run before returning from trampoline
1153  *
1154  * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1155  *    flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
1156  *    orig_call = fentry_ip + MCOUNT_INSN_SIZE
1157  *    fentry = a set of program to run before calling original function
1158  *    fexit = a set of program to run after original function
1159  *
1160  * 3. replace direct call instruction anywhere in the function body
1161  *    or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1162  *    With flags = 0
1163  *      fentry = a set of programs to run before returning from trampoline
1164  *    With flags = BPF_TRAMP_F_CALL_ORIG
1165  *      orig_call = original callback addr or direct function addr
1166  *      fentry = a set of program to run before calling original function
1167  *      fexit = a set of program to run after original function
1168  */
1169 struct bpf_tramp_image;
1170 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1171 				const struct btf_func_model *m, u32 flags,
1172 				struct bpf_tramp_links *tlinks,
1173 				void *func_addr);
1174 void *arch_alloc_bpf_trampoline(unsigned int size);
1175 void arch_free_bpf_trampoline(void *image, unsigned int size);
1176 int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
1177 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1178 			     struct bpf_tramp_links *tlinks, void *func_addr);
1179 
1180 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1181 					     struct bpf_tramp_run_ctx *run_ctx);
1182 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1183 					     struct bpf_tramp_run_ctx *run_ctx);
1184 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
1185 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
1186 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
1187 				      struct bpf_tramp_run_ctx *run_ctx);
1188 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
1189 				      struct bpf_tramp_run_ctx *run_ctx);
1190 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
1191 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
1192 
1193 struct bpf_ksym {
1194 	unsigned long		 start;
1195 	unsigned long		 end;
1196 	char			 name[KSYM_NAME_LEN];
1197 	struct list_head	 lnode;
1198 	struct latch_tree_node	 tnode;
1199 	bool			 prog;
1200 };
1201 
1202 enum bpf_tramp_prog_type {
1203 	BPF_TRAMP_FENTRY,
1204 	BPF_TRAMP_FEXIT,
1205 	BPF_TRAMP_MODIFY_RETURN,
1206 	BPF_TRAMP_MAX,
1207 	BPF_TRAMP_REPLACE, /* more than MAX */
1208 };
1209 
1210 struct bpf_tramp_image {
1211 	void *image;
1212 	int size;
1213 	struct bpf_ksym ksym;
1214 	struct percpu_ref pcref;
1215 	void *ip_after_call;
1216 	void *ip_epilogue;
1217 	union {
1218 		struct rcu_head rcu;
1219 		struct work_struct work;
1220 	};
1221 };
1222 
1223 struct bpf_trampoline {
1224 	/* hlist for trampoline_table */
1225 	struct hlist_node hlist;
1226 	struct ftrace_ops *fops;
1227 	/* serializes access to fields of this trampoline */
1228 	struct mutex mutex;
1229 	refcount_t refcnt;
1230 	u32 flags;
1231 	u64 key;
1232 	struct {
1233 		struct btf_func_model model;
1234 		void *addr;
1235 		bool ftrace_managed;
1236 	} func;
1237 	/* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1238 	 * program by replacing one of its functions. func.addr is the address
1239 	 * of the function it replaced.
1240 	 */
1241 	struct bpf_prog *extension_prog;
1242 	/* list of BPF programs using this trampoline */
1243 	struct hlist_head progs_hlist[BPF_TRAMP_MAX];
1244 	/* Number of attached programs. A counter per kind. */
1245 	int progs_cnt[BPF_TRAMP_MAX];
1246 	/* Executable image of trampoline */
1247 	struct bpf_tramp_image *cur_image;
1248 };
1249 
1250 struct bpf_attach_target_info {
1251 	struct btf_func_model fmodel;
1252 	long tgt_addr;
1253 	struct module *tgt_mod;
1254 	const char *tgt_name;
1255 	const struct btf_type *tgt_type;
1256 };
1257 
1258 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1259 
1260 struct bpf_dispatcher_prog {
1261 	struct bpf_prog *prog;
1262 	refcount_t users;
1263 };
1264 
1265 struct bpf_dispatcher {
1266 	/* dispatcher mutex */
1267 	struct mutex mutex;
1268 	void *func;
1269 	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
1270 	int num_progs;
1271 	void *image;
1272 	void *rw_image;
1273 	u32 image_off;
1274 	struct bpf_ksym ksym;
1275 #ifdef CONFIG_HAVE_STATIC_CALL
1276 	struct static_call_key *sc_key;
1277 	void *sc_tramp;
1278 #endif
1279 };
1280 
1281 #ifndef __bpfcall
1282 #define __bpfcall __nocfi
1283 #endif
1284 
bpf_dispatcher_nop_func(const void * ctx,const struct bpf_insn * insnsi,bpf_func_t bpf_func)1285 static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(
1286 	const void *ctx,
1287 	const struct bpf_insn *insnsi,
1288 	bpf_func_t bpf_func)
1289 {
1290 	return bpf_func(ctx, insnsi);
1291 }
1292 
1293 /* the implementation of the opaque uapi struct bpf_dynptr */
1294 struct bpf_dynptr_kern {
1295 	void *data;
1296 	/* Size represents the number of usable bytes of dynptr data.
1297 	 * If for example the offset is at 4 for a local dynptr whose data is
1298 	 * of type u64, the number of usable bytes is 4.
1299 	 *
1300 	 * The upper 8 bits are reserved. It is as follows:
1301 	 * Bits 0 - 23 = size
1302 	 * Bits 24 - 30 = dynptr type
1303 	 * Bit 31 = whether dynptr is read-only
1304 	 */
1305 	u32 size;
1306 	u32 offset;
1307 } __aligned(8);
1308 
1309 enum bpf_dynptr_type {
1310 	BPF_DYNPTR_TYPE_INVALID,
1311 	/* Points to memory that is local to the bpf program */
1312 	BPF_DYNPTR_TYPE_LOCAL,
1313 	/* Underlying data is a ringbuf record */
1314 	BPF_DYNPTR_TYPE_RINGBUF,
1315 	/* Underlying data is a sk_buff */
1316 	BPF_DYNPTR_TYPE_SKB,
1317 	/* Underlying data is a xdp_buff */
1318 	BPF_DYNPTR_TYPE_XDP,
1319 };
1320 
1321 int bpf_dynptr_check_size(u32 size);
1322 u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
1323 const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
1324 void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
1325 bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
1326 
1327 #ifdef CONFIG_BPF_JIT
1328 int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1329 			     struct bpf_trampoline *tr,
1330 			     struct bpf_prog *tgt_prog);
1331 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1332 			       struct bpf_trampoline *tr,
1333 			       struct bpf_prog *tgt_prog);
1334 struct bpf_trampoline *bpf_trampoline_get(u64 key,
1335 					  struct bpf_attach_target_info *tgt_info);
1336 void bpf_trampoline_put(struct bpf_trampoline *tr);
1337 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
1338 
1339 /*
1340  * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1341  * indirection with a direct call to the bpf program. If the architecture does
1342  * not have STATIC_CALL, avoid a double-indirection.
1343  */
1344 #ifdef CONFIG_HAVE_STATIC_CALL
1345 
1346 #define __BPF_DISPATCHER_SC_INIT(_name)				\
1347 	.sc_key = &STATIC_CALL_KEY(_name),			\
1348 	.sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1349 
1350 #define __BPF_DISPATCHER_SC(name)				\
1351 	DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1352 
1353 #define __BPF_DISPATCHER_CALL(name)				\
1354 	static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1355 
1356 #define __BPF_DISPATCHER_UPDATE(_d, _new)			\
1357 	__static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1358 
1359 #else
1360 #define __BPF_DISPATCHER_SC_INIT(name)
1361 #define __BPF_DISPATCHER_SC(name)
1362 #define __BPF_DISPATCHER_CALL(name)		bpf_func(ctx, insnsi)
1363 #define __BPF_DISPATCHER_UPDATE(_d, _new)
1364 #endif
1365 
1366 #define BPF_DISPATCHER_INIT(_name) {				\
1367 	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\
1368 	.func = &_name##_func,					\
1369 	.progs = {},						\
1370 	.num_progs = 0,						\
1371 	.image = NULL,						\
1372 	.image_off = 0,						\
1373 	.ksym = {						\
1374 		.name  = #_name,				\
1375 		.lnode = LIST_HEAD_INIT(_name.ksym.lnode),	\
1376 	},							\
1377 	__BPF_DISPATCHER_SC_INIT(_name##_call)			\
1378 }
1379 
1380 #define DEFINE_BPF_DISPATCHER(name)					\
1381 	__BPF_DISPATCHER_SC(name);					\
1382 	noinline __bpfcall unsigned int bpf_dispatcher_##name##_func(	\
1383 		const void *ctx,					\
1384 		const struct bpf_insn *insnsi,				\
1385 		bpf_func_t bpf_func)					\
1386 	{								\
1387 		return __BPF_DISPATCHER_CALL(name);			\
1388 	}								\
1389 	EXPORT_SYMBOL(bpf_dispatcher_##name##_func);			\
1390 	struct bpf_dispatcher bpf_dispatcher_##name =			\
1391 		BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1392 
1393 #define DECLARE_BPF_DISPATCHER(name)					\
1394 	unsigned int bpf_dispatcher_##name##_func(			\
1395 		const void *ctx,					\
1396 		const struct bpf_insn *insnsi,				\
1397 		bpf_func_t bpf_func);					\
1398 	extern struct bpf_dispatcher bpf_dispatcher_##name;
1399 
1400 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1401 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1402 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1403 				struct bpf_prog *to);
1404 /* Called only from JIT-enabled code, so there's no need for stubs. */
1405 void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
1406 void bpf_image_ksym_add(struct bpf_ksym *ksym);
1407 void bpf_image_ksym_del(struct bpf_ksym *ksym);
1408 void bpf_ksym_add(struct bpf_ksym *ksym);
1409 void bpf_ksym_del(struct bpf_ksym *ksym);
1410 int bpf_jit_charge_modmem(u32 size);
1411 void bpf_jit_uncharge_modmem(u32 size);
1412 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1413 #else
bpf_trampoline_link_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr,struct bpf_prog * tgt_prog)1414 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1415 					   struct bpf_trampoline *tr,
1416 					   struct bpf_prog *tgt_prog)
1417 {
1418 	return -ENOTSUPP;
1419 }
bpf_trampoline_unlink_prog(struct bpf_tramp_link * link,struct bpf_trampoline * tr,struct bpf_prog * tgt_prog)1420 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1421 					     struct bpf_trampoline *tr,
1422 					     struct bpf_prog *tgt_prog)
1423 {
1424 	return -ENOTSUPP;
1425 }
bpf_trampoline_get(u64 key,struct bpf_attach_target_info * tgt_info)1426 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
1427 							struct bpf_attach_target_info *tgt_info)
1428 {
1429 	return NULL;
1430 }
bpf_trampoline_put(struct bpf_trampoline * tr)1431 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
1432 #define DEFINE_BPF_DISPATCHER(name)
1433 #define DECLARE_BPF_DISPATCHER(name)
1434 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1435 #define BPF_DISPATCHER_PTR(name) NULL
bpf_dispatcher_change_prog(struct bpf_dispatcher * d,struct bpf_prog * from,struct bpf_prog * to)1436 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
1437 					      struct bpf_prog *from,
1438 					      struct bpf_prog *to) {}
is_bpf_image_address(unsigned long address)1439 static inline bool is_bpf_image_address(unsigned long address)
1440 {
1441 	return false;
1442 }
bpf_prog_has_trampoline(const struct bpf_prog * prog)1443 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
1444 {
1445 	return false;
1446 }
1447 #endif
1448 
1449 struct bpf_func_info_aux {
1450 	u16 linkage;
1451 	bool unreliable;
1452 	bool called : 1;
1453 	bool verified : 1;
1454 };
1455 
1456 enum bpf_jit_poke_reason {
1457 	BPF_POKE_REASON_TAIL_CALL,
1458 };
1459 
1460 /* Descriptor of pokes pointing /into/ the JITed image. */
1461 struct bpf_jit_poke_descriptor {
1462 	void *tailcall_target;
1463 	void *tailcall_bypass;
1464 	void *bypass_addr;
1465 	void *aux;
1466 	union {
1467 		struct {
1468 			struct bpf_map *map;
1469 			u32 key;
1470 		} tail_call;
1471 	};
1472 	bool tailcall_target_stable;
1473 	u8 adj_off;
1474 	u16 reason;
1475 	u32 insn_idx;
1476 };
1477 
1478 /* reg_type info for ctx arguments */
1479 struct bpf_ctx_arg_aux {
1480 	u32 offset;
1481 	enum bpf_reg_type reg_type;
1482 	struct btf *btf;
1483 	u32 btf_id;
1484 };
1485 
1486 struct btf_mod_pair {
1487 	struct btf *btf;
1488 	struct module *module;
1489 };
1490 
1491 struct bpf_kfunc_desc_tab;
1492 
1493 struct bpf_prog_aux {
1494 	atomic64_t refcnt;
1495 	u32 used_map_cnt;
1496 	u32 used_btf_cnt;
1497 	u32 max_ctx_offset;
1498 	u32 max_pkt_offset;
1499 	u32 max_tp_access;
1500 	u32 stack_depth;
1501 	u32 id;
1502 	u32 func_cnt; /* used by non-func prog as the number of func progs */
1503 	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1504 	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1505 	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
1506 	u32 ctx_arg_info_size;
1507 	u32 max_rdonly_access;
1508 	u32 max_rdwr_access;
1509 	struct btf *attach_btf;
1510 	const struct bpf_ctx_arg_aux *ctx_arg_info;
1511 	void __percpu *priv_stack_ptr;
1512 	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
1513 	struct bpf_prog *dst_prog;
1514 	struct bpf_trampoline *dst_trampoline;
1515 	enum bpf_prog_type saved_dst_prog_type;
1516 	enum bpf_attach_type saved_dst_attach_type;
1517 	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
1518 	bool dev_bound; /* Program is bound to the netdev. */
1519 	bool offload_requested; /* Program is bound and offloaded to the netdev. */
1520 	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
1521 	bool attach_tracing_prog; /* true if tracing another tracing program */
1522 	bool func_proto_unreliable;
1523 	bool tail_call_reachable;
1524 	bool xdp_has_frags;
1525 	bool exception_cb;
1526 	bool exception_boundary;
1527 	bool is_extended; /* true if extended by freplace program */
1528 	bool jits_use_priv_stack;
1529 	bool priv_stack_requested;
1530 	bool changes_pkt_data;
1531 	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
1532 	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
1533 	struct bpf_arena *arena;
1534 	void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
1535 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1536 	const struct btf_type *attach_func_proto;
1537 	/* function name for valid attach_btf_id */
1538 	const char *attach_func_name;
1539 	struct bpf_prog **func;
1540 	void *jit_data; /* JIT specific data. arch dependent */
1541 	struct bpf_jit_poke_descriptor *poke_tab;
1542 	struct bpf_kfunc_desc_tab *kfunc_tab;
1543 	struct bpf_kfunc_btf_tab *kfunc_btf_tab;
1544 	u32 size_poke_tab;
1545 #ifdef CONFIG_FINEIBT
1546 	struct bpf_ksym ksym_prefix;
1547 #endif
1548 	struct bpf_ksym ksym;
1549 	const struct bpf_prog_ops *ops;
1550 	struct bpf_map **used_maps;
1551 	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1552 	struct btf_mod_pair *used_btfs;
1553 	struct bpf_prog *prog;
1554 	struct user_struct *user;
1555 	u64 load_time; /* ns since boottime */
1556 	u32 verified_insns;
1557 	int cgroup_atype; /* enum cgroup_bpf_attach_type */
1558 	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1559 	char name[BPF_OBJ_NAME_LEN];
1560 	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
1561 #ifdef CONFIG_SECURITY
1562 	void *security;
1563 #endif
1564 	struct bpf_token *token;
1565 	struct bpf_prog_offload *offload;
1566 	struct btf *btf;
1567 	struct bpf_func_info *func_info;
1568 	struct bpf_func_info_aux *func_info_aux;
1569 	/* bpf_line_info loaded from userspace.  linfo->insn_off
1570 	 * has the xlated insn offset.
1571 	 * Both the main and sub prog share the same linfo.
1572 	 * The subprog can access its first linfo by
1573 	 * using the linfo_idx.
1574 	 */
1575 	struct bpf_line_info *linfo;
1576 	/* jited_linfo is the jited addr of the linfo.  It has a
1577 	 * one to one mapping to linfo:
1578 	 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1579 	 * Both the main and sub prog share the same jited_linfo.
1580 	 * The subprog can access its first jited_linfo by
1581 	 * using the linfo_idx.
1582 	 */
1583 	void **jited_linfo;
1584 	u32 func_info_cnt;
1585 	u32 nr_linfo;
1586 	/* subprog can use linfo_idx to access its first linfo and
1587 	 * jited_linfo.
1588 	 * main prog always has linfo_idx == 0
1589 	 */
1590 	u32 linfo_idx;
1591 	struct module *mod;
1592 	u32 num_exentries;
1593 	struct exception_table_entry *extable;
1594 	union {
1595 		struct work_struct work;
1596 		struct rcu_head	rcu;
1597 	};
1598 };
1599 
1600 struct bpf_prog {
1601 	u16			pages;		/* Number of allocated pages */
1602 	u16			jited:1,	/* Is our filter JIT'ed? */
1603 				jit_requested:1,/* archs need to JIT the prog */
1604 				gpl_compatible:1, /* Is filter GPL compatible? */
1605 				cb_access:1,	/* Is control block accessed? */
1606 				dst_needed:1,	/* Do we need dst entry? */
1607 				blinding_requested:1, /* needs constant blinding */
1608 				blinded:1,	/* Was blinded */
1609 				is_func:1,	/* program is a bpf function */
1610 				kprobe_override:1, /* Do we override a kprobe? */
1611 				has_callchain_buf:1, /* callchain buffer allocated? */
1612 				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
1613 				call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
1614 				call_get_func_ip:1, /* Do we call get_func_ip() */
1615 				tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
1616 				sleepable:1;	/* BPF program is sleepable */
1617 	enum bpf_prog_type	type;		/* Type of BPF program */
1618 	enum bpf_attach_type	expected_attach_type; /* For some prog types */
1619 	u32			len;		/* Number of filter blocks */
1620 	u32			jited_len;	/* Size of jited insns in bytes */
1621 	u8			tag[BPF_TAG_SIZE];
1622 	struct bpf_prog_stats __percpu *stats;
1623 	int __percpu		*active;
1624 	unsigned int		(*bpf_func)(const void *ctx,
1625 					    const struct bpf_insn *insn);
1626 	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
1627 	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
1628 	/* Instructions for interpreter */
1629 	union {
1630 		DECLARE_FLEX_ARRAY(struct sock_filter, insns);
1631 		DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
1632 	};
1633 };
1634 
1635 struct bpf_array_aux {
1636 	/* Programs with direct jumps into programs part of this array. */
1637 	struct list_head poke_progs;
1638 	struct bpf_map *map;
1639 	struct mutex poke_mutex;
1640 	struct work_struct work;
1641 };
1642 
1643 struct bpf_link {
1644 	atomic64_t refcnt;
1645 	u32 id;
1646 	enum bpf_link_type type;
1647 	const struct bpf_link_ops *ops;
1648 	struct bpf_prog *prog;
1649 	/* whether BPF link itself has "sleepable" semantics, which can differ
1650 	 * from underlying BPF program having a "sleepable" semantics, as BPF
1651 	 * link's semantics is determined by target attach hook
1652 	 */
1653 	bool sleepable;
1654 	/* rcu is used before freeing, work can be used to schedule that
1655 	 * RCU-based freeing before that, so they never overlap
1656 	 */
1657 	union {
1658 		struct rcu_head rcu;
1659 		struct work_struct work;
1660 	};
1661 };
1662 
1663 struct bpf_link_ops {
1664 	void (*release)(struct bpf_link *link);
1665 	/* deallocate link resources callback, called without RCU grace period
1666 	 * waiting
1667 	 */
1668 	void (*dealloc)(struct bpf_link *link);
1669 	/* deallocate link resources callback, called after RCU grace period;
1670 	 * if either the underlying BPF program is sleepable or BPF link's
1671 	 * target hook is sleepable, we'll go through tasks trace RCU GP and
1672 	 * then "classic" RCU GP; this need for chaining tasks trace and
1673 	 * classic RCU GPs is designated by setting bpf_link->sleepable flag
1674 	 */
1675 	void (*dealloc_deferred)(struct bpf_link *link);
1676 	int (*detach)(struct bpf_link *link);
1677 	int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
1678 			   struct bpf_prog *old_prog);
1679 	void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
1680 	int (*fill_link_info)(const struct bpf_link *link,
1681 			      struct bpf_link_info *info);
1682 	int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
1683 			  struct bpf_map *old_map);
1684 	__poll_t (*poll)(struct file *file, struct poll_table_struct *pts);
1685 };
1686 
1687 struct bpf_tramp_link {
1688 	struct bpf_link link;
1689 	struct hlist_node tramp_hlist;
1690 	u64 cookie;
1691 };
1692 
1693 struct bpf_shim_tramp_link {
1694 	struct bpf_tramp_link link;
1695 	struct bpf_trampoline *trampoline;
1696 };
1697 
1698 struct bpf_tracing_link {
1699 	struct bpf_tramp_link link;
1700 	enum bpf_attach_type attach_type;
1701 	struct bpf_trampoline *trampoline;
1702 	struct bpf_prog *tgt_prog;
1703 };
1704 
1705 struct bpf_raw_tp_link {
1706 	struct bpf_link link;
1707 	struct bpf_raw_event_map *btp;
1708 	u64 cookie;
1709 };
1710 
1711 struct bpf_link_primer {
1712 	struct bpf_link *link;
1713 	struct file *file;
1714 	int fd;
1715 	u32 id;
1716 };
1717 
1718 struct bpf_mount_opts {
1719 	kuid_t uid;
1720 	kgid_t gid;
1721 	umode_t mode;
1722 
1723 	/* BPF token-related delegation options */
1724 	u64 delegate_cmds;
1725 	u64 delegate_maps;
1726 	u64 delegate_progs;
1727 	u64 delegate_attachs;
1728 };
1729 
1730 struct bpf_token {
1731 	struct work_struct work;
1732 	atomic64_t refcnt;
1733 	struct user_namespace *userns;
1734 	u64 allowed_cmds;
1735 	u64 allowed_maps;
1736 	u64 allowed_progs;
1737 	u64 allowed_attachs;
1738 #ifdef CONFIG_SECURITY
1739 	void *security;
1740 #endif
1741 };
1742 
1743 struct bpf_struct_ops_value;
1744 struct btf_member;
1745 
1746 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
1747 /**
1748  * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
1749  *			   define a BPF_MAP_TYPE_STRUCT_OPS map type composed
1750  *			   of BPF_PROG_TYPE_STRUCT_OPS progs.
1751  * @verifier_ops: A structure of callbacks that are invoked by the verifier
1752  *		  when determining whether the struct_ops progs in the
1753  *		  struct_ops map are valid.
1754  * @init: A callback that is invoked a single time, and before any other
1755  *	  callback, to initialize the structure. A nonzero return value means
1756  *	  the subsystem could not be initialized.
1757  * @check_member: When defined, a callback invoked by the verifier to allow
1758  *		  the subsystem to determine if an entry in the struct_ops map
1759  *		  is valid. A nonzero return value means that the map is
1760  *		  invalid and should be rejected by the verifier.
1761  * @init_member: A callback that is invoked for each member of the struct_ops
1762  *		 map to allow the subsystem to initialize the member. A nonzero
1763  *		 value means the member could not be initialized. This callback
1764  *		 is exclusive with the @type, @type_id, @value_type, and
1765  *		 @value_id fields.
1766  * @reg: A callback that is invoked when the struct_ops map has been
1767  *	 initialized and is being attached to. Zero means the struct_ops map
1768  *	 has been successfully registered and is live. A nonzero return value
1769  *	 means the struct_ops map could not be registered.
1770  * @unreg: A callback that is invoked when the struct_ops map should be
1771  *	   unregistered.
1772  * @update: A callback that is invoked when the live struct_ops map is being
1773  *	    updated to contain new values. This callback is only invoked when
1774  *	    the struct_ops map is loaded with BPF_F_LINK. If not defined, the
1775  *	    it is assumed that the struct_ops map cannot be updated.
1776  * @validate: A callback that is invoked after all of the members have been
1777  *	      initialized. This callback should perform static checks on the
1778  *	      map, meaning that it should either fail or succeed
1779  *	      deterministically. A struct_ops map that has been validated may
1780  *	      not necessarily succeed in being registered if the call to @reg
1781  *	      fails. For example, a valid struct_ops map may be loaded, but
1782  *	      then fail to be registered due to there being another active
1783  *	      struct_ops map on the system in the subsystem already. For this
1784  *	      reason, if this callback is not defined, the check is skipped as
1785  *	      the struct_ops map will have final verification performed in
1786  *	      @reg.
1787  * @type: BTF type.
1788  * @value_type: Value type.
1789  * @name: The name of the struct bpf_struct_ops object.
1790  * @func_models: Func models
1791  * @type_id: BTF type id.
1792  * @value_id: BTF value id.
1793  */
1794 struct bpf_struct_ops {
1795 	const struct bpf_verifier_ops *verifier_ops;
1796 	int (*init)(struct btf *btf);
1797 	int (*check_member)(const struct btf_type *t,
1798 			    const struct btf_member *member,
1799 			    const struct bpf_prog *prog);
1800 	int (*init_member)(const struct btf_type *t,
1801 			   const struct btf_member *member,
1802 			   void *kdata, const void *udata);
1803 	int (*reg)(void *kdata, struct bpf_link *link);
1804 	void (*unreg)(void *kdata, struct bpf_link *link);
1805 	int (*update)(void *kdata, void *old_kdata, struct bpf_link *link);
1806 	int (*validate)(void *kdata);
1807 	void *cfi_stubs;
1808 	struct module *owner;
1809 	const char *name;
1810 	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
1811 };
1812 
1813 /* Every member of a struct_ops type has an instance even a member is not
1814  * an operator (function pointer). The "info" field will be assigned to
1815  * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the
1816  * argument information required by the verifier to verify the program.
1817  *
1818  * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the
1819  * corresponding entry for an given argument.
1820  */
1821 struct bpf_struct_ops_arg_info {
1822 	struct bpf_ctx_arg_aux *info;
1823 	u32 cnt;
1824 };
1825 
1826 struct bpf_struct_ops_desc {
1827 	struct bpf_struct_ops *st_ops;
1828 
1829 	const struct btf_type *type;
1830 	const struct btf_type *value_type;
1831 	u32 type_id;
1832 	u32 value_id;
1833 
1834 	/* Collection of argument information for each member */
1835 	struct bpf_struct_ops_arg_info *arg_info;
1836 };
1837 
1838 enum bpf_struct_ops_state {
1839 	BPF_STRUCT_OPS_STATE_INIT,
1840 	BPF_STRUCT_OPS_STATE_INUSE,
1841 	BPF_STRUCT_OPS_STATE_TOBEFREE,
1842 	BPF_STRUCT_OPS_STATE_READY,
1843 };
1844 
1845 struct bpf_struct_ops_common_value {
1846 	refcount_t refcnt;
1847 	enum bpf_struct_ops_state state;
1848 };
1849 
1850 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
1851 /* This macro helps developer to register a struct_ops type and generate
1852  * type information correctly. Developers should use this macro to register
1853  * a struct_ops type instead of calling __register_bpf_struct_ops() directly.
1854  */
1855 #define register_bpf_struct_ops(st_ops, type)				\
1856 	({								\
1857 		struct bpf_struct_ops_##type {				\
1858 			struct bpf_struct_ops_common_value common;	\
1859 			struct type data ____cacheline_aligned_in_smp;	\
1860 		};							\
1861 		BTF_TYPE_EMIT(struct bpf_struct_ops_##type);		\
1862 		__register_bpf_struct_ops(st_ops);			\
1863 	})
1864 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1865 bool bpf_struct_ops_get(const void *kdata);
1866 void bpf_struct_ops_put(const void *kdata);
1867 int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);
1868 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
1869 				       void *value);
1870 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
1871 				      struct bpf_tramp_link *link,
1872 				      const struct btf_func_model *model,
1873 				      void *stub_func,
1874 				      void **image, u32 *image_off,
1875 				      bool allow_alloc);
1876 void bpf_struct_ops_image_free(void *image);
bpf_try_module_get(const void * data,struct module * owner)1877 static inline bool bpf_try_module_get(const void *data, struct module *owner)
1878 {
1879 	if (owner == BPF_MODULE_OWNER)
1880 		return bpf_struct_ops_get(data);
1881 	else
1882 		return try_module_get(owner);
1883 }
bpf_module_put(const void * data,struct module * owner)1884 static inline void bpf_module_put(const void *data, struct module *owner)
1885 {
1886 	if (owner == BPF_MODULE_OWNER)
1887 		bpf_struct_ops_put(data);
1888 	else
1889 		module_put(owner);
1890 }
1891 int bpf_struct_ops_link_create(union bpf_attr *attr);
1892 
1893 #ifdef CONFIG_NET
1894 /* Define it here to avoid the use of forward declaration */
1895 struct bpf_dummy_ops_state {
1896 	int val;
1897 };
1898 
1899 struct bpf_dummy_ops {
1900 	int (*test_1)(struct bpf_dummy_ops_state *cb);
1901 	int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
1902 		      char a3, unsigned long a4);
1903 	int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
1904 };
1905 
1906 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
1907 			    union bpf_attr __user *uattr);
1908 #endif
1909 int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
1910 			     struct btf *btf,
1911 			     struct bpf_verifier_log *log);
1912 void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);
1913 void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);
1914 #else
1915 #define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })
bpf_try_module_get(const void * data,struct module * owner)1916 static inline bool bpf_try_module_get(const void *data, struct module *owner)
1917 {
1918 	return try_module_get(owner);
1919 }
bpf_module_put(const void * data,struct module * owner)1920 static inline void bpf_module_put(const void *data, struct module *owner)
1921 {
1922 	module_put(owner);
1923 }
bpf_struct_ops_supported(const struct bpf_struct_ops * st_ops,u32 moff)1924 static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
1925 {
1926 	return -ENOTSUPP;
1927 }
bpf_struct_ops_map_sys_lookup_elem(struct bpf_map * map,void * key,void * value)1928 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
1929 						     void *key,
1930 						     void *value)
1931 {
1932 	return -EINVAL;
1933 }
bpf_struct_ops_link_create(union bpf_attr * attr)1934 static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
1935 {
1936 	return -EOPNOTSUPP;
1937 }
bpf_map_struct_ops_info_fill(struct bpf_map_info * info,struct bpf_map * map)1938 static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1939 {
1940 }
1941 
bpf_struct_ops_desc_release(struct bpf_struct_ops_desc * st_ops_desc)1942 static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
1943 {
1944 }
1945 
1946 #endif
1947 
1948 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1949 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1950 				    int cgroup_atype);
1951 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
1952 #else
bpf_trampoline_link_cgroup_shim(struct bpf_prog * prog,int cgroup_atype)1953 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1954 						  int cgroup_atype)
1955 {
1956 	return -EOPNOTSUPP;
1957 }
bpf_trampoline_unlink_cgroup_shim(struct bpf_prog * prog)1958 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
1959 {
1960 }
1961 #endif
1962 
1963 struct bpf_array {
1964 	struct bpf_map map;
1965 	u32 elem_size;
1966 	u32 index_mask;
1967 	struct bpf_array_aux *aux;
1968 	union {
1969 		DECLARE_FLEX_ARRAY(char, value) __aligned(8);
1970 		DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
1971 		DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
1972 	};
1973 };
1974 
1975 #define BPF_COMPLEXITY_LIMIT_INSNS      1000000 /* yes. 1M insns */
1976 #define MAX_TAIL_CALL_CNT 33
1977 
1978 /* Maximum number of loops for bpf_loop and bpf_iter_num.
1979  * It's enum to expose it (and thus make it discoverable) through BTF.
1980  */
1981 enum {
1982 	BPF_MAX_LOOPS = 8 * 1024 * 1024,
1983 };
1984 
1985 #define BPF_F_ACCESS_MASK	(BPF_F_RDONLY |		\
1986 				 BPF_F_RDONLY_PROG |	\
1987 				 BPF_F_WRONLY |		\
1988 				 BPF_F_WRONLY_PROG)
1989 
1990 #define BPF_MAP_CAN_READ	BIT(0)
1991 #define BPF_MAP_CAN_WRITE	BIT(1)
1992 
1993 /* Maximum number of user-producer ring buffer samples that can be drained in
1994  * a call to bpf_user_ringbuf_drain().
1995  */
1996 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
1997 
bpf_map_flags_to_cap(struct bpf_map * map)1998 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
1999 {
2000 	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
2001 
2002 	/* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
2003 	 * not possible.
2004 	 */
2005 	if (access_flags & BPF_F_RDONLY_PROG)
2006 		return BPF_MAP_CAN_READ;
2007 	else if (access_flags & BPF_F_WRONLY_PROG)
2008 		return BPF_MAP_CAN_WRITE;
2009 	else
2010 		return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
2011 }
2012 
bpf_map_flags_access_ok(u32 access_flags)2013 static inline bool bpf_map_flags_access_ok(u32 access_flags)
2014 {
2015 	return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
2016 	       (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
2017 }
2018 
2019 struct bpf_event_entry {
2020 	struct perf_event *event;
2021 	struct file *perf_file;
2022 	struct file *map_file;
2023 	struct rcu_head rcu;
2024 };
2025 
map_type_contains_progs(struct bpf_map * map)2026 static inline bool map_type_contains_progs(struct bpf_map *map)
2027 {
2028 	return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
2029 	       map->map_type == BPF_MAP_TYPE_DEVMAP ||
2030 	       map->map_type == BPF_MAP_TYPE_CPUMAP;
2031 }
2032 
2033 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);
2034 int bpf_prog_calc_tag(struct bpf_prog *fp);
2035 
2036 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
2037 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
2038 
2039 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
2040 					unsigned long off, unsigned long len);
2041 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
2042 					const struct bpf_insn *src,
2043 					struct bpf_insn *dst,
2044 					struct bpf_prog *prog,
2045 					u32 *target_size);
2046 
2047 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
2048 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
2049 
2050 /* an array of programs to be executed under rcu_lock.
2051  *
2052  * Typical usage:
2053  * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
2054  *
2055  * the structure returned by bpf_prog_array_alloc() should be populated
2056  * with program pointers and the last pointer must be NULL.
2057  * The user has to keep refcnt on the program and make sure the program
2058  * is removed from the array before bpf_prog_put().
2059  * The 'struct bpf_prog_array *' should only be replaced with xchg()
2060  * since other cpus are walking the array of pointers in parallel.
2061  */
2062 struct bpf_prog_array_item {
2063 	struct bpf_prog *prog;
2064 	union {
2065 		struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
2066 		u64 bpf_cookie;
2067 	};
2068 };
2069 
2070 struct bpf_prog_array {
2071 	struct rcu_head rcu;
2072 	struct bpf_prog_array_item items[];
2073 };
2074 
2075 struct bpf_empty_prog_array {
2076 	struct bpf_prog_array hdr;
2077 	struct bpf_prog *null_prog;
2078 };
2079 
2080 /* to avoid allocating empty bpf_prog_array for cgroups that
2081  * don't have bpf program attached use one global 'bpf_empty_prog_array'
2082  * It will not be modified the caller of bpf_prog_array_alloc()
2083  * (since caller requested prog_cnt == 0)
2084  * that pointer should be 'freed' by bpf_prog_array_free()
2085  */
2086 extern struct bpf_empty_prog_array bpf_empty_prog_array;
2087 
2088 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
2089 void bpf_prog_array_free(struct bpf_prog_array *progs);
2090 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
2091 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
2092 int bpf_prog_array_length(struct bpf_prog_array *progs);
2093 bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
2094 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
2095 				__u32 __user *prog_ids, u32 cnt);
2096 
2097 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
2098 				struct bpf_prog *old_prog);
2099 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
2100 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
2101 			     struct bpf_prog *prog);
2102 int bpf_prog_array_copy_info(struct bpf_prog_array *array,
2103 			     u32 *prog_ids, u32 request_cnt,
2104 			     u32 *prog_cnt);
2105 int bpf_prog_array_copy(struct bpf_prog_array *old_array,
2106 			struct bpf_prog *exclude_prog,
2107 			struct bpf_prog *include_prog,
2108 			u64 bpf_cookie,
2109 			struct bpf_prog_array **new_array);
2110 
2111 struct bpf_run_ctx {};
2112 
2113 struct bpf_cg_run_ctx {
2114 	struct bpf_run_ctx run_ctx;
2115 	const struct bpf_prog_array_item *prog_item;
2116 	int retval;
2117 };
2118 
2119 struct bpf_trace_run_ctx {
2120 	struct bpf_run_ctx run_ctx;
2121 	u64 bpf_cookie;
2122 	bool is_uprobe;
2123 };
2124 
2125 struct bpf_tramp_run_ctx {
2126 	struct bpf_run_ctx run_ctx;
2127 	u64 bpf_cookie;
2128 	struct bpf_run_ctx *saved_run_ctx;
2129 };
2130 
bpf_set_run_ctx(struct bpf_run_ctx * new_ctx)2131 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
2132 {
2133 	struct bpf_run_ctx *old_ctx = NULL;
2134 
2135 #ifdef CONFIG_BPF_SYSCALL
2136 	old_ctx = current->bpf_ctx;
2137 	current->bpf_ctx = new_ctx;
2138 #endif
2139 	return old_ctx;
2140 }
2141 
bpf_reset_run_ctx(struct bpf_run_ctx * old_ctx)2142 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
2143 {
2144 #ifdef CONFIG_BPF_SYSCALL
2145 	current->bpf_ctx = old_ctx;
2146 #endif
2147 }
2148 
2149 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
2150 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE			(1 << 0)
2151 /* BPF program asks to set CN on the packet. */
2152 #define BPF_RET_SET_CN						(1 << 0)
2153 
2154 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);
2155 
2156 static __always_inline u32
bpf_prog_run_array(const struct bpf_prog_array * array,const void * ctx,bpf_prog_run_fn run_prog)2157 bpf_prog_run_array(const struct bpf_prog_array *array,
2158 		   const void *ctx, bpf_prog_run_fn run_prog)
2159 {
2160 	const struct bpf_prog_array_item *item;
2161 	const struct bpf_prog *prog;
2162 	struct bpf_run_ctx *old_run_ctx;
2163 	struct bpf_trace_run_ctx run_ctx;
2164 	u32 ret = 1;
2165 
2166 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
2167 
2168 	if (unlikely(!array))
2169 		return ret;
2170 
2171 	run_ctx.is_uprobe = false;
2172 
2173 	migrate_disable();
2174 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2175 	item = &array->items[0];
2176 	while ((prog = READ_ONCE(item->prog))) {
2177 		run_ctx.bpf_cookie = item->bpf_cookie;
2178 		ret &= run_prog(prog, ctx);
2179 		item++;
2180 	}
2181 	bpf_reset_run_ctx(old_run_ctx);
2182 	migrate_enable();
2183 	return ret;
2184 }
2185 
2186 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
2187  *
2188  * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
2189  * overall. As a result, we must use the bpf_prog_array_free_sleepable
2190  * in order to use the tasks_trace rcu grace period.
2191  *
2192  * When a non-sleepable program is inside the array, we take the rcu read
2193  * section and disable preemption for that program alone, so it can access
2194  * rcu-protected dynamically sized maps.
2195  */
2196 static __always_inline u32
bpf_prog_run_array_uprobe(const struct bpf_prog_array * array,const void * ctx,bpf_prog_run_fn run_prog)2197 bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,
2198 			  const void *ctx, bpf_prog_run_fn run_prog)
2199 {
2200 	const struct bpf_prog_array_item *item;
2201 	const struct bpf_prog *prog;
2202 	struct bpf_run_ctx *old_run_ctx;
2203 	struct bpf_trace_run_ctx run_ctx;
2204 	u32 ret = 1;
2205 
2206 	might_fault();
2207 	RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held");
2208 
2209 	if (unlikely(!array))
2210 		return ret;
2211 
2212 	migrate_disable();
2213 
2214 	run_ctx.is_uprobe = true;
2215 
2216 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2217 	item = &array->items[0];
2218 	while ((prog = READ_ONCE(item->prog))) {
2219 		if (!prog->sleepable)
2220 			rcu_read_lock();
2221 
2222 		run_ctx.bpf_cookie = item->bpf_cookie;
2223 		ret &= run_prog(prog, ctx);
2224 		item++;
2225 
2226 		if (!prog->sleepable)
2227 			rcu_read_unlock();
2228 	}
2229 	bpf_reset_run_ctx(old_run_ctx);
2230 	migrate_enable();
2231 	return ret;
2232 }
2233 
2234 #ifdef CONFIG_BPF_SYSCALL
2235 DECLARE_PER_CPU(int, bpf_prog_active);
2236 extern struct mutex bpf_stats_enabled_mutex;
2237 
2238 /*
2239  * Block execution of BPF programs attached to instrumentation (perf,
2240  * kprobes, tracepoints) to prevent deadlocks on map operations as any of
2241  * these events can happen inside a region which holds a map bucket lock
2242  * and can deadlock on it.
2243  */
bpf_disable_instrumentation(void)2244 static inline void bpf_disable_instrumentation(void)
2245 {
2246 	migrate_disable();
2247 	this_cpu_inc(bpf_prog_active);
2248 }
2249 
bpf_enable_instrumentation(void)2250 static inline void bpf_enable_instrumentation(void)
2251 {
2252 	this_cpu_dec(bpf_prog_active);
2253 	migrate_enable();
2254 }
2255 
2256 extern const struct super_operations bpf_super_ops;
2257 extern const struct file_operations bpf_map_fops;
2258 extern const struct file_operations bpf_prog_fops;
2259 extern const struct file_operations bpf_iter_fops;
2260 
2261 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2262 	extern const struct bpf_prog_ops _name ## _prog_ops; \
2263 	extern const struct bpf_verifier_ops _name ## _verifier_ops;
2264 #define BPF_MAP_TYPE(_id, _ops) \
2265 	extern const struct bpf_map_ops _ops;
2266 #define BPF_LINK_TYPE(_id, _name)
2267 #include <linux/bpf_types.h>
2268 #undef BPF_PROG_TYPE
2269 #undef BPF_MAP_TYPE
2270 #undef BPF_LINK_TYPE
2271 
2272 extern const struct bpf_prog_ops bpf_offload_prog_ops;
2273 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
2274 extern const struct bpf_verifier_ops xdp_analyzer_ops;
2275 
2276 struct bpf_prog *bpf_prog_get(u32 ufd);
2277 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2278 				       bool attach_drv);
2279 void bpf_prog_add(struct bpf_prog *prog, int i);
2280 void bpf_prog_sub(struct bpf_prog *prog, int i);
2281 void bpf_prog_inc(struct bpf_prog *prog);
2282 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
2283 void bpf_prog_put(struct bpf_prog *prog);
2284 
2285 void bpf_prog_free_id(struct bpf_prog *prog);
2286 void bpf_map_free_id(struct bpf_map *map);
2287 
2288 struct btf_field *btf_record_find(const struct btf_record *rec,
2289 				  u32 offset, u32 field_mask);
2290 void btf_record_free(struct btf_record *rec);
2291 void bpf_map_free_record(struct bpf_map *map);
2292 struct btf_record *btf_record_dup(const struct btf_record *rec);
2293 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
2294 void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
2295 void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
2296 void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
2297 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
2298 
2299 struct bpf_map *bpf_map_get(u32 ufd);
2300 struct bpf_map *bpf_map_get_with_uref(u32 ufd);
2301 
2302 /*
2303  * The __bpf_map_get() and __btf_get_by_fd() functions parse a file
2304  * descriptor and return a corresponding map or btf object.
2305  * Their names are double underscored to emphasize the fact that they
2306  * do not increase refcnt. To also increase refcnt use corresponding
2307  * bpf_map_get() and btf_get_by_fd() functions.
2308  */
2309 
__bpf_map_get(struct fd f)2310 static inline struct bpf_map *__bpf_map_get(struct fd f)
2311 {
2312 	if (fd_empty(f))
2313 		return ERR_PTR(-EBADF);
2314 	if (unlikely(fd_file(f)->f_op != &bpf_map_fops))
2315 		return ERR_PTR(-EINVAL);
2316 	return fd_file(f)->private_data;
2317 }
2318 
__btf_get_by_fd(struct fd f)2319 static inline struct btf *__btf_get_by_fd(struct fd f)
2320 {
2321 	if (fd_empty(f))
2322 		return ERR_PTR(-EBADF);
2323 	if (unlikely(fd_file(f)->f_op != &btf_fops))
2324 		return ERR_PTR(-EINVAL);
2325 	return fd_file(f)->private_data;
2326 }
2327 
2328 void bpf_map_inc(struct bpf_map *map);
2329 void bpf_map_inc_with_uref(struct bpf_map *map);
2330 struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
2331 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
2332 void bpf_map_put_with_uref(struct bpf_map *map);
2333 void bpf_map_put(struct bpf_map *map);
2334 void *bpf_map_area_alloc(u64 size, int numa_node);
2335 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
2336 void bpf_map_area_free(void *base);
2337 bool bpf_map_write_active(const struct bpf_map *map);
2338 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
2339 int  generic_map_lookup_batch(struct bpf_map *map,
2340 			      const union bpf_attr *attr,
2341 			      union bpf_attr __user *uattr);
2342 int  generic_map_update_batch(struct bpf_map *map, struct file *map_file,
2343 			      const union bpf_attr *attr,
2344 			      union bpf_attr __user *uattr);
2345 int  generic_map_delete_batch(struct bpf_map *map,
2346 			      const union bpf_attr *attr,
2347 			      union bpf_attr __user *uattr);
2348 struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
2349 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
2350 
2351 int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid,
2352 			unsigned long nr_pages, struct page **page_array);
2353 #ifdef CONFIG_MEMCG
2354 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2355 			   int node);
2356 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
2357 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
2358 		       gfp_t flags);
2359 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
2360 				    size_t align, gfp_t flags);
2361 #else
2362 /*
2363  * These specialized allocators have to be macros for their allocations to be
2364  * accounted separately (to have separate alloc_tag).
2365  */
2366 #define bpf_map_kmalloc_node(_map, _size, _flags, _node)	\
2367 		kmalloc_node(_size, _flags, _node)
2368 #define bpf_map_kzalloc(_map, _size, _flags)			\
2369 		kzalloc(_size, _flags)
2370 #define bpf_map_kvcalloc(_map, _n, _size, _flags)		\
2371 		kvcalloc(_n, _size, _flags)
2372 #define bpf_map_alloc_percpu(_map, _size, _align, _flags)	\
2373 		__alloc_percpu_gfp(_size, _align, _flags)
2374 #endif
2375 
2376 static inline int
bpf_map_init_elem_count(struct bpf_map * map)2377 bpf_map_init_elem_count(struct bpf_map *map)
2378 {
2379 	size_t size = sizeof(*map->elem_count), align = size;
2380 	gfp_t flags = GFP_USER | __GFP_NOWARN;
2381 
2382 	map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
2383 	if (!map->elem_count)
2384 		return -ENOMEM;
2385 
2386 	return 0;
2387 }
2388 
2389 static inline void
bpf_map_free_elem_count(struct bpf_map * map)2390 bpf_map_free_elem_count(struct bpf_map *map)
2391 {
2392 	free_percpu(map->elem_count);
2393 }
2394 
bpf_map_inc_elem_count(struct bpf_map * map)2395 static inline void bpf_map_inc_elem_count(struct bpf_map *map)
2396 {
2397 	this_cpu_inc(*map->elem_count);
2398 }
2399 
bpf_map_dec_elem_count(struct bpf_map * map)2400 static inline void bpf_map_dec_elem_count(struct bpf_map *map)
2401 {
2402 	this_cpu_dec(*map->elem_count);
2403 }
2404 
2405 extern int sysctl_unprivileged_bpf_disabled;
2406 
2407 bool bpf_token_capable(const struct bpf_token *token, int cap);
2408 
bpf_allow_ptr_leaks(const struct bpf_token * token)2409 static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)
2410 {
2411 	return bpf_token_capable(token, CAP_PERFMON);
2412 }
2413 
bpf_allow_uninit_stack(const struct bpf_token * token)2414 static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
2415 {
2416 	return bpf_token_capable(token, CAP_PERFMON);
2417 }
2418 
bpf_bypass_spec_v1(const struct bpf_token * token)2419 static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
2420 {
2421 	return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
2422 }
2423 
bpf_bypass_spec_v4(const struct bpf_token * token)2424 static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
2425 {
2426 	return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
2427 }
2428 
2429 int bpf_map_new_fd(struct bpf_map *map, int flags);
2430 int bpf_prog_new_fd(struct bpf_prog *prog);
2431 
2432 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2433 		   const struct bpf_link_ops *ops, struct bpf_prog *prog);
2434 void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
2435 			     const struct bpf_link_ops *ops, struct bpf_prog *prog,
2436 			     bool sleepable);
2437 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
2438 int bpf_link_settle(struct bpf_link_primer *primer);
2439 void bpf_link_cleanup(struct bpf_link_primer *primer);
2440 void bpf_link_inc(struct bpf_link *link);
2441 struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link);
2442 void bpf_link_put(struct bpf_link *link);
2443 int bpf_link_new_fd(struct bpf_link *link);
2444 struct bpf_link *bpf_link_get_from_fd(u32 ufd);
2445 struct bpf_link *bpf_link_get_curr_or_next(u32 *id);
2446 
2447 void bpf_token_inc(struct bpf_token *token);
2448 void bpf_token_put(struct bpf_token *token);
2449 int bpf_token_create(union bpf_attr *attr);
2450 struct bpf_token *bpf_token_get_from_fd(u32 ufd);
2451 
2452 bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
2453 bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
2454 bool bpf_token_allow_prog_type(const struct bpf_token *token,
2455 			       enum bpf_prog_type prog_type,
2456 			       enum bpf_attach_type attach_type);
2457 
2458 int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
2459 int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
2460 struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,
2461 			    umode_t mode);
2462 
2463 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
2464 #define DEFINE_BPF_ITER_FUNC(target, args...)			\
2465 	extern int bpf_iter_ ## target(args);			\
2466 	int __init bpf_iter_ ## target(args) { return 0; }
2467 
2468 /*
2469  * The task type of iterators.
2470  *
2471  * For BPF task iterators, they can be parameterized with various
2472  * parameters to visit only some of tasks.
2473  *
2474  * BPF_TASK_ITER_ALL (default)
2475  *	Iterate over resources of every task.
2476  *
2477  * BPF_TASK_ITER_TID
2478  *	Iterate over resources of a task/tid.
2479  *
2480  * BPF_TASK_ITER_TGID
2481  *	Iterate over resources of every task of a process / task group.
2482  */
2483 enum bpf_iter_task_type {
2484 	BPF_TASK_ITER_ALL = 0,
2485 	BPF_TASK_ITER_TID,
2486 	BPF_TASK_ITER_TGID,
2487 };
2488 
2489 struct bpf_iter_aux_info {
2490 	/* for map_elem iter */
2491 	struct bpf_map *map;
2492 
2493 	/* for cgroup iter */
2494 	struct {
2495 		struct cgroup *start; /* starting cgroup */
2496 		enum bpf_cgroup_iter_order order;
2497 	} cgroup;
2498 	struct {
2499 		enum bpf_iter_task_type	type;
2500 		u32 pid;
2501 	} task;
2502 };
2503 
2504 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
2505 					union bpf_iter_link_info *linfo,
2506 					struct bpf_iter_aux_info *aux);
2507 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
2508 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
2509 					struct seq_file *seq);
2510 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
2511 					 struct bpf_link_info *info);
2512 typedef const struct bpf_func_proto *
2513 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
2514 			     const struct bpf_prog *prog);
2515 
2516 enum bpf_iter_feature {
2517 	BPF_ITER_RESCHED	= BIT(0),
2518 };
2519 
2520 #define BPF_ITER_CTX_ARG_MAX 2
2521 struct bpf_iter_reg {
2522 	const char *target;
2523 	bpf_iter_attach_target_t attach_target;
2524 	bpf_iter_detach_target_t detach_target;
2525 	bpf_iter_show_fdinfo_t show_fdinfo;
2526 	bpf_iter_fill_link_info_t fill_link_info;
2527 	bpf_iter_get_func_proto_t get_func_proto;
2528 	u32 ctx_arg_info_size;
2529 	u32 feature;
2530 	struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
2531 	const struct bpf_iter_seq_info *seq_info;
2532 };
2533 
2534 struct bpf_iter_meta {
2535 	__bpf_md_ptr(struct seq_file *, seq);
2536 	u64 session_id;
2537 	u64 seq_num;
2538 };
2539 
2540 struct bpf_iter__bpf_map_elem {
2541 	__bpf_md_ptr(struct bpf_iter_meta *, meta);
2542 	__bpf_md_ptr(struct bpf_map *, map);
2543 	__bpf_md_ptr(void *, key);
2544 	__bpf_md_ptr(void *, value);
2545 };
2546 
2547 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
2548 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
2549 bool bpf_iter_prog_supported(struct bpf_prog *prog);
2550 const struct bpf_func_proto *
2551 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
2552 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
2553 int bpf_iter_new_fd(struct bpf_link *link);
2554 bool bpf_link_is_iter(struct bpf_link *link);
2555 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
2556 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
2557 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
2558 			      struct seq_file *seq);
2559 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
2560 				struct bpf_link_info *info);
2561 
2562 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
2563 				   struct bpf_func_state *caller,
2564 				   struct bpf_func_state *callee);
2565 
2566 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
2567 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
2568 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
2569 			   u64 flags);
2570 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
2571 			    u64 flags);
2572 
2573 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
2574 
2575 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
2576 				 void *key, void *value, u64 map_flags);
2577 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2578 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
2579 				void *key, void *value, u64 map_flags);
2580 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2581 
2582 int bpf_get_file_flag(int flags);
2583 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
2584 			     size_t actual_size);
2585 
2586 /* verify correctness of eBPF program */
2587 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
2588 
2589 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
2590 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
2591 #endif
2592 
2593 struct btf *bpf_get_btf_vmlinux(void);
2594 
2595 /* Map specifics */
2596 struct xdp_frame;
2597 struct sk_buff;
2598 struct bpf_dtab_netdev;
2599 struct bpf_cpu_map_entry;
2600 
2601 void __dev_flush(struct list_head *flush_list);
2602 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2603 		    struct net_device *dev_rx);
2604 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2605 		    struct net_device *dev_rx);
2606 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2607 			  struct bpf_map *map, bool exclude_ingress);
2608 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
2609 			     const struct bpf_prog *xdp_prog);
2610 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2611 			   const struct bpf_prog *xdp_prog,
2612 			   struct bpf_map *map, bool exclude_ingress);
2613 
2614 void __cpu_map_flush(struct list_head *flush_list);
2615 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
2616 		    struct net_device *dev_rx);
2617 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2618 			     struct sk_buff *skb);
2619 
2620 /* Return map's numa specified by userspace */
bpf_map_attr_numa_node(const union bpf_attr * attr)2621 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
2622 {
2623 	return (attr->map_flags & BPF_F_NUMA_NODE) ?
2624 		attr->numa_node : NUMA_NO_NODE;
2625 }
2626 
2627 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
2628 int array_map_alloc_check(union bpf_attr *attr);
2629 
2630 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
2631 			  union bpf_attr __user *uattr);
2632 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
2633 			  union bpf_attr __user *uattr);
2634 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2635 			      const union bpf_attr *kattr,
2636 			      union bpf_attr __user *uattr);
2637 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2638 				     const union bpf_attr *kattr,
2639 				     union bpf_attr __user *uattr);
2640 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
2641 			     const union bpf_attr *kattr,
2642 			     union bpf_attr __user *uattr);
2643 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2644 				const union bpf_attr *kattr,
2645 				union bpf_attr __user *uattr);
2646 int bpf_prog_test_run_nf(struct bpf_prog *prog,
2647 			 const union bpf_attr *kattr,
2648 			 union bpf_attr __user *uattr);
2649 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
2650 		    const struct bpf_prog *prog,
2651 		    struct bpf_insn_access_aux *info);
2652 
bpf_tracing_ctx_access(int off,int size,enum bpf_access_type type)2653 static inline bool bpf_tracing_ctx_access(int off, int size,
2654 					  enum bpf_access_type type)
2655 {
2656 	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
2657 		return false;
2658 	if (type != BPF_READ)
2659 		return false;
2660 	if (off % size != 0)
2661 		return false;
2662 	return true;
2663 }
2664 
bpf_tracing_btf_ctx_access(int off,int size,enum bpf_access_type type,const struct bpf_prog * prog,struct bpf_insn_access_aux * info)2665 static inline bool bpf_tracing_btf_ctx_access(int off, int size,
2666 					      enum bpf_access_type type,
2667 					      const struct bpf_prog *prog,
2668 					      struct bpf_insn_access_aux *info)
2669 {
2670 	if (!bpf_tracing_ctx_access(off, size, type))
2671 		return false;
2672 	return btf_ctx_access(off, size, type, prog, info);
2673 }
2674 
2675 int btf_struct_access(struct bpf_verifier_log *log,
2676 		      const struct bpf_reg_state *reg,
2677 		      int off, int size, enum bpf_access_type atype,
2678 		      u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
2679 bool btf_struct_ids_match(struct bpf_verifier_log *log,
2680 			  const struct btf *btf, u32 id, int off,
2681 			  const struct btf *need_btf, u32 need_type_id,
2682 			  bool strict);
2683 
2684 int btf_distill_func_proto(struct bpf_verifier_log *log,
2685 			   struct btf *btf,
2686 			   const struct btf_type *func_proto,
2687 			   const char *func_name,
2688 			   struct btf_func_model *m);
2689 
2690 struct bpf_reg_state;
2691 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
2692 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
2693 			 struct btf *btf, const struct btf_type *t);
2694 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
2695 				    int comp_idx, const char *tag_key);
2696 int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,
2697 			   int comp_idx, const char *tag_key, int last_id);
2698 
2699 struct bpf_prog *bpf_prog_by_id(u32 id);
2700 struct bpf_link *bpf_link_by_id(u32 id);
2701 
2702 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
2703 						 const struct bpf_prog *prog);
2704 void bpf_task_storage_free(struct task_struct *task);
2705 void bpf_cgrp_storage_free(struct cgroup *cgroup);
2706 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
2707 const struct btf_func_model *
2708 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2709 			 const struct bpf_insn *insn);
2710 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2711 		       u16 btf_fd_idx, u8 **func_addr);
2712 
2713 struct bpf_core_ctx {
2714 	struct bpf_verifier_log *log;
2715 	const struct btf *btf;
2716 };
2717 
2718 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
2719 				const struct bpf_reg_state *reg,
2720 				const char *field_name, u32 btf_id, const char *suffix);
2721 
2722 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
2723 			       const struct btf *reg_btf, u32 reg_id,
2724 			       const struct btf *arg_btf, u32 arg_id);
2725 
2726 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
2727 		   int relo_idx, void *insn);
2728 
unprivileged_ebpf_enabled(void)2729 static inline bool unprivileged_ebpf_enabled(void)
2730 {
2731 	return !sysctl_unprivileged_bpf_disabled;
2732 }
2733 
2734 /* Not all bpf prog type has the bpf_ctx.
2735  * For the bpf prog type that has initialized the bpf_ctx,
2736  * this function can be used to decide if a kernel function
2737  * is called by a bpf program.
2738  */
has_current_bpf_ctx(void)2739 static inline bool has_current_bpf_ctx(void)
2740 {
2741 	return !!current->bpf_ctx;
2742 }
2743 
2744 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
2745 
2746 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2747 		     enum bpf_dynptr_type type, u32 offset, u32 size);
2748 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
2749 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
2750 
2751 #else /* !CONFIG_BPF_SYSCALL */
bpf_prog_get(u32 ufd)2752 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
2753 {
2754 	return ERR_PTR(-EOPNOTSUPP);
2755 }
2756 
bpf_prog_get_type_dev(u32 ufd,enum bpf_prog_type type,bool attach_drv)2757 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
2758 						     enum bpf_prog_type type,
2759 						     bool attach_drv)
2760 {
2761 	return ERR_PTR(-EOPNOTSUPP);
2762 }
2763 
bpf_prog_add(struct bpf_prog * prog,int i)2764 static inline void bpf_prog_add(struct bpf_prog *prog, int i)
2765 {
2766 }
2767 
bpf_prog_sub(struct bpf_prog * prog,int i)2768 static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
2769 {
2770 }
2771 
bpf_prog_put(struct bpf_prog * prog)2772 static inline void bpf_prog_put(struct bpf_prog *prog)
2773 {
2774 }
2775 
bpf_prog_inc(struct bpf_prog * prog)2776 static inline void bpf_prog_inc(struct bpf_prog *prog)
2777 {
2778 }
2779 
2780 static inline struct bpf_prog *__must_check
bpf_prog_inc_not_zero(struct bpf_prog * prog)2781 bpf_prog_inc_not_zero(struct bpf_prog *prog)
2782 {
2783 	return ERR_PTR(-EOPNOTSUPP);
2784 }
2785 
bpf_link_init(struct bpf_link * link,enum bpf_link_type type,const struct bpf_link_ops * ops,struct bpf_prog * prog)2786 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2787 				 const struct bpf_link_ops *ops,
2788 				 struct bpf_prog *prog)
2789 {
2790 }
2791 
bpf_link_init_sleepable(struct bpf_link * link,enum bpf_link_type type,const struct bpf_link_ops * ops,struct bpf_prog * prog,bool sleepable)2792 static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
2793 					   const struct bpf_link_ops *ops, struct bpf_prog *prog,
2794 					   bool sleepable)
2795 {
2796 }
2797 
bpf_link_prime(struct bpf_link * link,struct bpf_link_primer * primer)2798 static inline int bpf_link_prime(struct bpf_link *link,
2799 				 struct bpf_link_primer *primer)
2800 {
2801 	return -EOPNOTSUPP;
2802 }
2803 
bpf_link_settle(struct bpf_link_primer * primer)2804 static inline int bpf_link_settle(struct bpf_link_primer *primer)
2805 {
2806 	return -EOPNOTSUPP;
2807 }
2808 
bpf_link_cleanup(struct bpf_link_primer * primer)2809 static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
2810 {
2811 }
2812 
bpf_link_inc(struct bpf_link * link)2813 static inline void bpf_link_inc(struct bpf_link *link)
2814 {
2815 }
2816 
bpf_link_inc_not_zero(struct bpf_link * link)2817 static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
2818 {
2819 	return NULL;
2820 }
2821 
bpf_link_put(struct bpf_link * link)2822 static inline void bpf_link_put(struct bpf_link *link)
2823 {
2824 }
2825 
bpf_obj_get_user(const char __user * pathname,int flags)2826 static inline int bpf_obj_get_user(const char __user *pathname, int flags)
2827 {
2828 	return -EOPNOTSUPP;
2829 }
2830 
bpf_token_capable(const struct bpf_token * token,int cap)2831 static inline bool bpf_token_capable(const struct bpf_token *token, int cap)
2832 {
2833 	return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));
2834 }
2835 
bpf_token_inc(struct bpf_token * token)2836 static inline void bpf_token_inc(struct bpf_token *token)
2837 {
2838 }
2839 
bpf_token_put(struct bpf_token * token)2840 static inline void bpf_token_put(struct bpf_token *token)
2841 {
2842 }
2843 
bpf_token_get_from_fd(u32 ufd)2844 static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
2845 {
2846 	return ERR_PTR(-EOPNOTSUPP);
2847 }
2848 
__dev_flush(struct list_head * flush_list)2849 static inline void __dev_flush(struct list_head *flush_list)
2850 {
2851 }
2852 
2853 struct xdp_frame;
2854 struct bpf_dtab_netdev;
2855 struct bpf_cpu_map_entry;
2856 
2857 static inline
dev_xdp_enqueue(struct net_device * dev,struct xdp_frame * xdpf,struct net_device * dev_rx)2858 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2859 		    struct net_device *dev_rx)
2860 {
2861 	return 0;
2862 }
2863 
2864 static inline
dev_map_enqueue(struct bpf_dtab_netdev * dst,struct xdp_frame * xdpf,struct net_device * dev_rx)2865 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2866 		    struct net_device *dev_rx)
2867 {
2868 	return 0;
2869 }
2870 
2871 static inline
dev_map_enqueue_multi(struct xdp_frame * xdpf,struct net_device * dev_rx,struct bpf_map * map,bool exclude_ingress)2872 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2873 			  struct bpf_map *map, bool exclude_ingress)
2874 {
2875 	return 0;
2876 }
2877 
2878 struct sk_buff;
2879 
dev_map_generic_redirect(struct bpf_dtab_netdev * dst,struct sk_buff * skb,const struct bpf_prog * xdp_prog)2880 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
2881 					   struct sk_buff *skb,
2882 					   const struct bpf_prog *xdp_prog)
2883 {
2884 	return 0;
2885 }
2886 
2887 static inline
dev_map_redirect_multi(struct net_device * dev,struct sk_buff * skb,const struct bpf_prog * xdp_prog,struct bpf_map * map,bool exclude_ingress)2888 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2889 			   const struct bpf_prog *xdp_prog,
2890 			   struct bpf_map *map, bool exclude_ingress)
2891 {
2892 	return 0;
2893 }
2894 
__cpu_map_flush(struct list_head * flush_list)2895 static inline void __cpu_map_flush(struct list_head *flush_list)
2896 {
2897 }
2898 
cpu_map_enqueue(struct bpf_cpu_map_entry * rcpu,struct xdp_frame * xdpf,struct net_device * dev_rx)2899 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
2900 				  struct xdp_frame *xdpf,
2901 				  struct net_device *dev_rx)
2902 {
2903 	return 0;
2904 }
2905 
cpu_map_generic_redirect(struct bpf_cpu_map_entry * rcpu,struct sk_buff * skb)2906 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2907 					   struct sk_buff *skb)
2908 {
2909 	return -EOPNOTSUPP;
2910 }
2911 
bpf_prog_get_type_path(const char * name,enum bpf_prog_type type)2912 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
2913 				enum bpf_prog_type type)
2914 {
2915 	return ERR_PTR(-EOPNOTSUPP);
2916 }
2917 
bpf_prog_test_run_xdp(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2918 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
2919 					const union bpf_attr *kattr,
2920 					union bpf_attr __user *uattr)
2921 {
2922 	return -ENOTSUPP;
2923 }
2924 
bpf_prog_test_run_skb(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2925 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
2926 					const union bpf_attr *kattr,
2927 					union bpf_attr __user *uattr)
2928 {
2929 	return -ENOTSUPP;
2930 }
2931 
bpf_prog_test_run_tracing(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2932 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2933 					    const union bpf_attr *kattr,
2934 					    union bpf_attr __user *uattr)
2935 {
2936 	return -ENOTSUPP;
2937 }
2938 
bpf_prog_test_run_flow_dissector(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2939 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2940 						   const union bpf_attr *kattr,
2941 						   union bpf_attr __user *uattr)
2942 {
2943 	return -ENOTSUPP;
2944 }
2945 
bpf_prog_test_run_sk_lookup(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)2946 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2947 					      const union bpf_attr *kattr,
2948 					      union bpf_attr __user *uattr)
2949 {
2950 	return -ENOTSUPP;
2951 }
2952 
bpf_map_put(struct bpf_map * map)2953 static inline void bpf_map_put(struct bpf_map *map)
2954 {
2955 }
2956 
bpf_prog_by_id(u32 id)2957 static inline struct bpf_prog *bpf_prog_by_id(u32 id)
2958 {
2959 	return ERR_PTR(-ENOTSUPP);
2960 }
2961 
btf_struct_access(struct bpf_verifier_log * log,const struct bpf_reg_state * reg,int off,int size,enum bpf_access_type atype,u32 * next_btf_id,enum bpf_type_flag * flag,const char ** field_name)2962 static inline int btf_struct_access(struct bpf_verifier_log *log,
2963 				    const struct bpf_reg_state *reg,
2964 				    int off, int size, enum bpf_access_type atype,
2965 				    u32 *next_btf_id, enum bpf_type_flag *flag,
2966 				    const char **field_name)
2967 {
2968 	return -EACCES;
2969 }
2970 
2971 static inline const struct bpf_func_proto *
bpf_base_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)2972 bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2973 {
2974 	return NULL;
2975 }
2976 
bpf_task_storage_free(struct task_struct * task)2977 static inline void bpf_task_storage_free(struct task_struct *task)
2978 {
2979 }
2980 
bpf_prog_has_kfunc_call(const struct bpf_prog * prog)2981 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2982 {
2983 	return false;
2984 }
2985 
2986 static inline const struct btf_func_model *
bpf_jit_find_kfunc_model(const struct bpf_prog * prog,const struct bpf_insn * insn)2987 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2988 			 const struct bpf_insn *insn)
2989 {
2990 	return NULL;
2991 }
2992 
2993 static inline int
bpf_get_kfunc_addr(const struct bpf_prog * prog,u32 func_id,u16 btf_fd_idx,u8 ** func_addr)2994 bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2995 		   u16 btf_fd_idx, u8 **func_addr)
2996 {
2997 	return -ENOTSUPP;
2998 }
2999 
unprivileged_ebpf_enabled(void)3000 static inline bool unprivileged_ebpf_enabled(void)
3001 {
3002 	return false;
3003 }
3004 
has_current_bpf_ctx(void)3005 static inline bool has_current_bpf_ctx(void)
3006 {
3007 	return false;
3008 }
3009 
bpf_prog_inc_misses_counter(struct bpf_prog * prog)3010 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
3011 {
3012 }
3013 
bpf_cgrp_storage_free(struct cgroup * cgroup)3014 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
3015 {
3016 }
3017 
bpf_dynptr_init(struct bpf_dynptr_kern * ptr,void * data,enum bpf_dynptr_type type,u32 offset,u32 size)3018 static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
3019 				   enum bpf_dynptr_type type, u32 offset, u32 size)
3020 {
3021 }
3022 
bpf_dynptr_set_null(struct bpf_dynptr_kern * ptr)3023 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
3024 {
3025 }
3026 
bpf_dynptr_set_rdonly(struct bpf_dynptr_kern * ptr)3027 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
3028 {
3029 }
3030 #endif /* CONFIG_BPF_SYSCALL */
3031 
3032 static __always_inline int
bpf_probe_read_kernel_common(void * dst,u32 size,const void * unsafe_ptr)3033 bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
3034 {
3035 	int ret = -EFAULT;
3036 
3037 	if (IS_ENABLED(CONFIG_BPF_EVENTS))
3038 		ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
3039 	if (unlikely(ret < 0))
3040 		memset(dst, 0, size);
3041 	return ret;
3042 }
3043 
3044 void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len);
3045 
bpf_prog_get_type(u32 ufd,enum bpf_prog_type type)3046 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
3047 						 enum bpf_prog_type type)
3048 {
3049 	return bpf_prog_get_type_dev(ufd, type, false);
3050 }
3051 
3052 void __bpf_free_used_maps(struct bpf_prog_aux *aux,
3053 			  struct bpf_map **used_maps, u32 len);
3054 
3055 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
3056 
3057 int bpf_prog_offload_compile(struct bpf_prog *prog);
3058 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
3059 int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
3060 			       struct bpf_prog *prog);
3061 
3062 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
3063 
3064 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
3065 int bpf_map_offload_update_elem(struct bpf_map *map,
3066 				void *key, void *value, u64 flags);
3067 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
3068 int bpf_map_offload_get_next_key(struct bpf_map *map,
3069 				 void *key, void *next_key);
3070 
3071 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
3072 
3073 struct bpf_offload_dev *
3074 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
3075 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
3076 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
3077 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
3078 				    struct net_device *netdev);
3079 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
3080 				       struct net_device *netdev);
3081 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
3082 
3083 void unpriv_ebpf_notify(int new_state);
3084 
3085 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
3086 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3087 			      struct bpf_prog_aux *prog_aux);
3088 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
3089 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
3090 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
3091 void bpf_dev_bound_netdev_unregister(struct net_device *dev);
3092 
bpf_prog_is_dev_bound(const struct bpf_prog_aux * aux)3093 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3094 {
3095 	return aux->dev_bound;
3096 }
3097 
bpf_prog_is_offloaded(const struct bpf_prog_aux * aux)3098 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
3099 {
3100 	return aux->offload_requested;
3101 }
3102 
3103 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);
3104 
bpf_map_is_offloaded(struct bpf_map * map)3105 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3106 {
3107 	return unlikely(map->ops == &bpf_map_offload_ops);
3108 }
3109 
3110 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
3111 void bpf_map_offload_map_free(struct bpf_map *map);
3112 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
3113 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3114 			      const union bpf_attr *kattr,
3115 			      union bpf_attr __user *uattr);
3116 
3117 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
3118 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
3119 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
3120 int sock_map_bpf_prog_query(const union bpf_attr *attr,
3121 			    union bpf_attr __user *uattr);
3122 int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog);
3123 
3124 void sock_map_unhash(struct sock *sk);
3125 void sock_map_destroy(struct sock *sk);
3126 void sock_map_close(struct sock *sk, long timeout);
3127 #else
bpf_dev_bound_kfunc_check(struct bpf_verifier_log * log,struct bpf_prog_aux * prog_aux)3128 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3129 					    struct bpf_prog_aux *prog_aux)
3130 {
3131 	return -EOPNOTSUPP;
3132 }
3133 
bpf_dev_bound_resolve_kfunc(struct bpf_prog * prog,u32 func_id)3134 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
3135 						u32 func_id)
3136 {
3137 	return NULL;
3138 }
3139 
bpf_prog_dev_bound_init(struct bpf_prog * prog,union bpf_attr * attr)3140 static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
3141 					  union bpf_attr *attr)
3142 {
3143 	return -EOPNOTSUPP;
3144 }
3145 
bpf_prog_dev_bound_inherit(struct bpf_prog * new_prog,struct bpf_prog * old_prog)3146 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
3147 					     struct bpf_prog *old_prog)
3148 {
3149 	return -EOPNOTSUPP;
3150 }
3151 
bpf_dev_bound_netdev_unregister(struct net_device * dev)3152 static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
3153 {
3154 }
3155 
bpf_prog_is_dev_bound(const struct bpf_prog_aux * aux)3156 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3157 {
3158 	return false;
3159 }
3160 
bpf_prog_is_offloaded(struct bpf_prog_aux * aux)3161 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
3162 {
3163 	return false;
3164 }
3165 
bpf_prog_dev_bound_match(const struct bpf_prog * lhs,const struct bpf_prog * rhs)3166 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
3167 {
3168 	return false;
3169 }
3170 
bpf_map_is_offloaded(struct bpf_map * map)3171 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3172 {
3173 	return false;
3174 }
3175 
bpf_map_offload_map_alloc(union bpf_attr * attr)3176 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
3177 {
3178 	return ERR_PTR(-EOPNOTSUPP);
3179 }
3180 
bpf_map_offload_map_free(struct bpf_map * map)3181 static inline void bpf_map_offload_map_free(struct bpf_map *map)
3182 {
3183 }
3184 
bpf_map_offload_map_mem_usage(const struct bpf_map * map)3185 static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
3186 {
3187 	return 0;
3188 }
3189 
bpf_prog_test_run_syscall(struct bpf_prog * prog,const union bpf_attr * kattr,union bpf_attr __user * uattr)3190 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3191 					    const union bpf_attr *kattr,
3192 					    union bpf_attr __user *uattr)
3193 {
3194 	return -ENOTSUPP;
3195 }
3196 
3197 #ifdef CONFIG_BPF_SYSCALL
sock_map_get_from_fd(const union bpf_attr * attr,struct bpf_prog * prog)3198 static inline int sock_map_get_from_fd(const union bpf_attr *attr,
3199 				       struct bpf_prog *prog)
3200 {
3201 	return -EINVAL;
3202 }
3203 
sock_map_prog_detach(const union bpf_attr * attr,enum bpf_prog_type ptype)3204 static inline int sock_map_prog_detach(const union bpf_attr *attr,
3205 				       enum bpf_prog_type ptype)
3206 {
3207 	return -EOPNOTSUPP;
3208 }
3209 
sock_map_update_elem_sys(struct bpf_map * map,void * key,void * value,u64 flags)3210 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
3211 					   u64 flags)
3212 {
3213 	return -EOPNOTSUPP;
3214 }
3215 
sock_map_bpf_prog_query(const union bpf_attr * attr,union bpf_attr __user * uattr)3216 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
3217 					  union bpf_attr __user *uattr)
3218 {
3219 	return -EINVAL;
3220 }
3221 
sock_map_link_create(const union bpf_attr * attr,struct bpf_prog * prog)3222 static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
3223 {
3224 	return -EOPNOTSUPP;
3225 }
3226 #endif /* CONFIG_BPF_SYSCALL */
3227 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
3228 
3229 static __always_inline void
bpf_prog_inc_misses_counters(const struct bpf_prog_array * array)3230 bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
3231 {
3232 	const struct bpf_prog_array_item *item;
3233 	struct bpf_prog *prog;
3234 
3235 	if (unlikely(!array))
3236 		return;
3237 
3238 	item = &array->items[0];
3239 	while ((prog = READ_ONCE(item->prog))) {
3240 		bpf_prog_inc_misses_counter(prog);
3241 		item++;
3242 	}
3243 }
3244 
3245 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
3246 void bpf_sk_reuseport_detach(struct sock *sk);
3247 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
3248 				       void *value);
3249 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
3250 				       void *value, u64 map_flags);
3251 #else
bpf_sk_reuseport_detach(struct sock * sk)3252 static inline void bpf_sk_reuseport_detach(struct sock *sk)
3253 {
3254 }
3255 
3256 #ifdef CONFIG_BPF_SYSCALL
bpf_fd_reuseport_array_lookup_elem(struct bpf_map * map,void * key,void * value)3257 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
3258 						     void *key, void *value)
3259 {
3260 	return -EOPNOTSUPP;
3261 }
3262 
bpf_fd_reuseport_array_update_elem(struct bpf_map * map,void * key,void * value,u64 map_flags)3263 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
3264 						     void *key, void *value,
3265 						     u64 map_flags)
3266 {
3267 	return -EOPNOTSUPP;
3268 }
3269 #endif /* CONFIG_BPF_SYSCALL */
3270 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
3271 
3272 /* verifier prototypes for helper functions called from eBPF programs */
3273 extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
3274 extern const struct bpf_func_proto bpf_map_update_elem_proto;
3275 extern const struct bpf_func_proto bpf_map_delete_elem_proto;
3276 extern const struct bpf_func_proto bpf_map_push_elem_proto;
3277 extern const struct bpf_func_proto bpf_map_pop_elem_proto;
3278 extern const struct bpf_func_proto bpf_map_peek_elem_proto;
3279 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;
3280 
3281 extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
3282 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
3283 extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
3284 extern const struct bpf_func_proto bpf_tail_call_proto;
3285 extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
3286 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
3287 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
3288 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
3289 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
3290 extern const struct bpf_func_proto bpf_get_current_comm_proto;
3291 extern const struct bpf_func_proto bpf_get_stackid_proto;
3292 extern const struct bpf_func_proto bpf_get_stack_proto;
3293 extern const struct bpf_func_proto bpf_get_stack_sleepable_proto;
3294 extern const struct bpf_func_proto bpf_get_task_stack_proto;
3295 extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto;
3296 extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
3297 extern const struct bpf_func_proto bpf_get_stack_proto_pe;
3298 extern const struct bpf_func_proto bpf_sock_map_update_proto;
3299 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
3300 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
3301 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
3302 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
3303 extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto;
3304 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
3305 extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
3306 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
3307 extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
3308 extern const struct bpf_func_proto bpf_spin_lock_proto;
3309 extern const struct bpf_func_proto bpf_spin_unlock_proto;
3310 extern const struct bpf_func_proto bpf_get_local_storage_proto;
3311 extern const struct bpf_func_proto bpf_strtol_proto;
3312 extern const struct bpf_func_proto bpf_strtoul_proto;
3313 extern const struct bpf_func_proto bpf_tcp_sock_proto;
3314 extern const struct bpf_func_proto bpf_jiffies64_proto;
3315 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
3316 extern const struct bpf_func_proto bpf_event_output_data_proto;
3317 extern const struct bpf_func_proto bpf_ringbuf_output_proto;
3318 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
3319 extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
3320 extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
3321 extern const struct bpf_func_proto bpf_ringbuf_query_proto;
3322 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
3323 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
3324 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
3325 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
3326 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
3327 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
3328 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
3329 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
3330 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
3331 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
3332 extern const struct bpf_func_proto bpf_copy_from_user_proto;
3333 extern const struct bpf_func_proto bpf_snprintf_btf_proto;
3334 extern const struct bpf_func_proto bpf_snprintf_proto;
3335 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
3336 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
3337 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
3338 extern const struct bpf_func_proto bpf_sock_from_file_proto;
3339 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
3340 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
3341 extern const struct bpf_func_proto bpf_task_storage_get_proto;
3342 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
3343 extern const struct bpf_func_proto bpf_task_storage_delete_proto;
3344 extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
3345 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
3346 extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
3347 extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
3348 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
3349 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
3350 extern const struct bpf_func_proto bpf_find_vma_proto;
3351 extern const struct bpf_func_proto bpf_loop_proto;
3352 extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
3353 extern const struct bpf_func_proto bpf_set_retval_proto;
3354 extern const struct bpf_func_proto bpf_get_retval_proto;
3355 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
3356 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
3357 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
3358 
3359 const struct bpf_func_proto *tracing_prog_func_proto(
3360   enum bpf_func_id func_id, const struct bpf_prog *prog);
3361 
3362 /* Shared helpers among cBPF and eBPF. */
3363 void bpf_user_rnd_init_once(void);
3364 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3365 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3366 
3367 #if defined(CONFIG_NET)
3368 bool bpf_sock_common_is_valid_access(int off, int size,
3369 				     enum bpf_access_type type,
3370 				     struct bpf_insn_access_aux *info);
3371 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3372 			      struct bpf_insn_access_aux *info);
3373 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3374 				const struct bpf_insn *si,
3375 				struct bpf_insn *insn_buf,
3376 				struct bpf_prog *prog,
3377 				u32 *target_size);
3378 int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
3379 			       struct bpf_dynptr *ptr);
3380 #else
bpf_sock_common_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3381 static inline bool bpf_sock_common_is_valid_access(int off, int size,
3382 						   enum bpf_access_type type,
3383 						   struct bpf_insn_access_aux *info)
3384 {
3385 	return false;
3386 }
bpf_sock_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3387 static inline bool bpf_sock_is_valid_access(int off, int size,
3388 					    enum bpf_access_type type,
3389 					    struct bpf_insn_access_aux *info)
3390 {
3391 	return false;
3392 }
bpf_sock_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)3393 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3394 					      const struct bpf_insn *si,
3395 					      struct bpf_insn *insn_buf,
3396 					      struct bpf_prog *prog,
3397 					      u32 *target_size)
3398 {
3399 	return 0;
3400 }
bpf_dynptr_from_skb_rdonly(struct __sk_buff * skb,u64 flags,struct bpf_dynptr * ptr)3401 static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
3402 					     struct bpf_dynptr *ptr)
3403 {
3404 	return -EOPNOTSUPP;
3405 }
3406 #endif
3407 
3408 #ifdef CONFIG_INET
3409 struct sk_reuseport_kern {
3410 	struct sk_buff *skb;
3411 	struct sock *sk;
3412 	struct sock *selected_sk;
3413 	struct sock *migrating_sk;
3414 	void *data_end;
3415 	u32 hash;
3416 	u32 reuseport_id;
3417 	bool bind_inany;
3418 };
3419 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3420 				  struct bpf_insn_access_aux *info);
3421 
3422 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3423 				    const struct bpf_insn *si,
3424 				    struct bpf_insn *insn_buf,
3425 				    struct bpf_prog *prog,
3426 				    u32 *target_size);
3427 
3428 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3429 				  struct bpf_insn_access_aux *info);
3430 
3431 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3432 				    const struct bpf_insn *si,
3433 				    struct bpf_insn *insn_buf,
3434 				    struct bpf_prog *prog,
3435 				    u32 *target_size);
3436 #else
bpf_tcp_sock_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3437 static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
3438 						enum bpf_access_type type,
3439 						struct bpf_insn_access_aux *info)
3440 {
3441 	return false;
3442 }
3443 
bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)3444 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3445 						  const struct bpf_insn *si,
3446 						  struct bpf_insn *insn_buf,
3447 						  struct bpf_prog *prog,
3448 						  u32 *target_size)
3449 {
3450 	return 0;
3451 }
bpf_xdp_sock_is_valid_access(int off,int size,enum bpf_access_type type,struct bpf_insn_access_aux * info)3452 static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
3453 						enum bpf_access_type type,
3454 						struct bpf_insn_access_aux *info)
3455 {
3456 	return false;
3457 }
3458 
bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,const struct bpf_insn * si,struct bpf_insn * insn_buf,struct bpf_prog * prog,u32 * target_size)3459 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3460 						  const struct bpf_insn *si,
3461 						  struct bpf_insn *insn_buf,
3462 						  struct bpf_prog *prog,
3463 						  u32 *target_size)
3464 {
3465 	return 0;
3466 }
3467 #endif /* CONFIG_INET */
3468 
3469 enum bpf_text_poke_type {
3470 	BPF_MOD_CALL,
3471 	BPF_MOD_JUMP,
3472 };
3473 
3474 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
3475 		       void *addr1, void *addr2);
3476 
3477 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
3478 			       struct bpf_prog *new, struct bpf_prog *old);
3479 
3480 void *bpf_arch_text_copy(void *dst, void *src, size_t len);
3481 int bpf_arch_text_invalidate(void *dst, size_t len);
3482 
3483 struct btf_id_set;
3484 bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
3485 
3486 #define MAX_BPRINTF_VARARGS		12
3487 #define MAX_BPRINTF_BUF			1024
3488 
3489 struct bpf_bprintf_data {
3490 	u32 *bin_args;
3491 	char *buf;
3492 	bool get_bin_args;
3493 	bool get_buf;
3494 };
3495 
3496 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
3497 			u32 num_args, struct bpf_bprintf_data *data);
3498 void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
3499 
3500 #ifdef CONFIG_BPF_LSM
3501 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
3502 void bpf_cgroup_atype_put(int cgroup_atype);
3503 #else
bpf_cgroup_atype_get(u32 attach_btf_id,int cgroup_atype)3504 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
bpf_cgroup_atype_put(int cgroup_atype)3505 static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
3506 #endif /* CONFIG_BPF_LSM */
3507 
3508 struct key;
3509 
3510 #ifdef CONFIG_KEYS
3511 struct bpf_key {
3512 	struct key *key;
3513 	bool has_ref;
3514 };
3515 #endif /* CONFIG_KEYS */
3516 
type_is_alloc(u32 type)3517 static inline bool type_is_alloc(u32 type)
3518 {
3519 	return type & MEM_ALLOC;
3520 }
3521 
bpf_memcg_flags(gfp_t flags)3522 static inline gfp_t bpf_memcg_flags(gfp_t flags)
3523 {
3524 	if (memcg_bpf_enabled())
3525 		return flags | __GFP_ACCOUNT;
3526 	return flags;
3527 }
3528 
bpf_is_subprog(const struct bpf_prog * prog)3529 static inline bool bpf_is_subprog(const struct bpf_prog *prog)
3530 {
3531 	return prog->aux->func_idx != 0;
3532 }
3533 
3534 #endif /* _LINUX_BPF_H */
3535