xref: /linux/include/linux/bpf.h (revision bd59a58663895778da607fd02a8f17a93db97d16)
1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3  */
4 #ifndef _LINUX_BPF_H
5 #define _LINUX_BPF_H 1
6 
7 #include <uapi/linux/bpf.h>
8 #include <uapi/linux/filter.h>
9 #include <linux/bpf_defs.h>
10 
11 #include <crypto/sha2.h>
12 #include <linux/workqueue.h>
13 #include <linux/file.h>
14 #include <linux/percpu.h>
15 #include <linux/err.h>
16 #include <linux/rbtree_latch.h>
17 #include <linux/numa.h>
18 #include <linux/mm_types.h>
19 #include <linux/wait.h>
20 #include <linux/irq_work_types.h>
21 #include <linux/refcount.h>
22 #include <linux/mutex.h>
23 #include <linux/module.h>
24 #include <linux/kallsyms.h>
25 #include <linux/capability.h>
26 #include <linux/sched/mm.h>
27 #include <linux/slab.h>
28 #include <linux/percpu-refcount.h>
29 #include <linux/stddef.h>
30 #include <linux/bpfptr.h>
31 #include <linux/btf.h>
32 #include <linux/rcupdate_trace.h>
33 #include <linux/static_call.h>
34 #include <linux/memcontrol.h>
35 #include <linux/cfi.h>
36 #include <linux/xattr.h>
37 #include <linux/key.h>
38 #include <linux/ftrace.h>
39 #include <asm/rqspinlock.h>
40 
41 struct bpf_verifier_env;
42 struct bpf_verifier_log;
43 struct perf_event;
44 struct bpf_prog;
45 struct bpf_prog_aux;
46 struct bpf_map;
47 struct bpf_arena;
48 struct sock;
49 struct seq_file;
50 struct btf;
51 struct btf_type;
52 struct exception_table_entry;
53 struct seq_operations;
54 struct bpf_iter_aux_info;
55 struct bpf_local_storage;
56 struct bpf_local_storage_map;
57 struct kobject;
58 struct mem_cgroup;
59 struct module;
60 struct bpf_func_state;
61 struct ftrace_ops;
62 struct cgroup;
63 struct bpf_token;
64 struct user_namespace;
65 struct super_block;
66 struct inode;
67 
68 extern struct idr btf_idr;
69 extern spinlock_t btf_idr_lock;
70 extern struct kobject *btf_kobj;
71 extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
72 extern bool bpf_global_ma_set;
73 extern struct workqueue_struct *bpf_prog_free_wq __ro_after_init;
74 
75 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
76 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
77 					struct bpf_iter_aux_info *aux);
78 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
79 typedef unsigned int (*bpf_func_t)(const void *,
80 				   const struct bpf_insn *);
81 struct bpf_iter_seq_info {
82 	const struct seq_operations *seq_ops;
83 	bpf_iter_init_seq_priv_t init_seq_private;
84 	bpf_iter_fini_seq_priv_t fini_seq_private;
85 	u32 seq_priv_size;
86 };
87 
88 /* map is generic key/value storage optionally accessible by eBPF programs */
89 struct bpf_map_ops {
90 	/* funcs callable from userspace (via syscall) */
91 	int (*map_alloc_check)(union bpf_attr *attr);
92 	struct bpf_map *(*map_alloc)(union bpf_attr *attr);
93 	void (*map_release)(struct bpf_map *map, struct file *map_file);
94 	void (*map_free)(struct bpf_map *map);
95 	void (*map_free_pre_rcu)(struct bpf_map *map);
96 	int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
97 	void (*map_release_uref)(struct bpf_map *map);
98 	void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
99 	int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
100 				union bpf_attr __user *uattr);
101 	int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
102 					  void *value, u64 flags);
103 	int (*map_lookup_and_delete_batch)(struct bpf_map *map,
104 					   const union bpf_attr *attr,
105 					   union bpf_attr __user *uattr);
106 	int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
107 				const union bpf_attr *attr,
108 				union bpf_attr __user *uattr);
109 	int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
110 				union bpf_attr __user *uattr);
111 
112 	/* funcs callable from userspace and from eBPF programs */
113 	void *(*map_lookup_elem)(struct bpf_map *map, void *key);
114 	long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
115 	long (*map_delete_elem)(struct bpf_map *map, void *key);
116 	long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
117 	long (*map_pop_elem)(struct bpf_map *map, void *value);
118 	long (*map_peek_elem)(struct bpf_map *map, void *value);
119 	void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
120 	int (*map_get_hash)(struct bpf_map *map);
121 
122 	/* funcs called by prog_array and perf_event_array map */
123 	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
124 				int fd);
125 	/* If need_defer is true, the implementation should guarantee that
126 	 * the to-be-put element is still alive before the bpf program, which
127 	 * may manipulate it, exists.
128 	 */
129 	void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);
130 	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
131 	u32 (*map_fd_sys_lookup_elem)(void *ptr);
132 	void (*map_seq_show_elem)(struct bpf_map *map, void *key,
133 				  struct seq_file *m);
134 	int (*map_check_btf)(struct bpf_map *map,
135 			     const struct btf *btf,
136 			     const struct btf_type *key_type,
137 			     const struct btf_type *value_type);
138 
139 	/* Prog poke tracking helpers. */
140 	int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
141 	void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
142 	void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
143 			     struct bpf_prog *new);
144 
145 	/* Direct value access helpers. */
146 	int (*map_direct_value_addr)(const struct bpf_map *map,
147 				     u64 *imm, u32 off);
148 	int (*map_direct_value_meta)(const struct bpf_map *map,
149 				     u64 imm, u32 *off);
150 	int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
151 	vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf);
152 	__poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
153 			     struct poll_table_struct *pts);
154 	unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
155 					       unsigned long len, unsigned long pgoff,
156 					       unsigned long flags);
157 
158 	/* Functions called by bpf_local_storage maps */
159 	int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
160 					void *owner, u32 size);
161 	void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
162 					   void *owner, u32 size);
163 	struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
164 
165 	/* Misc helpers.*/
166 	long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);
167 
168 	/* map_meta_equal must be implemented for maps that can be
169 	 * used as an inner map.  It is a runtime check to ensure
170 	 * an inner map can be inserted to an outer map.
171 	 *
172 	 * Some properties of the inner map has been used during the
173 	 * verification time.  When inserting an inner map at the runtime,
174 	 * map_meta_equal has to ensure the inserting map has the same
175 	 * properties that the verifier has used earlier.
176 	 */
177 	bool (*map_meta_equal)(const struct bpf_map *meta0,
178 			       const struct bpf_map *meta1);
179 
180 
181 	int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
182 					      struct bpf_func_state *caller,
183 					      struct bpf_func_state *callee);
184 	long (*map_for_each_callback)(struct bpf_map *map,
185 				     bpf_callback_t callback_fn,
186 				     void *callback_ctx, u64 flags);
187 
188 	u64 (*map_mem_usage)(const struct bpf_map *map);
189 
190 	/* BTF id of struct allocated by map_alloc */
191 	int *map_btf_id;
192 
193 	/* bpf_iter info used to open a seq_file */
194 	const struct bpf_iter_seq_info *iter_seq_info;
195 };
196 
197 enum {
198 	/* Support at most 11 fields in a BTF type */
199 	BTF_FIELDS_MAX	   = 11,
200 };
201 
202 enum btf_field_type {
203 	BPF_SPIN_LOCK  = (1 << 0),
204 	BPF_TIMER      = (1 << 1),
205 	BPF_KPTR_UNREF = (1 << 2),
206 	BPF_KPTR_REF   = (1 << 3),
207 	BPF_KPTR_PERCPU = (1 << 4),
208 	BPF_KPTR       = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
209 	BPF_LIST_HEAD  = (1 << 5),
210 	BPF_LIST_NODE  = (1 << 6),
211 	BPF_RB_ROOT    = (1 << 7),
212 	BPF_RB_NODE    = (1 << 8),
213 	BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
214 	BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
215 	BPF_REFCOUNT   = (1 << 9),
216 	BPF_WORKQUEUE  = (1 << 10),
217 	BPF_UPTR       = (1 << 11),
218 	BPF_RES_SPIN_LOCK = (1 << 12),
219 	BPF_TASK_WORK  = (1 << 13),
220 	BPF_RCU_HEAD   = (1 << 14),
221 };
222 
223 enum bpf_cgroup_storage_type {
224 	BPF_CGROUP_STORAGE_SHARED,
225 	BPF_CGROUP_STORAGE_PERCPU,
226 	__BPF_CGROUP_STORAGE_MAX
227 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
228 };
229 
230 #ifdef CONFIG_CGROUP_BPF
231 # define for_each_cgroup_storage_type(stype) \
232 	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
233 #else
234 # define for_each_cgroup_storage_type(stype) for (; false; )
235 #endif /* CONFIG_CGROUP_BPF */
236 
237 typedef void (*btf_dtor_kfunc_t)(void *);
238 
239 struct btf_field_kptr {
240 	struct btf *btf;
241 	struct module *module;
242 	/* dtor used if btf_is_kernel(btf), otherwise the type is
243 	 * program-allocated, dtor is NULL,  and __bpf_obj_drop_impl is used
244 	 */
245 	btf_dtor_kfunc_t dtor;
246 	u32 btf_id;
247 };
248 
249 struct btf_field_graph_root {
250 	struct btf *btf;
251 	u32 value_btf_id;
252 	u32 node_offset;
253 	struct btf_record *value_rec;
254 };
255 
256 struct btf_field {
257 	u32 offset;
258 	u32 size;
259 	enum btf_field_type type;
260 	union {
261 		struct btf_field_kptr kptr;
262 		struct btf_field_graph_root graph_root;
263 	};
264 };
265 
266 struct btf_record {
267 	u32 cnt;
268 	u32 field_mask;
269 	int spin_lock_off;
270 	int res_spin_lock_off;
271 	int timer_off;
272 	int wq_off;
273 	int refcount_off;
274 	int task_work_off;
275 	int rcu_head_off;
276 	struct btf_field fields[];
277 };
278 
279 /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
280 struct bpf_rb_node_kern {
281 	struct rb_node rb_node;
282 	void *owner;
283 } __attribute__((aligned(8)));
284 
285 /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
286 struct bpf_list_node_kern {
287 	struct list_head list_head;
288 	void *owner;
289 } __attribute__((aligned(8)));
290 
291 /* 'Ownership' of program-containing map is claimed by the first program
292  * that is going to use this map or by the first program which FD is
293  * stored in the map to make sure that all callers and callees have the
294  * same prog type, JITed flag and xdp_has_frags flag.
295  */
296 struct bpf_map_owner {
297 	enum bpf_prog_type type;
298 	bool jited;
299 	bool xdp_has_frags;
300 	bool sleepable;
301 	u64 storage_cookie[MAX_BPF_CGROUP_STORAGE_TYPE];
302 	const struct btf_type *attach_func_proto;
303 	enum bpf_attach_type expected_attach_type;
304 };
305 
306 struct bpf_map {
307 	u8 sha[SHA256_DIGEST_SIZE];
308 	const struct bpf_map_ops *ops;
309 	struct bpf_map *inner_map_meta;
310 #ifdef CONFIG_SECURITY
311 	void *security;
312 #endif
313 	enum bpf_map_type map_type;
314 	u32 key_size;
315 	u32 value_size;
316 	u32 max_entries;
317 	u64 map_extra; /* any per-map-type extra fields */
318 	u32 map_flags;
319 	u32 id;
320 	struct btf_record *record;
321 	int numa_node;
322 	u32 btf_key_type_id;
323 	u32 btf_value_type_id;
324 	u32 btf_vmlinux_value_type_id;
325 	struct btf *btf;
326 #ifdef CONFIG_MEMCG
327 	struct obj_cgroup *objcg;
328 #endif
329 	char name[BPF_OBJ_NAME_LEN];
330 	struct mutex freeze_mutex;
331 	atomic64_t refcnt;
332 	atomic64_t usercnt;
333 	/* rcu is used before freeing and work is only used during freeing */
334 	union {
335 		struct work_struct work;
336 		struct rcu_head rcu;
337 	};
338 	atomic64_t writecnt;
339 	spinlock_t owner_lock;
340 	struct bpf_map_owner *owner;
341 	bool bypass_spec_v1;
342 	bool frozen; /* write-once; write-protected by freeze_mutex */
343 	bool free_after_mult_rcu_gp;
344 	bool free_after_rcu_gp;
345 	bool was_inner_map;
346 	atomic64_t sleepable_refcnt;
347 	s64 __percpu *elem_count;
348 	u64 cookie; /* write-once */
349 	char *excl_prog_sha;
350 	/*
351 	 * Which programs use the map, see bpf_map_claim(): 0 - none so far,
352 	 * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED
353 	 * set - only that one and it stored the addresses of its functions into
354 	 * the map, BPF_MAP_USER_MANY - more than one.
355 	 */
356 	unsigned long user;
357 };
358 
359 #define BPF_MAP_USER_MANY	1UL
360 #define BPF_MAP_USER_PATCHED	1UL
361 
362 static inline const char *btf_field_type_name(enum btf_field_type type)
363 {
364 	switch (type) {
365 	case BPF_SPIN_LOCK:
366 		return "bpf_spin_lock";
367 	case BPF_RES_SPIN_LOCK:
368 		return "bpf_res_spin_lock";
369 	case BPF_TIMER:
370 		return "bpf_timer";
371 	case BPF_WORKQUEUE:
372 		return "bpf_wq";
373 	case BPF_KPTR_UNREF:
374 	case BPF_KPTR_REF:
375 		return "kptr";
376 	case BPF_KPTR_PERCPU:
377 		return "percpu_kptr";
378 	case BPF_UPTR:
379 		return "uptr";
380 	case BPF_LIST_HEAD:
381 		return "bpf_list_head";
382 	case BPF_LIST_NODE:
383 		return "bpf_list_node";
384 	case BPF_RB_ROOT:
385 		return "bpf_rb_root";
386 	case BPF_RB_NODE:
387 		return "bpf_rb_node";
388 	case BPF_REFCOUNT:
389 		return "bpf_refcount";
390 	case BPF_TASK_WORK:
391 		return "bpf_task_work";
392 	case BPF_RCU_HEAD:
393 		return "bpf_rcu_head";
394 	default:
395 		WARN_ON_ONCE(1);
396 		return "unknown";
397 	}
398 }
399 
400 #if IS_ENABLED(CONFIG_DEBUG_KERNEL)
401 #define BPF_WARN_ONCE(cond, format...) WARN_ONCE(cond, format)
402 #else
403 #define BPF_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond)
404 #endif
405 
406 static inline u32 btf_field_type_size(enum btf_field_type type)
407 {
408 	switch (type) {
409 	case BPF_SPIN_LOCK:
410 		return sizeof(struct bpf_spin_lock);
411 	case BPF_RES_SPIN_LOCK:
412 		return sizeof(struct bpf_res_spin_lock);
413 	case BPF_TIMER:
414 		return sizeof(struct bpf_timer);
415 	case BPF_WORKQUEUE:
416 		return sizeof(struct bpf_wq);
417 	case BPF_KPTR_UNREF:
418 	case BPF_KPTR_REF:
419 	case BPF_KPTR_PERCPU:
420 	case BPF_UPTR:
421 		return sizeof(u64);
422 	case BPF_LIST_HEAD:
423 		return sizeof(struct bpf_list_head);
424 	case BPF_LIST_NODE:
425 		return sizeof(struct bpf_list_node);
426 	case BPF_RB_ROOT:
427 		return sizeof(struct bpf_rb_root);
428 	case BPF_RB_NODE:
429 		return sizeof(struct bpf_rb_node);
430 	case BPF_REFCOUNT:
431 		return sizeof(struct bpf_refcount);
432 	case BPF_TASK_WORK:
433 		return sizeof(struct bpf_task_work);
434 	case BPF_RCU_HEAD:
435 		return sizeof(struct bpf_rcu_head);
436 	default:
437 		WARN_ON_ONCE(1);
438 		return 0;
439 	}
440 }
441 
442 static inline u32 btf_field_type_align(enum btf_field_type type)
443 {
444 	switch (type) {
445 	case BPF_SPIN_LOCK:
446 		return __alignof__(struct bpf_spin_lock);
447 	case BPF_RES_SPIN_LOCK:
448 		return __alignof__(struct bpf_res_spin_lock);
449 	case BPF_TIMER:
450 		return __alignof__(struct bpf_timer);
451 	case BPF_WORKQUEUE:
452 		return __alignof__(struct bpf_wq);
453 	case BPF_KPTR_UNREF:
454 	case BPF_KPTR_REF:
455 	case BPF_KPTR_PERCPU:
456 	case BPF_UPTR:
457 		return __alignof__(u64);
458 	case BPF_LIST_HEAD:
459 		return __alignof__(struct bpf_list_head);
460 	case BPF_LIST_NODE:
461 		return __alignof__(struct bpf_list_node);
462 	case BPF_RB_ROOT:
463 		return __alignof__(struct bpf_rb_root);
464 	case BPF_RB_NODE:
465 		return __alignof__(struct bpf_rb_node);
466 	case BPF_REFCOUNT:
467 		return __alignof__(struct bpf_refcount);
468 	case BPF_TASK_WORK:
469 		return __alignof__(struct bpf_task_work);
470 	case BPF_RCU_HEAD:
471 		return __alignof__(struct bpf_rcu_head);
472 	default:
473 		WARN_ON_ONCE(1);
474 		return 0;
475 	}
476 }
477 
478 static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
479 {
480 	memset(addr, 0, field->size);
481 
482 	switch (field->type) {
483 	case BPF_REFCOUNT:
484 		refcount_set((refcount_t *)addr, 1);
485 		break;
486 	case BPF_RB_NODE:
487 		RB_CLEAR_NODE((struct rb_node *)addr);
488 		break;
489 	case BPF_LIST_HEAD:
490 	case BPF_LIST_NODE:
491 		INIT_LIST_HEAD((struct list_head *)addr);
492 		break;
493 	case BPF_RB_ROOT:
494 		/* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
495 	case BPF_SPIN_LOCK:
496 	case BPF_RES_SPIN_LOCK:
497 	case BPF_TIMER:
498 	case BPF_WORKQUEUE:
499 	case BPF_KPTR_UNREF:
500 	case BPF_KPTR_REF:
501 	case BPF_KPTR_PERCPU:
502 	case BPF_UPTR:
503 	case BPF_TASK_WORK:
504 	case BPF_RCU_HEAD:
505 		break;
506 	default:
507 		WARN_ON_ONCE(1);
508 		return;
509 	}
510 }
511 
512 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
513 {
514 	if (IS_ERR_OR_NULL(rec))
515 		return false;
516 	return rec->field_mask & type;
517 }
518 
519 static inline bool btf_field_is_nmi_safe(enum btf_field_type type)
520 {
521 	switch (type) {
522 	case BPF_SPIN_LOCK:
523 	case BPF_RES_SPIN_LOCK:
524 	case BPF_TIMER:
525 	case BPF_WORKQUEUE:
526 	case BPF_TASK_WORK:
527 	case BPF_KPTR_UNREF:
528 	case BPF_REFCOUNT:
529 		return true;
530 	default:
531 		return false;
532 	}
533 }
534 
535 static inline bool btf_record_has_nmi_unsafe_fields(const struct btf_record *rec)
536 {
537 	int i;
538 
539 	if (IS_ERR_OR_NULL(rec))
540 		return false;
541 	for (i = 0; i < rec->cnt; i++) {
542 		if (!btf_field_is_nmi_safe(rec->fields[i].type))
543 			return true;
544 	}
545 	return false;
546 }
547 
548 static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
549 {
550 	int i;
551 
552 	if (IS_ERR_OR_NULL(rec))
553 		return;
554 	for (i = 0; i < rec->cnt; i++)
555 		bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
556 }
557 
558 /* 'dst' must be a temporary buffer and should not point to memory that is being
559  * used in parallel by a bpf program or bpf syscall, otherwise the access from
560  * the bpf program or bpf syscall may be corrupted by the reinitialization,
561  * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
562  * allocator, it is still possible for 'dst' to be used in parallel by a bpf
563  * program or bpf syscall.
564  */
565 static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
566 {
567 	bpf_obj_init(map->record, dst);
568 }
569 
570 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
571  * forced to use 'long' read/writes to try to atomically copy long counters.
572  * Best-effort only.  No barriers here, since it _will_ race with concurrent
573  * updates from BPF programs. Called from bpf syscall and mostly used with
574  * size 8 or 16 bytes, so ask compiler to inline it.
575  */
576 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
577 {
578 	const long *lsrc = src;
579 	long *ldst = dst;
580 
581 	size /= sizeof(long);
582 	while (size--)
583 		data_race(*ldst++ = *lsrc++);
584 }
585 
586 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
587 static inline void bpf_obj_memcpy(struct btf_record *rec,
588 				  void *dst, void *src, u32 size,
589 				  bool long_memcpy)
590 {
591 	u32 curr_off = 0;
592 	int i;
593 
594 	if (IS_ERR_OR_NULL(rec)) {
595 		if (long_memcpy)
596 			bpf_long_memcpy(dst, src, size);
597 		else
598 			data_race(memcpy(dst, src, size));
599 		return;
600 	}
601 
602 	for (i = 0; i < rec->cnt; i++) {
603 		u32 next_off = rec->fields[i].offset;
604 		u32 sz = next_off - curr_off;
605 
606 		data_race(memcpy(dst + curr_off, src + curr_off, sz));
607 		curr_off += rec->fields[i].size + sz;
608 	}
609 	data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off));
610 }
611 
612 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
613 {
614 	bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
615 }
616 
617 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
618 {
619 	bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true);
620 }
621 
622 static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
623 {
624 	unsigned long *src_uptr, *dst_uptr;
625 	const struct btf_field *field;
626 	int i;
627 
628 	if (!btf_record_has_field(rec, BPF_UPTR))
629 		return;
630 
631 	for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
632 		if (field->type != BPF_UPTR)
633 			continue;
634 
635 		src_uptr = src + field->offset;
636 		dst_uptr = dst + field->offset;
637 		swap(*src_uptr, *dst_uptr);
638 	}
639 }
640 
641 static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
642 {
643 	u32 curr_off = 0;
644 	int i;
645 
646 	if (IS_ERR_OR_NULL(rec)) {
647 		memset(dst, 0, size);
648 		return;
649 	}
650 
651 	for (i = 0; i < rec->cnt; i++) {
652 		u32 next_off = rec->fields[i].offset;
653 		u32 sz = next_off - curr_off;
654 
655 		memset(dst + curr_off, 0, sz);
656 		curr_off += rec->fields[i].size + sz;
657 	}
658 	memset(dst + curr_off, 0, size - curr_off);
659 }
660 
661 static inline void zero_map_value(struct bpf_map *map, void *dst)
662 {
663 	bpf_obj_memzero(map->record, dst, map->value_size);
664 }
665 
666 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
667 			   bool lock_src);
668 void bpf_timer_cancel_and_free(void *timer);
669 void bpf_wq_cancel_and_free(void *timer);
670 void bpf_task_work_cancel_and_free(void *timer);
671 void bpf_list_head_free(const struct btf_field *field, void *list_head,
672 			struct bpf_spin_lock *spin_lock);
673 void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
674 		      struct bpf_spin_lock *spin_lock);
675 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
676 u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
677 u64 bpf_arena_map_kern_vm_start(struct bpf_map *map);
678 struct bpf_map *bpf_prog_arena(struct bpf_prog *prog);
679 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
680 
681 struct bpf_offload_dev;
682 struct bpf_offloaded_map;
683 
684 struct bpf_map_dev_ops {
685 	int (*map_get_next_key)(struct bpf_offloaded_map *map,
686 				void *key, void *next_key);
687 	int (*map_lookup_elem)(struct bpf_offloaded_map *map,
688 			       void *key, void *value);
689 	int (*map_update_elem)(struct bpf_offloaded_map *map,
690 			       void *key, void *value, u64 flags);
691 	int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
692 };
693 
694 struct bpf_offloaded_map {
695 	struct bpf_map map;
696 	struct net_device *netdev;
697 	const struct bpf_map_dev_ops *dev_ops;
698 	void *dev_priv;
699 	struct list_head offloads;
700 };
701 
702 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
703 {
704 	return container_of(map, struct bpf_offloaded_map, map);
705 }
706 
707 static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
708 {
709 	return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
710 }
711 
712 static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
713 {
714 	return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
715 		map->ops->map_seq_show_elem;
716 }
717 
718 int map_check_no_btf(struct bpf_map *map,
719 		     const struct btf *btf,
720 		     const struct btf_type *key_type,
721 		     const struct btf_type *value_type);
722 
723 bool bpf_map_meta_equal(const struct bpf_map *meta0,
724 			const struct bpf_map *meta1);
725 
726 static inline bool bpf_map_has_internal_structs(struct bpf_map *map)
727 {
728 	return btf_record_has_field(map->record, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK);
729 }
730 
731 void bpf_map_free_internal_structs(struct bpf_map *map, void *obj);
732 
733 int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags,
734 				   struct bpf_dynptr *ptr__uninit);
735 
736 #if defined(CONFIG_MMU) && defined(CONFIG_64BIT)
737 void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id,
738 					  u64 flags);
739 void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt);
740 void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id,
741 				      u64 flags);
742 #else
743 static inline void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt,
744 							int node_id, u64 flags)
745 {
746 	return NULL;
747 }
748 
749 static inline void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt)
750 {
751 }
752 
753 static inline void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt,
754 						    int node_id, u64 flags)
755 {
756 	return NULL;
757 }
758 #endif
759 
760 extern const struct bpf_map_ops bpf_map_offload_ops;
761 
762 /* bpf_type_flag contains a set of flags that are applicable to the values of
763  * arg_type, ret_type and reg_type. For example, a pointer value may be null,
764  * or a memory is read-only. We classify types into two categories: base types
765  * and extended types. Extended types are base types combined with a type flag.
766  *
767  * Currently there are no more than 32 base types in arg_type, ret_type and
768  * reg_types.
769  */
770 #define BPF_BASE_TYPE_BITS	8
771 
772 enum bpf_type_flag {
773 	/* PTR may be NULL. */
774 	PTR_MAYBE_NULL		= BIT(0 + BPF_BASE_TYPE_BITS),
775 
776 	/* MEM is read-only. When applied on bpf_arg, it indicates the arg is
777 	 * compatible with both mutable and immutable memory.
778 	 */
779 	MEM_RDONLY		= BIT(1 + BPF_BASE_TYPE_BITS),
780 
781 	/* MEM points to BPF ring buffer reservation. */
782 	MEM_RINGBUF		= BIT(2 + BPF_BASE_TYPE_BITS),
783 
784 	/* MEM is in user address space. */
785 	MEM_USER		= BIT(3 + BPF_BASE_TYPE_BITS),
786 
787 	/* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
788 	 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
789 	 * order to drop this tag, it must be passed into bpf_per_cpu_ptr()
790 	 * or bpf_this_cpu_ptr(), which will return the pointer corresponding
791 	 * to the specified cpu.
792 	 */
793 	MEM_PERCPU		= BIT(4 + BPF_BASE_TYPE_BITS),
794 
795 	/* Indicates that the argument will be released. */
796 	OBJ_RELEASE		= BIT(5 + BPF_BASE_TYPE_BITS),
797 
798 	/* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
799 	 * unreferenced and referenced kptr loaded from map value using a load
800 	 * instruction, so that they can only be dereferenced but not escape the
801 	 * BPF program into the kernel (i.e. cannot be passed as arguments to
802 	 * kfunc or bpf helpers).
803 	 */
804 	PTR_UNTRUSTED		= BIT(6 + BPF_BASE_TYPE_BITS),
805 
806 	/*
807 	 * MEM can be uninitialized. Generic memory outputs need not be fully
808 	 * initialized by the callee.
809 	 */
810 	MEM_UNINIT		= BIT(7 + BPF_BASE_TYPE_BITS),
811 
812 	/* DYNPTR points to memory local to the bpf program. */
813 	DYNPTR_TYPE_LOCAL	= BIT(8 + BPF_BASE_TYPE_BITS),
814 
815 	/* DYNPTR points to a kernel-produced ringbuf record. */
816 	DYNPTR_TYPE_RINGBUF	= BIT(9 + BPF_BASE_TYPE_BITS),
817 
818 	/* Size is known at compile time. */
819 	MEM_FIXED_SIZE		= BIT(10 + BPF_BASE_TYPE_BITS),
820 
821 	/* MEM is of an allocated object of type in program BTF. This is used to
822 	 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
823 	 */
824 	MEM_ALLOC		= BIT(11 + BPF_BASE_TYPE_BITS),
825 
826 	/* PTR was passed from the kernel in a trusted context, and may be
827 	 * passed to kfuncs or BPF helper functions.
828 	 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
829 	 * PTR_UNTRUSTED refers to a kptr that was read directly from a map
830 	 * without invoking bpf_kptr_xchg(). What we really need to know is
831 	 * whether a pointer is safe to pass to a kfunc or BPF helper function.
832 	 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
833 	 * helpers, they do not cover all possible instances of unsafe
834 	 * pointers. For example, a pointer that was obtained from walking a
835 	 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
836 	 * fact that it may be NULL, invalid, etc. This is due to backwards
837 	 * compatibility requirements, as this was the behavior that was first
838 	 * introduced when kptrs were added. The behavior is now considered
839 	 * deprecated, and PTR_UNTRUSTED will eventually be removed.
840 	 *
841 	 * PTR_TRUSTED, on the other hand, is a pointer that the kernel
842 	 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
843 	 * For example, pointers passed to tracepoint arguments are considered
844 	 * PTR_TRUSTED, as are pointers that are passed to struct_ops
845 	 * callbacks. As alluded to above, pointers that are obtained from
846 	 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
847 	 * struct task_struct *task is PTR_TRUSTED, then accessing
848 	 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
849 	 * in a BPF register. Similarly, pointers passed to certain programs
850 	 * types such as kretprobes are not guaranteed to be valid, as they may
851 	 * for example contain an object that was recently freed.
852 	 */
853 	PTR_TRUSTED		= BIT(12 + BPF_BASE_TYPE_BITS),
854 
855 	/* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
856 	MEM_RCU			= BIT(13 + BPF_BASE_TYPE_BITS),
857 
858 	/* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
859 	 * Currently only valid for linked-list and rbtree nodes. If the nodes
860 	 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
861 	 */
862 	NON_OWN_REF		= BIT(14 + BPF_BASE_TYPE_BITS),
863 
864 	/* DYNPTR points to sk_buff */
865 	DYNPTR_TYPE_SKB		= BIT(15 + BPF_BASE_TYPE_BITS),
866 
867 	/* DYNPTR points to xdp_buff */
868 	DYNPTR_TYPE_XDP		= BIT(16 + BPF_BASE_TYPE_BITS),
869 
870 	/* Memory must be aligned on some architectures, used in combination with
871 	 * MEM_FIXED_SIZE.
872 	 */
873 	MEM_ALIGNED		= BIT(17 + BPF_BASE_TYPE_BITS),
874 
875 	/* MEM is being written to, often combined with MEM_UNINIT. Non-presence
876 	 * of MEM_WRITE means that MEM is only being read. MEM_WRITE without the
877 	 * MEM_UNINIT means that memory needs to be initialized since it is also
878 	 * read.
879 	 */
880 	MEM_WRITE		= BIT(18 + BPF_BASE_TYPE_BITS),
881 
882 	/* DYNPTR points to skb_metadata_end()-skb_metadata_len() */
883 	DYNPTR_TYPE_SKB_META	= BIT(19 + BPF_BASE_TYPE_BITS),
884 
885 	/* DYNPTR points to file */
886 	DYNPTR_TYPE_FILE	= BIT(20 + BPF_BASE_TYPE_BITS),
887 
888 	__BPF_TYPE_FLAG_MAX,
889 	__BPF_TYPE_LAST_FLAG	= __BPF_TYPE_FLAG_MAX - 1,
890 };
891 
892 #define DYNPTR_TYPE_FLAG_MASK	(DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
893 				 | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META | DYNPTR_TYPE_FILE)
894 
895 /* Max number of base types. */
896 #define BPF_BASE_TYPE_LIMIT	(1UL << BPF_BASE_TYPE_BITS)
897 
898 /* Max number of all types. */
899 #define BPF_TYPE_LIMIT		(__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
900 
901 /* function argument constraints */
902 enum bpf_arg_type {
903 	ARG_UNUSED = 0,		/* unused argument; terminates argument iteration */
904 
905 	/* the following constraints used to prototype
906 	 * bpf_map_lookup/update/delete_elem() functions
907 	 */
908 	ARG_CONST_MAP_PTR,	/* const argument used as pointer to bpf_map */
909 	ARG_PTR_TO_MAP_KEY,	/* pointer to stack used as map key */
910 	ARG_PTR_TO_MAP_VALUE,	/* pointer to stack used as map value */
911 
912 	/* Used to prototype bpf_memcmp() and other functions that access data
913 	 * on eBPF program stack
914 	 */
915 	ARG_PTR_TO_MEM,		/* pointer to valid memory (stack, packet, map value) */
916 	ARG_PTR_TO_ARENA,
917 
918 	ARG_MEM_SIZE,		/* number of bytes accessed from memory */
919 	ARG_MEM_SIZE_OR_ZERO,	/* number of bytes accessed from memory or 0 */
920 
921 	ARG_PTR_TO_CTX,		/* pointer to context */
922 	ARG_ANYTHING,		/* any (initialized) argument is ok */
923 	ARG_SCALAR,		/* scalar argument */
924 	ARG_PTR_TO_SPIN_LOCK,	/* pointer to bpf_spin_lock */
925 	ARG_PTR_TO_SOCK_COMMON,	/* pointer to sock_common */
926 	ARG_PTR_TO_SOCKET,	/* pointer to bpf_sock (fullsock) */
927 	ARG_PTR_TO_BTF_ID,	/* pointer to in-kernel struct */
928 	ARG_PTR_TO_RINGBUF_MEM,	/* pointer to dynamically reserved ringbuf memory */
929 	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */
930 	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
931 	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */
932 	ARG_PTR_TO_FUNC,	/* pointer to a bpf program function */
933 	ARG_PTR_TO_STACK,	/* pointer to stack */
934 	ARG_PTR_TO_CONST_STR,	/* pointer to a null terminated read-only string */
935 	ARG_PTR_TO_TIMER,	/* pointer to bpf_timer */
936 	ARG_KPTR_XCHG_DEST,	/* pointer to destination that kptrs are bpf_kptr_xchg'd into */
937 	ARG_PTR_TO_DYNPTR,      /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
938 
939 	ARG_CONST_SCALAR,	/* scalar known at verification time */
940 	ARG_CONST_MEM_SIZE,	/* ARG_MEM_SIZE that must be constant */
941 	ARG_PTR_TO_ALLOC_BTF_ID,	/* pointer to an allocated object */
942 	ARG_PTR_TO_REFCOUNTED_KPTR,	/* pointer to a refcounted local kptr */
943 	ARG_PTR_TO_ITER,	/* pointer to an iterator */
944 	ARG_PTR_TO_LIST_HEAD,	/* pointer to bpf_list_head */
945 	ARG_PTR_TO_LIST_NODE,	/* pointer to bpf_list_node */
946 	ARG_PTR_TO_RB_ROOT,	/* pointer to bpf_rb_root */
947 	ARG_PTR_TO_RB_NODE,	/* pointer to bpf_rb_node */
948 	ARG_PTR_TO_WORKQUEUE,	/* pointer to bpf_wq */
949 	ARG_PTR_TO_TASK_WORK,	/* pointer to bpf_task_work */
950 	ARG_PTR_TO_RCU_HEAD,	/* pointer to bpf_rcu_head */
951 	ARG_PTR_TO_IRQ_FLAG,	/* pointer to saved IRQ flags on the stack */
952 	ARG_PTR_TO_RES_SPIN_LOCK,	/* pointer to bpf_res_spin_lock */
953 	ARG_PTR_TO_CTX_OUT,	/* hook output argument passed through from ctx */
954 	ARG_PTR_TO_PROG_AUX,	/* pointer to the caller's bpf_prog_aux */
955 	ARG_IGNORE,		/* argument the verifier does not check at all */
956 	__BPF_ARG_TYPE_MAX,
957 
958 	/* Extended arg_types. */
959 	ARG_PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
960 	ARG_PTR_TO_MEM_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
961 	ARG_PTR_TO_CTX_OR_NULL		= PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
962 	ARG_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
963 	ARG_PTR_TO_STACK_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
964 	ARG_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
965 	/* Pointer to memory does not need to be initialized, since helper function
966 	 * fills all bytes or clears them in error case.
967 	 */
968 	ARG_PTR_TO_UNINIT_MEM		= MEM_UNINIT | MEM_WRITE | ARG_PTR_TO_MEM,
969 	/* Pointer to valid memory of size known at compile time. */
970 	ARG_PTR_TO_FIXED_SIZE_MEM	= MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
971 
972 	/* This must be the last entry. Its purpose is to ensure the enum is
973 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
974 	 */
975 	__BPF_ARG_TYPE_LIMIT	= BPF_TYPE_LIMIT,
976 };
977 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
978 
979 /* type of values returned from helper functions */
980 enum bpf_return_type {
981 	RET_INTEGER,			/* function returns integer */
982 	RET_VOID,			/* function doesn't return anything */
983 	RET_PTR_TO_MAP_VALUE,		/* returns a pointer to map elem value */
984 	RET_PTR_TO_SOCKET,		/* returns a pointer to a socket */
985 	RET_PTR_TO_TCP_SOCK,		/* returns a pointer to a tcp_sock */
986 	RET_PTR_TO_SOCK_COMMON,		/* returns a pointer to a sock_common */
987 	RET_PTR_TO_MEM,			/* returns a pointer to memory */
988 	RET_PTR_TO_MEM_OR_BTF_ID,	/* returns a pointer to a valid memory or a btf_id */
989 	RET_PTR_TO_BTF_ID,		/* returns a pointer to a btf_id */
990 	__BPF_RET_TYPE_MAX,
991 
992 	/* Extended ret_types. */
993 	RET_PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
994 	RET_PTR_TO_SOCKET_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
995 	RET_PTR_TO_TCP_SOCK_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
996 	RET_PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
997 	RET_PTR_TO_RINGBUF_MEM_OR_NULL	= PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
998 	RET_PTR_TO_DYNPTR_MEM_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_MEM,
999 	RET_PTR_TO_BTF_ID_OR_NULL	= PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
1000 	RET_PTR_TO_BTF_ID_TRUSTED	= PTR_TRUSTED	 | RET_PTR_TO_BTF_ID,
1001 
1002 	/* This must be the last entry. Its purpose is to ensure the enum is
1003 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
1004 	 */
1005 	__BPF_RET_TYPE_LIMIT	= BPF_TYPE_LIMIT,
1006 };
1007 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
1008 
1009 /* The longest tracepoint has 12 args.
1010  * See include/trace/bpf_probe.h
1011  *
1012  * Also reuse this macro for maximum number of arguments a BPF function
1013  * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via
1014  * stack arg slots. The JIT may map some stack arg slots to registers based
1015  * on the native calling convention (e.g., arg 6 to R9 on x86-64).
1016  */
1017 #define MAX_BPF_FUNC_ARGS 12
1018 
1019 /* The maximum number of arguments passed through registers
1020  * a single function may have.
1021  */
1022 #define MAX_BPF_FUNC_REG_ARGS 5
1023 
1024 /* A by-value argument takes two eightbytes at most, so the maximum number of
1025  * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may
1026  * need that size for processing, although eventually the maximum slots will
1027  * be capped at MAX_BPF_FUNC_ARGS.
1028  */
1029 #define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS)
1030 
1031 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
1032  * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
1033  * instructions after verifying
1034  */
1035 struct bpf_func_proto {
1036 	u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
1037 	bool gpl_only;
1038 	bool pkt_access;
1039 	bool might_sleep;
1040 	/* set to true if helper follows contract for llvm
1041 	 * attribute bpf_fastcall:
1042 	 * - void functions do not scratch r0
1043 	 * - functions taking N arguments scratch only registers r1-rN
1044 	 */
1045 	bool allow_fastcall;
1046 	enum bpf_return_type ret_type;
1047 	union {
1048 		struct {
1049 			enum bpf_arg_type arg1_type;
1050 			enum bpf_arg_type arg2_type;
1051 			enum bpf_arg_type arg3_type;
1052 			enum bpf_arg_type arg4_type;
1053 			enum bpf_arg_type arg5_type;
1054 		};
1055 		enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS];
1056 	};
1057 	union {
1058 		struct {
1059 			const u32 *arg1_btf_id;
1060 			const u32 *arg2_btf_id;
1061 			const u32 *arg3_btf_id;
1062 			const u32 *arg4_btf_id;
1063 			const u32 *arg5_btf_id;
1064 		};
1065 		const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS];
1066 		struct {
1067 			size_t arg1_size;
1068 			size_t arg2_size;
1069 			size_t arg3_size;
1070 			size_t arg4_size;
1071 			size_t arg5_size;
1072 		};
1073 		size_t arg_size[MAX_BPF_FUNC_ARGS];
1074 	};
1075 	int *ret_btf_id; /* return value btf_id */
1076 	bool (*allowed)(const struct bpf_prog *prog);
1077 };
1078 
1079 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
1080  * the first argument to eBPF programs.
1081  * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
1082  */
1083 struct bpf_context;
1084 
1085 enum bpf_access_type {
1086 	BPF_READ = 1,
1087 	BPF_WRITE = 2
1088 };
1089 
1090 /* types of values stored in eBPF registers */
1091 /* Pointer types represent:
1092  * pointer
1093  * pointer + imm
1094  * pointer + (u16) var
1095  * pointer + (u16) var + imm
1096  * if (range > 0) then [ptr, ptr + range - off) is safe to access
1097  * if (id > 0) means that some 'var' was added
1098  * if (off > 0) means that 'imm' was added
1099  */
1100 enum bpf_reg_type {
1101 	NOT_INIT = 0,		 /* nothing was written into register */
1102 	SCALAR_VALUE,		 /* reg doesn't contain a valid pointer */
1103 	PTR_TO_CTX,		 /* reg points to bpf_context */
1104 	CONST_PTR_TO_MAP,	 /* reg points to struct bpf_map */
1105 	PTR_TO_MAP_VALUE,	 /* reg points to map element value */
1106 	PTR_TO_MAP_KEY,		 /* reg points to a map element key */
1107 	PTR_TO_STACK,		 /* reg == frame_pointer + offset */
1108 	PTR_TO_PACKET_META,	 /* skb->data - meta_len */
1109 	PTR_TO_PACKET,		 /* reg points to skb->data */
1110 	PTR_TO_PACKET_END,	 /* skb->data + headlen */
1111 	PTR_TO_FLOW_KEYS,	 /* reg points to bpf_flow_keys */
1112 	PTR_TO_SOCKET,		 /* reg points to struct bpf_sock */
1113 	PTR_TO_SOCK_COMMON,	 /* reg points to sock_common */
1114 	PTR_TO_TCP_SOCK,	 /* reg points to struct tcp_sock */
1115 	PTR_TO_TP_BUFFER,	 /* reg points to a writable raw tp's buffer */
1116 	PTR_TO_XDP_SOCK,	 /* reg points to struct xdp_sock */
1117 	/* PTR_TO_BTF_ID points to a kernel struct that does not need
1118 	 * to be null checked by the BPF program. This does not imply the
1119 	 * pointer is _not_ null and in practice this can easily be a null
1120 	 * pointer when reading pointer chains. The assumption is program
1121 	 * context will handle null pointer dereference typically via fault
1122 	 * handling. The verifier must keep this in mind and can make no
1123 	 * assumptions about null or non-null when doing branch analysis.
1124 	 * Further, when passed into helpers the helpers can not, without
1125 	 * additional context, assume the value is non-null.
1126 	 */
1127 	PTR_TO_BTF_ID,
1128 	PTR_TO_MEM,		 /* reg points to valid memory region */
1129 	PTR_TO_ARENA,
1130 	PTR_TO_BUF,		 /* reg points to a read/write buffer */
1131 	PTR_TO_FUNC,		 /* reg points to a bpf program function */
1132 	PTR_TO_INSN,		 /* reg points to a bpf program instruction */
1133 	CONST_PTR_TO_DYNPTR,	 /* reg points to a const struct bpf_dynptr */
1134 	__BPF_REG_TYPE_MAX,
1135 
1136 	/* Extended reg_types. */
1137 	PTR_TO_MAP_VALUE_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
1138 	PTR_TO_SOCKET_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_SOCKET,
1139 	PTR_TO_SOCK_COMMON_OR_NULL	= PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
1140 	PTR_TO_TCP_SOCK_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
1141 	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
1142 	 * been checked for null. Used primarily to inform the verifier
1143 	 * an explicit null check is required for this struct.
1144 	 */
1145 	PTR_TO_BTF_ID_OR_NULL		= PTR_MAYBE_NULL | PTR_TO_BTF_ID,
1146 
1147 	/* This must be the last entry. Its purpose is to ensure the enum is
1148 	 * wide enough to hold the higher bits reserved for bpf_type_flag.
1149 	 */
1150 	__BPF_REG_TYPE_LIMIT	= BPF_TYPE_LIMIT,
1151 };
1152 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
1153 
1154 /* The information passed from prog-specific *_is_valid_access
1155  * back to the verifier.
1156  */
1157 struct bpf_insn_access_aux {
1158 	enum bpf_reg_type reg_type;
1159 	bool is_ldsx;
1160 	union {
1161 		int ctx_field_size;
1162 		struct {
1163 			struct btf *btf;
1164 			u32 btf_id;
1165 			u32 ref_id;
1166 		};
1167 	};
1168 	u32 mem_size;
1169 	struct bpf_verifier_log *log; /* for verbose logs */
1170 	bool is_retval; /* is accessing function return value ? */
1171 };
1172 
1173 static inline void
1174 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
1175 {
1176 	aux->ctx_field_size = size;
1177 }
1178 
1179 static bool bpf_is_ldimm64(const struct bpf_insn *insn)
1180 {
1181 	return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
1182 }
1183 
1184 static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
1185 {
1186 	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
1187 }
1188 
1189 struct bpf_prog_ops {
1190 	int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
1191 			union bpf_attr __user *uattr);
1192 };
1193 
1194 struct bpf_reg_state;
1195 struct bpf_verifier_ops {
1196 	/* return eBPF function prototype for verification */
1197 	const struct bpf_func_proto *
1198 	(*get_func_proto)(enum bpf_func_id func_id,
1199 			  const struct bpf_prog *prog);
1200 
1201 	/* return true if 'size' wide access at offset 'off' within bpf_context
1202 	 * with 'type' (read or write) is allowed
1203 	 */
1204 	bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
1205 				const struct bpf_prog *prog,
1206 				struct bpf_insn_access_aux *info);
1207 	int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
1208 			    const struct bpf_prog *prog);
1209 	int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog,
1210 			    s16 ctx_stack_off);
1211 	int (*gen_ld_abs)(const struct bpf_insn *orig,
1212 			  struct bpf_insn *insn_buf);
1213 	u32 (*convert_ctx_access)(enum bpf_access_type type,
1214 				  const struct bpf_insn *src,
1215 				  struct bpf_insn *dst,
1216 				  struct bpf_prog *prog, u32 *target_size);
1217 	int (*btf_struct_access)(struct bpf_verifier_log *log,
1218 				 const struct bpf_reg_state *reg,
1219 				 int off, int size);
1220 };
1221 
1222 struct bpf_prog_offload_ops {
1223 	/* verifier basic callbacks */
1224 	int (*insn_hook)(struct bpf_verifier_env *env,
1225 			 int insn_idx, int prev_insn_idx);
1226 	int (*finalize)(struct bpf_verifier_env *env);
1227 	/* verifier optimization callbacks (called after .finalize) */
1228 	int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
1229 			    struct bpf_insn *insn);
1230 	int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
1231 	/* program management callbacks */
1232 	int (*prepare)(struct bpf_prog *prog);
1233 	int (*translate)(struct bpf_prog *prog);
1234 	void (*destroy)(struct bpf_prog *prog);
1235 };
1236 
1237 struct bpf_prog_offload {
1238 	struct bpf_prog		*prog;
1239 	struct net_device	*netdev;
1240 	struct bpf_offload_dev	*offdev;
1241 	void			*dev_priv;
1242 	struct list_head	offloads;
1243 	bool			dev_state;
1244 	bool			opt_failed;
1245 	void			*jited_image;
1246 	u32			jited_len;
1247 };
1248 
1249 /* The argument is aligned to 16 bytes. */
1250 #define BTF_FMODEL_ALIGN16_ARG		BIT(0)
1251 
1252 /* The argument is signed. */
1253 #define BTF_FMODEL_SIGNED_ARG		BIT(1)
1254 
1255 /* The argument is an arena pointer. */
1256 #define BTF_FMODEL_ARENA_ARG		BIT(2)
1257 
1258 /* The argument is nullable. */
1259 #define BTF_FMODEL_NULLABLE_ARG		BIT(3)
1260 
1261 struct btf_func_model {
1262 	u8 ret_size;
1263 	u8 ret_flags;
1264 	u8 nr_args;
1265 	u8 arg_size[MAX_BPF_FUNC_ARGS];
1266 	u8 arg_flags[MAX_BPF_FUNC_ARGS];
1267 };
1268 
1269 static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg)
1270 {
1271 	return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64);
1272 }
1273 
1274 /* Restore arguments before returning from trampoline to let original function
1275  * continue executing. This flag is used for fentry progs when there are no
1276  * fexit progs.
1277  */
1278 #define BPF_TRAMP_F_RESTORE_REGS	BIT(0)
1279 /* Call original function after fentry progs, but before fexit progs.
1280  * Makes sense for fentry/fexit, normal calls and indirect calls.
1281  */
1282 #define BPF_TRAMP_F_CALL_ORIG		BIT(1)
1283 /* Skip current frame and return to parent.  Makes sense for fentry/fexit
1284  * programs only. Should not be used with normal calls and indirect calls.
1285  */
1286 #define BPF_TRAMP_F_SKIP_FRAME		BIT(2)
1287 /* Store IP address of the caller on the trampoline stack,
1288  * so it's available for trampoline's programs.
1289  */
1290 #define BPF_TRAMP_F_IP_ARG		BIT(3)
1291 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1292 #define BPF_TRAMP_F_RET_FENTRY_RET	BIT(4)
1293 
1294 /* Get original function from stack instead of from provided direct address.
1295  * Makes sense for trampolines with fexit or fmod_ret programs.
1296  */
1297 #define BPF_TRAMP_F_ORIG_STACK		BIT(5)
1298 
1299 /* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1300  * e.g., a live patch. This flag is set and cleared by ftrace call backs,
1301  */
1302 #define BPF_TRAMP_F_SHARE_IPMODIFY	BIT(6)
1303 
1304 /* Indicate that current trampoline is in a tail call context. Then, it has to
1305  * cache and restore tail_call_cnt to avoid infinite tail call loop.
1306  */
1307 #define BPF_TRAMP_F_TAIL_CALL_CTX	BIT(7)
1308 
1309 /*
1310  * Indicate the trampoline should be suitable to receive indirect calls;
1311  * without this indirectly calling the generated code can result in #UD/#CP,
1312  * depending on the CFI options.
1313  *
1314  * Used by bpf_struct_ops.
1315  *
1316  * Incompatible with FENTRY usage, overloads @func_addr argument.
1317  */
1318 #define BPF_TRAMP_F_INDIRECT		BIT(8)
1319 
1320 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1321  * bytes on x86. The trampoline image has to fit in PAGE_SIZE.
1322  */
1323 enum {
1324 #if defined(__s390x__) || defined(__powerpc64__)
1325 	BPF_MAX_TRAMP_LINKS = 27,
1326 #elif defined(__x86_64__)
1327 	BPF_MAX_TRAMP_LINKS = 36,
1328 #elif defined(__aarch64__)
1329 	BPF_MAX_TRAMP_LINKS = 37,
1330 #else
1331 	BPF_MAX_TRAMP_LINKS = 38,
1332 #endif
1333 };
1334 
1335 #define BPF_TRAMP_COOKIE_INDEX_SHIFT	8
1336 #define BPF_TRAMP_IS_RETURN_SHIFT	63
1337 
1338 struct bpf_tramp_nodes {
1339 	struct bpf_tramp_node *nodes[BPF_MAX_TRAMP_LINKS];
1340 	int nr_nodes;
1341 };
1342 
1343 /*
1344  * The arena base against which a struct_ops trampoline converts the
1345  * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF
1346  * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline
1347  * converts nothing.
1348  */
1349 u64 bpf_tramp_arena_base(const struct btf_func_model *m,
1350 			 struct bpf_tramp_nodes *tnodes, u32 flags);
1351 
1352 struct bpf_tramp_run_ctx;
1353 
1354 /* Different use cases for BPF trampoline:
1355  * 1. replace nop at the function entry (kprobe equivalent)
1356  *    flags = BPF_TRAMP_F_RESTORE_REGS
1357  *    fentry = a set of programs to run before returning from trampoline
1358  *
1359  * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1360  *    flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
1361  *    orig_call = fentry_ip + MCOUNT_INSN_SIZE
1362  *    fentry = a set of program to run before calling original function
1363  *    fexit = a set of program to run after original function
1364  *
1365  * 3. replace direct call instruction anywhere in the function body
1366  *    or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1367  *    With flags = 0
1368  *      fentry = a set of programs to run before returning from trampoline
1369  *    With flags = BPF_TRAMP_F_CALL_ORIG
1370  *      orig_call = original callback addr or direct function addr
1371  *      fentry = a set of program to run before calling original function
1372  *      fexit = a set of program to run after original function
1373  */
1374 struct bpf_tramp_image;
1375 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1376 				const struct btf_func_model *m, u32 flags,
1377 				struct bpf_tramp_nodes *tnodes,
1378 				void *func_addr);
1379 void *arch_alloc_bpf_trampoline(unsigned int size);
1380 void arch_free_bpf_trampoline(void *image, unsigned int size);
1381 int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
1382 int arch_bpf_trampoline_skip(void *nop, void *target);
1383 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1384 			     struct bpf_tramp_nodes *tnodes, void *func_addr);
1385 
1386 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1387 					     struct bpf_tramp_run_ctx *run_ctx);
1388 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1389 					     struct bpf_tramp_run_ctx *run_ctx);
1390 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
1391 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
1392 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
1393 				      struct bpf_tramp_run_ctx *run_ctx);
1394 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
1395 				      struct bpf_tramp_run_ctx *run_ctx);
1396 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
1397 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
1398 
1399 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP
1400 static inline bool bpf_trampoline_use_jmp(u64 flags)
1401 {
1402 	return flags & BPF_TRAMP_F_CALL_ORIG && !(flags & BPF_TRAMP_F_SKIP_FRAME);
1403 }
1404 #else
1405 static inline bool bpf_trampoline_use_jmp(u64 flags)
1406 {
1407 	return false;
1408 }
1409 #endif
1410 
1411 struct bpf_ksym {
1412 	unsigned long		 start;
1413 	unsigned long		 end;
1414 	char			 name[KSYM_NAME_LEN];
1415 	struct list_head	 lnode;
1416 	struct latch_tree_node	 tnode;
1417 	bool			 prog;
1418 	u32			 fp_start;
1419 	u32			 fp_end;
1420 };
1421 
1422 enum bpf_tramp_prog_type {
1423 	BPF_TRAMP_FENTRY,
1424 	BPF_TRAMP_FEXIT,
1425 	BPF_TRAMP_MODIFY_RETURN,
1426 	BPF_TRAMP_MAX,
1427 	BPF_TRAMP_REPLACE, /* more than MAX */
1428 	BPF_TRAMP_FSESSION,
1429 };
1430 
1431 /*
1432  * Each prog call in a trampoline image is preceded by a nop. When the prog is
1433  * detached, the nop is patched to a jump to target, right after the call, so
1434  * that tasks still running in the image skip the prog.
1435  */
1436 struct bpf_tramp_skip {
1437 	struct bpf_prog *prog;
1438 	void *nop;
1439 	void *target;
1440 };
1441 
1442 struct bpf_tramp_image {
1443 	void *image;
1444 	int size;
1445 	struct bpf_ksym ksym;
1446 	struct percpu_ref pcref;
1447 	bool call_orig;
1448 	/* entry in tr->images, the image holds a reference on tr */
1449 	struct bpf_trampoline *tr;
1450 	struct list_head list;
1451 	int nr_skips;
1452 	struct bpf_tramp_skip *skips;
1453 	union {
1454 		struct rcu_head rcu;
1455 		struct work_struct work;
1456 	};
1457 };
1458 
1459 static inline void bpf_tramp_image_add_skip(struct bpf_tramp_image *im, struct bpf_prog *prog,
1460 					    void *nop, void *target)
1461 {
1462 	struct bpf_tramp_skip *skip;
1463 
1464 	/* struct_ops trampolines and arch_bpf_trampoline_size() have no image */
1465 	if (!im || !im->skips)
1466 		return;
1467 	skip = &im->skips[im->nr_skips++];
1468 	skip->prog = prog;
1469 	skip->nop = nop;
1470 	skip->target = target;
1471 }
1472 
1473 struct bpf_trampoline {
1474 	/* hlist for trampoline_key_table */
1475 	struct hlist_node hlist_key;
1476 	/* hlist for trampoline_ip_table */
1477 	struct hlist_node hlist_ip;
1478 	struct ftrace_ops *fops;
1479 	refcount_t refcnt;
1480 	u32 flags;
1481 	u64 key;
1482 	unsigned long ip;
1483 	struct {
1484 		struct btf_func_model model;
1485 		void *addr;
1486 		bool ftrace_managed;
1487 	} func;
1488 	/* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1489 	 * program by replacing one of its functions. func.addr is the address
1490 	 * of the function it replaced.
1491 	 */
1492 	struct bpf_prog *extension_prog;
1493 	/* list of BPF programs using this trampoline */
1494 	struct hlist_head progs_hlist[BPF_TRAMP_MAX];
1495 	/* Number of attached programs. A counter per kind. */
1496 	int progs_cnt[BPF_TRAMP_MAX];
1497 	/* Executable image of trampoline */
1498 	struct bpf_tramp_image *cur_image;
1499 	/* Images not freed yet, cur_image and older ones still in use */
1500 	struct list_head images;
1501 	/* Used as temporary old image storage for multi_attach */
1502 	struct {
1503 		struct bpf_tramp_image *old_image;
1504 		u32 old_flags;
1505 	} multi_attach;
1506 };
1507 
1508 struct bpf_attach_target_info {
1509 	struct btf_func_model fmodel;
1510 	long tgt_addr;
1511 	struct module *tgt_mod;
1512 	const char *tgt_name;
1513 	const struct btf_type *tgt_type;
1514 };
1515 
1516 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1517 
1518 struct bpf_dispatcher_prog {
1519 	struct bpf_prog *prog;
1520 	refcount_t users;
1521 };
1522 
1523 struct bpf_dispatcher {
1524 	/* dispatcher mutex */
1525 	struct mutex mutex;
1526 	void *func;
1527 	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
1528 	int num_progs;
1529 	void *image;
1530 	void *rw_image;
1531 	u32 image_off;
1532 	struct bpf_ksym ksym;
1533 #ifdef CONFIG_HAVE_STATIC_CALL
1534 	struct static_call_key *sc_key;
1535 	void *sc_tramp;
1536 #endif
1537 };
1538 
1539 #ifndef __bpfcall
1540 #define __bpfcall __nocfi
1541 #endif
1542 
1543 static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(
1544 	const void *ctx,
1545 	const struct bpf_insn *insnsi,
1546 	bpf_func_t bpf_func)
1547 {
1548 	return bpf_func(ctx, insnsi);
1549 }
1550 
1551 /* the implementation of the opaque uapi struct bpf_dynptr */
1552 struct bpf_dynptr_kern {
1553 	void *data;
1554 	/* Size represents the number of usable bytes of dynptr data.
1555 	 * If for example the offset is at 4 for a local dynptr whose data is
1556 	 * of type u64, the number of usable bytes is 4.
1557 	 *
1558 	 * The upper 8 bits are reserved. It is as follows:
1559 	 * Bits 0 - 23 = size
1560 	 * Bits 24 - 30 = dynptr type
1561 	 * Bit 31 = whether dynptr is read-only
1562 	 */
1563 	u32 size;
1564 	u32 offset;
1565 } __aligned(8);
1566 
1567 enum bpf_dynptr_type {
1568 	BPF_DYNPTR_TYPE_INVALID,
1569 	/* Points to memory that is local to the bpf program */
1570 	BPF_DYNPTR_TYPE_LOCAL,
1571 	/* Underlying data is a ringbuf record */
1572 	BPF_DYNPTR_TYPE_RINGBUF,
1573 	/* Underlying data is a sk_buff */
1574 	BPF_DYNPTR_TYPE_SKB,
1575 	/* Underlying data is a xdp_buff */
1576 	BPF_DYNPTR_TYPE_XDP,
1577 	/* Points to skb_metadata_end()-skb_metadata_len() */
1578 	BPF_DYNPTR_TYPE_SKB_META,
1579 	/* Underlying data is a file */
1580 	BPF_DYNPTR_TYPE_FILE,
1581 };
1582 
1583 int bpf_dynptr_check_size(u64 size);
1584 u64 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
1585 const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u64 len);
1586 void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u64 len);
1587 bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
1588 int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u64 offset,
1589 		       void *src, u64 len, u64 flags);
1590 void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u64 offset,
1591 			    void *buffer__nullable, u64 buffer__szk);
1592 
1593 static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u64 offset, u64 len)
1594 {
1595 	u64 size = __bpf_dynptr_size(ptr);
1596 
1597 	if (len > size || offset > size - len)
1598 		return -E2BIG;
1599 
1600 	return 0;
1601 }
1602 
1603 struct bpf_tracing_multi_link;
1604 
1605 #ifdef CONFIG_BPF_JIT
1606 int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
1607 			     struct bpf_trampoline *tr,
1608 			     struct bpf_prog *tgt_prog);
1609 int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
1610 			       struct bpf_trampoline *tr,
1611 			       struct bpf_prog *tgt_prog);
1612 struct bpf_trampoline *bpf_trampoline_get(u64 key,
1613 					  struct bpf_attach_target_info *tgt_info);
1614 void bpf_trampoline_put(struct bpf_trampoline *tr);
1615 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
1616 
1617 int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1618 				struct bpf_tracing_multi_link *link);
1619 void bpf_trampoline_multi_detach(struct bpf_prog *prog,
1620 				 struct bpf_tracing_multi_link *link);
1621 void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags);
1622 
1623 /*
1624  * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1625  * indirection with a direct call to the bpf program. If the architecture does
1626  * not have STATIC_CALL, avoid a double-indirection.
1627  */
1628 #ifdef CONFIG_HAVE_STATIC_CALL
1629 
1630 #define __BPF_DISPATCHER_SC_INIT(_name)				\
1631 	.sc_key = &STATIC_CALL_KEY(_name),			\
1632 	.sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1633 
1634 #define __BPF_DISPATCHER_SC(name)				\
1635 	DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1636 
1637 #define __BPF_DISPATCHER_CALL(name)				\
1638 	static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1639 
1640 #define __BPF_DISPATCHER_UPDATE(_d, _new)			\
1641 	__static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1642 
1643 #else
1644 #define __BPF_DISPATCHER_SC_INIT(name)
1645 #define __BPF_DISPATCHER_SC(name)
1646 #define __BPF_DISPATCHER_CALL(name)		bpf_func(ctx, insnsi)
1647 #define __BPF_DISPATCHER_UPDATE(_d, _new)
1648 #endif
1649 
1650 #define BPF_DISPATCHER_INIT(_name) {				\
1651 	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\
1652 	.func = &_name##_func,					\
1653 	.progs = {},						\
1654 	.num_progs = 0,						\
1655 	.image = NULL,						\
1656 	.image_off = 0,						\
1657 	.ksym = {						\
1658 		.name  = #_name,				\
1659 		.lnode = LIST_HEAD_INIT(_name.ksym.lnode),	\
1660 	},							\
1661 	__BPF_DISPATCHER_SC_INIT(_name##_call)			\
1662 }
1663 
1664 #define DEFINE_BPF_DISPATCHER(name)					\
1665 	__BPF_DISPATCHER_SC(name);					\
1666 	noinline __bpfcall unsigned int bpf_dispatcher_##name##_func(	\
1667 		const void *ctx,					\
1668 		const struct bpf_insn *insnsi,				\
1669 		bpf_func_t bpf_func)					\
1670 	{								\
1671 		return __BPF_DISPATCHER_CALL(name);			\
1672 	}								\
1673 	EXPORT_SYMBOL(bpf_dispatcher_##name##_func);			\
1674 	struct bpf_dispatcher bpf_dispatcher_##name =			\
1675 		BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1676 
1677 #define DECLARE_BPF_DISPATCHER(name)					\
1678 	unsigned int bpf_dispatcher_##name##_func(			\
1679 		const void *ctx,					\
1680 		const struct bpf_insn *insnsi,				\
1681 		bpf_func_t bpf_func);					\
1682 	extern struct bpf_dispatcher bpf_dispatcher_##name;
1683 
1684 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1685 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1686 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1687 				struct bpf_prog *to);
1688 /* Called only from JIT-enabled code, so there's no need for stubs. */
1689 void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
1690 void bpf_image_ksym_add(struct bpf_ksym *ksym);
1691 void bpf_image_ksym_del(struct bpf_ksym *ksym);
1692 void bpf_ksym_add(struct bpf_ksym *ksym);
1693 void bpf_ksym_del(struct bpf_ksym *ksym);
1694 bool bpf_has_frame_pointer(unsigned long ip);
1695 int bpf_jit_charge_modmem(u32 size);
1696 void bpf_jit_uncharge_modmem(u32 size);
1697 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1698 bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
1699 				 int insn_idx);
1700 u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog);
1701 #else
1702 static inline int bpf_trampoline_link_prog(struct bpf_tramp_node *node,
1703 					   struct bpf_trampoline *tr,
1704 					   struct bpf_prog *tgt_prog)
1705 {
1706 	return -ENOTSUPP;
1707 }
1708 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node,
1709 					     struct bpf_trampoline *tr,
1710 					     struct bpf_prog *tgt_prog)
1711 {
1712 	return -ENOTSUPP;
1713 }
1714 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
1715 							struct bpf_attach_target_info *tgt_info)
1716 {
1717 	return NULL;
1718 }
1719 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
1720 #define DEFINE_BPF_DISPATCHER(name)
1721 #define DECLARE_BPF_DISPATCHER(name)
1722 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1723 #define BPF_DISPATCHER_PTR(name) NULL
1724 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
1725 					      struct bpf_prog *from,
1726 					      struct bpf_prog *to) {}
1727 static inline bool is_bpf_image_address(unsigned long address)
1728 {
1729 	return false;
1730 }
1731 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
1732 {
1733 	return false;
1734 }
1735 static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids,
1736 					      struct bpf_tracing_multi_link *link)
1737 {
1738 	return -ENOTSUPP;
1739 }
1740 static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog,
1741 					       struct bpf_tracing_multi_link *link)
1742 {
1743 }
1744 static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {}
1745 #endif
1746 
1747 /* The contexts that a global function is called in, and is verified for. */
1748 enum bpf_func_context {
1749 	BPF_FUNC_CTX_NON_SLEEPABLE,
1750 	BPF_FUNC_CTX_SLEEPABLE,
1751 	/* Sleepable code has disabled preemption or interrupts: no RCU read lock. */
1752 	BPF_FUNC_CTX_NO_RCU_LOCK,
1753 	BPF_FUNC_CTX_CNT,
1754 };
1755 
1756 struct bpf_subprog_arg_info {
1757 	enum bpf_arg_type arg_type;
1758 	union {
1759 		u32 mem_size;
1760 		u32 btf_id;
1761 	};
1762 };
1763 
1764 struct bpf_func_info_aux {
1765 	u16 linkage;
1766 	bool unreliable;
1767 	bool called[BPF_FUNC_CTX_CNT];
1768 	bool verified[BPF_FUNC_CTX_CNT];
1769 	bool called_with_pkt_ptrs;
1770 	/* what the callers of a global function are checked against */
1771 	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
1772 };
1773 
1774 enum bpf_jit_poke_reason {
1775 	BPF_POKE_REASON_TAIL_CALL,
1776 };
1777 
1778 /* Descriptor of pokes pointing /into/ the JITed image. */
1779 struct bpf_jit_poke_descriptor {
1780 	void *tailcall_target;
1781 	void *tailcall_bypass;
1782 	void *bypass_addr;
1783 	void *aux;
1784 	union {
1785 		struct {
1786 			struct bpf_map *map;
1787 			u32 key;
1788 		} tail_call;
1789 	};
1790 	bool tailcall_target_stable;
1791 	u8 adj_off;
1792 	u16 reason;
1793 	u32 insn_idx;
1794 };
1795 
1796 /* reg_type info for ctx arguments */
1797 struct bpf_ctx_arg_aux {
1798 	u32 offset;
1799 	enum bpf_reg_type reg_type;
1800 	struct btf *btf;
1801 	u32 btf_id;
1802 	u32 ref_id;
1803 	u32 mem_size;
1804 	bool refcounted;
1805 };
1806 
1807 struct btf_mod_pair {
1808 	struct btf *btf;
1809 	struct module *module;
1810 };
1811 
1812 struct bpf_kfunc_desc_tab;
1813 
1814 enum bpf_stream_id {
1815 	BPF_STDOUT = 1,
1816 	BPF_STDERR = 2,
1817 };
1818 
1819 struct bpf_stream_elem {
1820 	struct llist_node node;
1821 	int total_len;
1822 	int consumed_len;
1823 	char str[];
1824 };
1825 
1826 enum {
1827 	/* 100k bytes */
1828 	BPF_STREAM_MAX_CAPACITY = 100000ULL,
1829 };
1830 
1831 struct bpf_stream {
1832 	refcount_t refcnt;
1833 	atomic_t capacity;	/* bytes reserved against the stream limit */
1834 	atomic_t readable;	/* published bytes available to readers */
1835 	struct llist_head log;	/* list of in-flight stream elements in LIFO order */
1836 
1837 	struct mutex lock;  /* lock protecting backlog_{head,tail} */
1838 	struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */
1839 	struct llist_node *backlog_tail; /* tail of the list above */
1840 	wait_queue_head_t waitq;
1841 	struct irq_work notify_work;
1842 	bool notify_used;	/* notify_work was queued at least once */
1843 	bool dead;
1844 };
1845 
1846 struct bpf_stream_stage {
1847 	struct llist_head log;
1848 	int len;
1849 };
1850 
1851 enum bpf_sig_verdict {
1852 	BPF_SIG_UNSIGNED = 0,
1853 	BPF_SIG_VERIFIED,
1854 };
1855 
1856 enum bpf_sig_keyring {
1857 	BPF_SIG_KEYRING_NONE = 0,
1858 	BPF_SIG_KEYRING_BUILTIN,
1859 	BPF_SIG_KEYRING_SECONDARY,
1860 	BPF_SIG_KEYRING_PLATFORM,
1861 	BPF_SIG_KEYRING_USER,
1862 	BPF_SIG_KEYRING_BPF,
1863 };
1864 
1865 struct bpf_prog_aux {
1866 	atomic64_t refcnt;
1867 	u32 used_map_cnt;
1868 	u32 used_btf_cnt;
1869 	u32 max_ctx_offset;
1870 	u32 max_pkt_offset;
1871 	u32 max_tp_access;
1872 	u32 stack_depth;
1873 	u32 id;
1874 	u32 func_cnt; /* used by non-func prog as the number of func progs */
1875 	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1876 	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1877 	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
1878 	u32 attach_st_ops_member_off;
1879 	u32 ctx_arg_info_size;
1880 	u32 max_rdonly_access;
1881 	u32 max_rdwr_access;
1882 	/* [min, max): the bytes of a rdwr buffer that may be written */
1883 	u32 min_rdwr_write;
1884 	u32 max_rdwr_write;
1885 	u32 subprog_start;
1886 	struct btf *attach_btf;
1887 	struct bpf_ctx_arg_aux *ctx_arg_info;
1888 	void __percpu *priv_stack_ptr;
1889 	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
1890 	struct bpf_prog *dst_prog;
1891 	struct bpf_trampoline *dst_trampoline;
1892 	enum bpf_prog_type saved_dst_prog_type;
1893 	enum bpf_attach_type saved_dst_attach_type;
1894 	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
1895 	bool dev_bound; /* Program is bound to the netdev. */
1896 	bool offload_requested; /* Program is bound and offloaded to the netdev. */
1897 	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
1898 	bool attach_tracing_prog; /* true if tracing another tracing program */
1899 	bool tramp_linked; /* true if it was ever linked to a trampoline */
1900 	bool sleepable_async_cb; /* has a bpf_wq or bpf_task_work callback */
1901 	bool func_proto_unreliable;
1902 	bool tail_call_reachable;
1903 	bool xdp_has_frags;
1904 	bool exception_cb;
1905 	bool exception_boundary;
1906 	bool jits_use_priv_stack;
1907 	bool priv_stack_requested;
1908 	bool changes_pkt_data;
1909 	bool has_prologue;
1910 	bool might_sleep;
1911 	bool can_unwind;
1912 	bool kprobe_write_ctx;
1913 	/* has insns that work with the arena base, see bpf_prog_has_arena_insns() */
1914 	bool uses_arena_insns;
1915 	struct {
1916 		s32 keyring_serial;
1917 		u8 keyring_type;
1918 		u8 verdict;
1919 	} sig;
1920 	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
1921 	struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */
1922 	struct bpf_arena *arena;
1923 	void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
1924 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1925 	const struct btf_type *attach_func_proto;
1926 	/* function name for valid attach_btf_id */
1927 	const char *attach_func_name;
1928 	struct bpf_prog **func;
1929 	struct bpf_prog_aux *main_prog_aux;
1930 	/* main prog: there when the prog has timed may_goto */
1931 	struct bpf_timed_may_goto __percpu *timed_may_goto;
1932 	void *jit_data; /* JIT specific data. arch dependent */
1933 	struct bpf_jit_poke_descriptor *poke_tab;
1934 	struct bpf_kfunc_desc_tab *kfunc_tab;
1935 	struct bpf_kfunc_btf_tab *kfunc_btf_tab;
1936 	u32 size_poke_tab;
1937 #ifdef CONFIG_FINEIBT
1938 	struct bpf_ksym ksym_prefix;
1939 #endif
1940 	struct bpf_ksym ksym;
1941 	const struct bpf_prog_ops *ops;
1942 	const struct bpf_struct_ops *st_ops;
1943 	struct bpf_map **used_maps;
1944 	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1945 	struct btf_mod_pair *used_btfs;
1946 	struct bpf_prog *prog;
1947 	struct user_struct *user;
1948 	u64 load_time; /* ns since boottime */
1949 	u32 verified_insns;
1950 	int cgroup_atype; /* enum cgroup_bpf_attach_type */
1951 	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1952 	char name[BPF_OBJ_NAME_LEN];
1953 	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
1954 	u16 stack_arg_sp_adjust;
1955 	u16 freplace_link_cnt; /* counts freplace links extending this prog */
1956 #ifdef CONFIG_SECURITY
1957 	void *security;
1958 #endif
1959 	struct bpf_token *token;
1960 	struct bpf_prog_offload *offload;
1961 	struct btf *btf;
1962 	struct bpf_func_info *func_info;
1963 	struct bpf_func_info_aux *func_info_aux;
1964 	/* bpf_line_info loaded from userspace.  linfo->insn_off
1965 	 * has the xlated insn offset.
1966 	 * Both the main and sub prog share the same linfo.
1967 	 * The subprog can access its first linfo by
1968 	 * using the linfo_idx.
1969 	 */
1970 	struct bpf_line_info *linfo;
1971 	/* jited_linfo is the jited addr of the linfo.  It has a
1972 	 * one to one mapping to linfo:
1973 	 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1974 	 * Both the main and sub prog share the same jited_linfo.
1975 	 * The subprog can access its first jited_linfo by
1976 	 * using the linfo_idx.
1977 	 */
1978 	void **jited_linfo;
1979 	u32 func_info_cnt;
1980 	u32 nr_linfo;
1981 	/* subprog can use linfo_idx to access its first linfo and
1982 	 * jited_linfo.
1983 	 * main prog always has linfo_idx == 0
1984 	 */
1985 	u32 linfo_idx;
1986 	struct module *mod;
1987 	u32 num_exentries;
1988 	struct exception_table_entry *extable;
1989 	union {
1990 		struct work_struct work;
1991 		struct rcu_head	rcu;
1992 		struct rcu_work rwork;
1993 	};
1994 	struct bpf_stream *stream[2];
1995 	spinlock_t st_ops_assoc_lock; /* protects st_ops_assoc */
1996 	struct bpf_map __rcu *st_ops_assoc;
1997 };
1998 
1999 #define BPF_NR_CONTEXTS        4       /* normal, softirq, hardirq, NMI */
2000 
2001 struct bpf_prog {
2002 	u16			pages;		/* Number of allocated pages */
2003 	u32			jited:1,	/* Is our filter JIT'ed? */
2004 				jit_requested:1,/* archs need to JIT the prog */
2005 				jit_required:1,	/* program strictly requires JIT compiler */
2006 				gpl_compatible:1, /* Is filter GPL compatible? */
2007 				cb_access:1,	/* Is control block accessed? */
2008 				dst_needed:1,	/* Do we need dst entry? */
2009 				blinding_requested:1, /* needs constant blinding */
2010 				blinded:1,	/* Was blinded */
2011 				is_func:1,	/* program is a bpf function */
2012 				kprobe_override:1, /* Do we override a kprobe? */
2013 				has_callchain_buf:1, /* callchain buffer allocated? */
2014 				enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
2015 				call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
2016 				call_get_func_ip:1, /* Do we call get_func_ip() */
2017 				call_session_cookie:1, /* Do we call bpf_session_cookie() */
2018 				tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
2019 				sleepable:1;	/* BPF program is sleepable */
2020 	enum bpf_prog_type	type;		/* Type of BPF program */
2021 	enum bpf_attach_type	expected_attach_type; /* For some prog types */
2022 	u32			len;		/* Number of filter blocks */
2023 	u32			jited_len;	/* Size of jited insns in bytes */
2024 	union {
2025 		u8 digest[SHA256_DIGEST_SIZE];
2026 		u8 tag[BPF_TAG_SIZE];
2027 	};
2028 	struct bpf_prog_stats __percpu *stats;
2029 	u8 __percpu		*active;	/* u8[BPF_NR_CONTEXTS] for recursion protection */
2030 	unsigned int		(*bpf_func)(const void *ctx,
2031 					    const struct bpf_insn *insn);
2032 	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
2033 	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
2034 	/* Instructions for interpreter */
2035 	union {
2036 		DECLARE_FLEX_ARRAY(struct sock_filter, insns);
2037 		DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
2038 	};
2039 };
2040 
2041 /*
2042  * Does the program, or a callback of it, run under RCU Tasks Trace protection
2043  * only, outside of any RCU read side critical section?
2044  */
2045 static inline bool bpf_prog_runs_sleepable(const struct bpf_prog *prog)
2046 {
2047 	return prog->sleepable || prog->aux->sleepable_async_cb;
2048 }
2049 
2050 struct bpf_array_aux {
2051 	/* Programs with direct jumps into programs part of this array. */
2052 	struct list_head poke_progs;
2053 	struct bpf_map *map;
2054 	struct mutex poke_mutex;
2055 	struct work_struct work;
2056 };
2057 
2058 struct bpf_link {
2059 	atomic64_t refcnt;
2060 	u32 id;
2061 	enum bpf_link_type type;
2062 	const struct bpf_link_ops *ops;
2063 	struct bpf_prog *prog;
2064 
2065 	u32 flags;
2066 	enum bpf_attach_type attach_type;
2067 
2068 	/* rcu is used before freeing, work can be used to schedule that
2069 	 * RCU-based freeing before that, so they never overlap
2070 	 */
2071 	union {
2072 		struct rcu_head rcu;
2073 		struct work_struct work;
2074 	};
2075 	/* whether BPF link itself has "sleepable" semantics, which can differ
2076 	 * from underlying BPF program having a "sleepable" semantics, as BPF
2077 	 * link's semantics is determined by target attach hook
2078 	 */
2079 	bool sleepable;
2080 };
2081 
2082 struct bpf_link_ops {
2083 	void (*release)(struct bpf_link *link);
2084 	/* deallocate link resources callback, called without RCU grace period
2085 	 * waiting
2086 	 */
2087 	void (*dealloc)(struct bpf_link *link);
2088 	/* deallocate link resources callback, called after RCU grace period;
2089 	 * if either the underlying BPF program is sleepable or BPF link's
2090 	 * target hook is sleepable, we'll go through tasks trace RCU GP and
2091 	 * then "classic" RCU GP; this need for chaining tasks trace and
2092 	 * classic RCU GPs is designated by setting bpf_link->sleepable flag
2093 	 *
2094 	 * For non-sleepable tracepoint links we go through SRCU gp instead,
2095 	 * since RCU is not used in that case. Sleepable tracepoints still
2096 	 * follow the scheme above.
2097 	 */
2098 	void (*dealloc_deferred)(struct bpf_link *link);
2099 	int (*detach)(struct bpf_link *link);
2100 	int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
2101 			   struct bpf_prog *old_prog);
2102 	void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
2103 	int (*fill_link_info)(const struct bpf_link *link,
2104 			      struct bpf_link_info *info);
2105 	int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
2106 			  struct bpf_map *old_map);
2107 	__poll_t (*poll)(struct file *file, struct poll_table_struct *pts);
2108 };
2109 
2110 struct bpf_tramp_node {
2111 	struct bpf_link *link;
2112 	struct hlist_node tramp_hlist;
2113 	u64 cookie;
2114 };
2115 
2116 struct bpf_tramp_link {
2117 	struct bpf_link link;
2118 	struct bpf_tramp_node node;
2119 };
2120 
2121 struct bpf_shim_tramp_link {
2122 	struct bpf_tramp_link link;
2123 	struct bpf_trampoline *trampoline;
2124 };
2125 
2126 struct bpf_tracing_link {
2127 	struct bpf_tramp_link link;
2128 	struct bpf_tramp_node fexit;
2129 	struct bpf_trampoline *trampoline;
2130 	struct bpf_prog *tgt_prog;
2131 };
2132 
2133 struct bpf_tracing_multi_node {
2134 	struct bpf_tramp_node node;
2135 	struct bpf_trampoline *trampoline;
2136 	struct ftrace_func_entry entry;
2137 };
2138 
2139 struct bpf_tracing_multi_data {
2140 	struct ftrace_hash *unreg;
2141 	struct ftrace_hash *modify;
2142 	struct ftrace_hash *reg;
2143 	struct ftrace_func_entry *entry;
2144 };
2145 
2146 struct bpf_tracing_multi_link {
2147 	struct bpf_link link;
2148 	struct bpf_tracing_multi_data data;
2149 	u64 *cookies;
2150 	struct bpf_tramp_node *fexits;
2151 	int nodes_cnt;
2152 	struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt);
2153 };
2154 
2155 struct bpf_raw_tp_link {
2156 	struct bpf_link link;
2157 	struct bpf_raw_event_map *btp;
2158 	u64 cookie;
2159 };
2160 
2161 struct bpf_link_primer {
2162 	struct bpf_link *link;
2163 	struct file *file;
2164 	int fd;
2165 	u32 id;
2166 };
2167 
2168 struct bpf_mount_opts {
2169 	kuid_t uid;
2170 	kgid_t gid;
2171 	umode_t mode;
2172 
2173 	/* BPF token-related delegation options */
2174 	u64 delegate_cmds;
2175 	u64 delegate_maps;
2176 	u64 delegate_progs;
2177 	u64 delegate_attachs;
2178 
2179 	struct simple_xattr_cache xa_cache;
2180 };
2181 
2182 struct bpf_token {
2183 	struct work_struct work;
2184 	atomic64_t refcnt;
2185 	struct user_namespace *userns;
2186 	u64 allowed_cmds;
2187 	u64 allowed_maps;
2188 	u64 allowed_progs;
2189 	u64 allowed_attachs;
2190 #ifdef CONFIG_SECURITY
2191 	void *security;
2192 #endif
2193 };
2194 
2195 struct bpf_struct_ops_value;
2196 struct btf_member;
2197 
2198 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
2199 /**
2200  * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
2201  *			   define a BPF_MAP_TYPE_STRUCT_OPS map type composed
2202  *			   of BPF_PROG_TYPE_STRUCT_OPS progs.
2203  * @verifier_ops: A structure of callbacks that are invoked by the verifier
2204  *		  when determining whether the struct_ops progs in the
2205  *		  struct_ops map are valid.
2206  * @init: A callback that is invoked a single time, and before any other
2207  *	  callback, to initialize the structure. A nonzero return value means
2208  *	  the subsystem could not be initialized.
2209  * @check_member: When defined, a callback invoked by the verifier to allow
2210  *		  the subsystem to determine if an entry in the struct_ops map
2211  *		  is valid. A nonzero return value means that the map is
2212  *		  invalid and should be rejected by the verifier.
2213  * @init_member: A callback that is invoked for each member of the struct_ops
2214  *		 map to allow the subsystem to initialize the member. A nonzero
2215  *		 value means the member could not be initialized. This callback
2216  *		 is exclusive with the @type, @type_id, @value_type, and
2217  *		 @value_id fields.
2218  * @reg: A callback that is invoked when the struct_ops map has been
2219  *	 initialized and is being attached to. Zero means the struct_ops map
2220  *	 has been successfully registered and is live. A nonzero return value
2221  *	 means the struct_ops map could not be registered.
2222  * @unreg: A callback that is invoked when the struct_ops map should be
2223  *	   unregistered.
2224  * @update: A callback that is invoked when the live struct_ops map is being
2225  *	    updated to contain new values. This callback is only invoked when
2226  *	    the struct_ops map is loaded with BPF_F_LINK. If not defined, the
2227  *	    it is assumed that the struct_ops map cannot be updated.
2228  * @validate: A callback that is invoked after all of the members have been
2229  *	      initialized. This callback should perform static checks on the
2230  *	      map, meaning that it should either fail or succeed
2231  *	      deterministically. A struct_ops map that has been validated may
2232  *	      not necessarily succeed in being registered if the call to @reg
2233  *	      fails. For example, a valid struct_ops map may be loaded, but
2234  *	      then fail to be registered due to there being another active
2235  *	      struct_ops map on the system in the subsystem already. For this
2236  *	      reason, if this callback is not defined, the check is skipped as
2237  *	      the struct_ops map will have final verification performed in
2238  *	      @reg.
2239  * @cfi_stubs: Pointer to a structure of stub functions for CFI. These stubs
2240  *	       provide the correct Control Flow Integrity hashes for the
2241  *	       trampolines generated by BPF struct_ops.
2242  * @owner: The module that owns this struct_ops. Used for module reference
2243  *	   counting to ensure the module providing the struct_ops cannot be
2244  *	   unloaded while in use.
2245  * @name: The name of the struct bpf_struct_ops object.
2246  * @func_models: Func models
2247  * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment.
2248  *		  0 means the struct_ops type does not support cgroup attachment.
2249  *		  If cgroup_atype is non-zero, the @reg and @unreg must be NULL
2250  *		  because the attachment/detachment will be handled by the bpf core.
2251  * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for
2252  *                           a tasks_rcu gp before freeing the struct_ops map
2253  *                           and its progs. It is unnecessary if the @unreg
2254  *                           has waited for the correct rcu gp or the @unreg
2255  *                           has ensured all struct_ops prog has finished running.
2256  * @free_after_mult_rcu_gp: Set to true if the cgroup prog array of this
2257  *                          struct_ops needs a tasks_trace_rcu gp, besides a
2258  *                          regular rcu gp, before it is freed. It is usually
2259  *                          needed if the struct_ops has sleepable prog. A
2260  *                          struct_ops map always waits for both.
2261  */
2262 struct bpf_struct_ops {
2263 	const struct bpf_verifier_ops *verifier_ops;
2264 	int (*init)(struct btf *btf);
2265 	int (*check_member)(const struct btf_type *t,
2266 			    const struct btf_member *member,
2267 			    const struct bpf_prog *prog);
2268 	int (*init_member)(const struct btf_type *t,
2269 			   const struct btf_member *member,
2270 			   void *kdata, const void *udata);
2271 	int (*reg)(void *kdata, struct bpf_link *link);
2272 	void (*unreg)(void *kdata, struct bpf_link *link);
2273 	int (*update)(void *kdata, void *old_kdata, struct bpf_link *link);
2274 	int (*validate)(void *kdata);
2275 	void *cfi_stubs;
2276 	struct module *owner;
2277 	const char *name;
2278 	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
2279 	int cgroup_atype;
2280 	bool free_after_tasks_rcu_gp;
2281 	bool free_after_mult_rcu_gp;
2282 };
2283 
2284 /* Every member of a struct_ops type has an instance even a member is not
2285  * an operator (function pointer). The "info" field will be assigned to
2286  * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the
2287  * argument information required by the verifier to verify the program.
2288  *
2289  * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the
2290  * corresponding entry for an given argument.
2291  */
2292 struct bpf_struct_ops_arg_info {
2293 	struct bpf_ctx_arg_aux *info;
2294 	u32 cnt;
2295 };
2296 
2297 struct bpf_struct_ops_desc {
2298 	struct bpf_struct_ops *st_ops;
2299 
2300 	const struct btf_type *type;
2301 	const struct btf_type *value_type;
2302 	u32 type_id;
2303 	u32 value_id;
2304 
2305 	/* Collection of argument information for each member */
2306 	struct bpf_struct_ops_arg_info *arg_info;
2307 };
2308 
2309 enum bpf_struct_ops_state {
2310 	BPF_STRUCT_OPS_STATE_INIT,
2311 	BPF_STRUCT_OPS_STATE_INUSE,
2312 	BPF_STRUCT_OPS_STATE_TOBEFREE,
2313 	BPF_STRUCT_OPS_STATE_READY,
2314 };
2315 
2316 struct bpf_struct_ops_common_value {
2317 	refcount_t refcnt;
2318 	enum bpf_struct_ops_state state;
2319 };
2320 
2321 static inline bool bpf_prog_get_recursion_context(struct bpf_prog *prog)
2322 {
2323 #ifdef CONFIG_ARM64
2324 	u8 rctx = interrupt_context_level();
2325 	u8 *active = this_cpu_ptr(prog->active);
2326 	u32 val;
2327 
2328 	preempt_disable();
2329 	active[rctx]++;
2330 	val = le32_to_cpu(*(__le32 *)active);
2331 	preempt_enable();
2332 	if (val != BIT(rctx * 8))
2333 		return false;
2334 
2335 	return true;
2336 #else
2337 	return this_cpu_inc_return(*(int __percpu *)(prog->active)) == 1;
2338 #endif
2339 }
2340 
2341 static inline void bpf_prog_put_recursion_context(struct bpf_prog *prog)
2342 {
2343 #ifdef CONFIG_ARM64
2344 	u8 rctx = interrupt_context_level();
2345 	u8 *active = this_cpu_ptr(prog->active);
2346 
2347 	preempt_disable();
2348 	active[rctx]--;
2349 	preempt_enable();
2350 #else
2351 	this_cpu_dec(*(int __percpu *)(prog->active));
2352 #endif
2353 }
2354 
2355 static inline bool is_tracing_multi(enum bpf_attach_type type)
2356 {
2357 	return type == BPF_TRACE_FENTRY_MULTI || type == BPF_TRACE_FEXIT_MULTI ||
2358 	       type == BPF_TRACE_FSESSION_MULTI;
2359 }
2360 
2361 static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes)
2362 {
2363 	return fentry_nodes->nr_nodes == 1 &&
2364 	       fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS;
2365 }
2366 
2367 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
2368 /* This macro helps developer to register a struct_ops type and generate
2369  * type information correctly. Developers should use this macro to register
2370  * a struct_ops type instead of calling __register_bpf_struct_ops() directly.
2371  */
2372 #define register_bpf_struct_ops(st_ops, type)				\
2373 	({								\
2374 		struct bpf_struct_ops_##type {				\
2375 			struct bpf_struct_ops_common_value common;	\
2376 			struct type data ____cacheline_aligned_in_smp;	\
2377 		};							\
2378 		BTF_TYPE_EMIT(struct bpf_struct_ops_##type);		\
2379 		__register_bpf_struct_ops(st_ops);			\
2380 	})
2381 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
2382 bool bpf_struct_ops_get(const void *kdata);
2383 void bpf_struct_ops_put(const void *kdata);
2384 int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff);
2385 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
2386 				       void *value);
2387 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_nodes *tnodes,
2388 				      struct bpf_tramp_node *node,
2389 				      const struct btf_func_model *model,
2390 				      void *stub_func,
2391 				      void **image, u32 *image_off,
2392 				      bool allow_alloc);
2393 void bpf_struct_ops_image_free(void *image);
2394 static inline bool bpf_try_module_get(const void *data, struct module *owner)
2395 {
2396 	if (owner == BPF_MODULE_OWNER)
2397 		return bpf_struct_ops_get(data);
2398 	else
2399 		return try_module_get(owner);
2400 }
2401 static inline void bpf_module_put(const void *data, struct module *owner)
2402 {
2403 	if (owner == BPF_MODULE_OWNER)
2404 		bpf_struct_ops_put(data);
2405 	else
2406 		module_put(owner);
2407 }
2408 int bpf_struct_ops_link_create(union bpf_attr *attr);
2409 int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map);
2410 void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog);
2411 void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux);
2412 u32 bpf_struct_ops_id(const void *kdata);
2413 int bpf_struct_ops_for_each_prog(const void *kdata,
2414 				 int (*cb)(struct bpf_prog *prog, void *data),
2415 				 void *data);
2416 void *bpf_struct_ops_map_kdata(struct bpf_map *map);
2417 void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map);
2418 bool bpf_struct_ops_valid_to_reg(struct bpf_map *map);
2419 int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map,
2420 				     struct bpf_map *expected_old_map);
2421 int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map);
2422 
2423 #ifdef CONFIG_NET
2424 /* Define it here to avoid the use of forward declaration */
2425 struct bpf_dummy_ops_state {
2426 	int val;
2427 };
2428 
2429 struct bpf_dummy_ops {
2430 	int (*test_1)(struct bpf_dummy_ops_state *cb);
2431 	int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
2432 		      char a3, unsigned long a4);
2433 	int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
2434 };
2435 
2436 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
2437 			    union bpf_attr __user *uattr);
2438 #endif
2439 int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
2440 			     struct btf *btf,
2441 			     struct bpf_verifier_log *log);
2442 void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);
2443 void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);
2444 #else
2445 #define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })
2446 static inline bool bpf_try_module_get(const void *data, struct module *owner)
2447 {
2448 	return try_module_get(owner);
2449 }
2450 static inline void bpf_module_put(const void *data, struct module *owner)
2451 {
2452 	module_put(owner);
2453 }
2454 static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
2455 {
2456 	return -ENOTSUPP;
2457 }
2458 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
2459 						     void *key,
2460 						     void *value)
2461 {
2462 	return -EINVAL;
2463 }
2464 static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
2465 {
2466 	return -EOPNOTSUPP;
2467 }
2468 static inline int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map)
2469 {
2470 	return -EOPNOTSUPP;
2471 }
2472 static inline void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog)
2473 {
2474 }
2475 static inline void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux)
2476 {
2477 	return NULL;
2478 }
2479 static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
2480 {
2481 }
2482 
2483 static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
2484 {
2485 }
2486 static inline u32 bpf_struct_ops_id(void *kdata)
2487 {
2488 	return 0;
2489 }
2490 static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map)
2491 {
2492 	return NULL;
2493 }
2494 static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map)
2495 {
2496 	return 0;
2497 }
2498 static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map)
2499 {
2500 	return NULL;
2501 }
2502 static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
2503 {
2504 	return false;
2505 }
2506 static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map,
2507 						   struct bpf_map *old_map,
2508 						   struct bpf_map *expected_old_map)
2509 {
2510 	return -EOPNOTSUPP;
2511 }
2512 
2513 #endif
2514 
2515 static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes)
2516 {
2517 	struct bpf_tramp_nodes fentries = nodes[BPF_TRAMP_FENTRY];
2518 	int cnt = 0;
2519 
2520 	for (int i = 0; i < nodes[BPF_TRAMP_FENTRY].nr_nodes; i++) {
2521 		if (fentries.nodes[i]->link->prog->expected_attach_type == BPF_TRACE_FSESSION)
2522 			cnt++;
2523 		if (fentries.nodes[i]->link->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI)
2524 			cnt++;
2525 	}
2526 
2527 	return cnt;
2528 }
2529 
2530 static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_node *node)
2531 {
2532 	return node->link->prog->call_session_cookie;
2533 }
2534 
2535 static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_nodes *nodes)
2536 {
2537 	struct bpf_tramp_nodes fentries = nodes[BPF_TRAMP_FENTRY];
2538 	int cnt = 0;
2539 
2540 	for (int i = 0; i < nodes[BPF_TRAMP_FENTRY].nr_nodes; i++) {
2541 		if (bpf_prog_calls_session_cookie(fentries.nodes[i]))
2542 			cnt++;
2543 	}
2544 
2545 	return cnt;
2546 }
2547 
2548 int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
2549 			       const struct bpf_ctx_arg_aux *info, u32 cnt);
2550 
2551 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
2552 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
2553 				    int cgroup_atype,
2554 				    enum bpf_attach_type attach_type);
2555 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
2556 #else
2557 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
2558 						  int cgroup_atype,
2559 						  enum bpf_attach_type attach_type)
2560 {
2561 	return -EOPNOTSUPP;
2562 }
2563 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
2564 {
2565 }
2566 #endif
2567 
2568 struct bpf_array {
2569 	struct bpf_map map;
2570 	u32 elem_size;
2571 	u32 index_mask;
2572 	struct bpf_array_aux *aux;
2573 	union {
2574 		DECLARE_FLEX_ARRAY(char, value) __aligned(8);
2575 		DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
2576 		DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
2577 	};
2578 };
2579 
2580 /*
2581  * The bpf_array_get_next_key() function may be used for all array-like
2582  * maps, i.e., maps with u32 keys with range [0 ,..., max_entries)
2583  */
2584 int bpf_array_get_next_key(struct bpf_map *map, void *key, void *next_key);
2585 
2586 #define BPF_COMPLEXITY_LIMIT_INSNS      1000000 /* yes. 1M insns */
2587 #define MAX_TAIL_CALL_CNT 33
2588 
2589 /* Maximum number of loops for bpf_loop and bpf_iter_num.
2590  * It's enum to expose it (and thus make it discoverable) through BTF.
2591  */
2592 enum {
2593 	BPF_MAX_LOOPS = 8 * 1024 * 1024,
2594 	BPF_MAX_TIMED_LOOPS = 0xffff,
2595 };
2596 
2597 #define BPF_F_ACCESS_MASK	(BPF_F_RDONLY |		\
2598 				 BPF_F_RDONLY_PROG |	\
2599 				 BPF_F_WRONLY |		\
2600 				 BPF_F_WRONLY_PROG)
2601 
2602 #define BPF_MAP_CAN_READ	BIT(0)
2603 #define BPF_MAP_CAN_WRITE	BIT(1)
2604 
2605 /* Maximum number of user-producer ring buffer samples that can be drained in
2606  * a call to bpf_user_ringbuf_drain().
2607  */
2608 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
2609 
2610 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
2611 {
2612 	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
2613 
2614 	/* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
2615 	 * not possible.
2616 	 */
2617 	if (access_flags & BPF_F_RDONLY_PROG)
2618 		return BPF_MAP_CAN_READ;
2619 	else if (access_flags & BPF_F_WRONLY_PROG)
2620 		return BPF_MAP_CAN_WRITE;
2621 	else
2622 		return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
2623 }
2624 
2625 static inline bool bpf_map_flags_access_ok(u32 access_flags)
2626 {
2627 	return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
2628 	       (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
2629 }
2630 
2631 static inline struct bpf_map_owner *bpf_map_owner_alloc(struct bpf_map *map)
2632 {
2633 	return kzalloc_obj(*map->owner, GFP_ATOMIC);
2634 }
2635 
2636 static inline void bpf_map_owner_free(struct bpf_map *map)
2637 {
2638 	kfree(map->owner);
2639 }
2640 
2641 struct bpf_event_entry {
2642 	struct perf_event *event;
2643 	struct file *perf_file;
2644 	struct file *map_file;
2645 	struct rcu_head rcu;
2646 };
2647 
2648 static inline bool map_type_contains_progs(struct bpf_map *map)
2649 {
2650 	return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
2651 	       map->map_type == BPF_MAP_TYPE_DEVMAP ||
2652 	       map->map_type == BPF_MAP_TYPE_CPUMAP;
2653 }
2654 
2655 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);
2656 int bpf_prog_calc_tag(struct bpf_prog *fp);
2657 
2658 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
2659 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
2660 
2661 const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void);
2662 
2663 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
2664 					unsigned long off, unsigned long len);
2665 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
2666 					const struct bpf_insn *src,
2667 					struct bpf_insn *dst,
2668 					struct bpf_prog *prog,
2669 					u32 *target_size);
2670 
2671 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
2672 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
2673 
2674 /* an array of programs to be executed under rcu_lock.
2675  *
2676  * Typical usage:
2677  * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
2678  *
2679  * the structure returned by bpf_prog_array_alloc() should be populated
2680  * with program pointers and the last pointer must be NULL.
2681  * The user has to keep refcnt on the program and make sure the program
2682  * is removed from the array before bpf_prog_put().
2683  * The 'struct bpf_prog_array *' should only be replaced with xchg()
2684  * since other cpus are walking the array of pointers in parallel.
2685  */
2686 struct bpf_prog_array_item {
2687 	union {
2688 		struct bpf_prog *prog;
2689 		void *kdata;
2690 	};
2691 	union {
2692 		struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
2693 		u64 bpf_cookie;
2694 	};
2695 };
2696 
2697 struct bpf_prog_array {
2698 	struct rcu_head rcu;
2699 	struct bpf_prog_array_item items[];
2700 };
2701 
2702 /* to avoid allocating empty bpf_prog_array for cgroups that
2703  * don't have bpf program attached use one global 'bpf_empty_prog_array'
2704  * It will not be modified the caller of bpf_prog_array_alloc()
2705  * (since caller requested prog_cnt == 0)
2706  * that pointer should be 'freed' by bpf_prog_array_free()
2707  */
2708 extern struct bpf_prog_array bpf_empty_prog_array;
2709 
2710 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
2711 void bpf_prog_array_free(struct bpf_prog_array *progs);
2712 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
2713 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
2714 int bpf_prog_array_length(struct bpf_prog_array *progs);
2715 bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
2716 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
2717 				__u32 __user *prog_ids, u32 cnt);
2718 
2719 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
2720 				struct bpf_prog *old_prog);
2721 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
2722 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
2723 			     struct bpf_prog *prog);
2724 int bpf_prog_array_copy_info(struct bpf_prog_array *array,
2725 			     u32 *prog_ids, u32 request_cnt,
2726 			     u32 *prog_cnt);
2727 int bpf_prog_array_copy(struct bpf_prog_array *old_array,
2728 			struct bpf_prog *exclude_prog,
2729 			struct bpf_prog *include_prog,
2730 			u64 bpf_cookie,
2731 			struct bpf_prog_array **new_array);
2732 struct bpf_prog *bpf_prog_dummy(void);
2733 
2734 struct bpf_run_ctx {};
2735 
2736 struct bpf_cg_run_ctx {
2737 	struct bpf_run_ctx run_ctx;
2738 	const struct bpf_prog_array_item *prog_item;
2739 	int retval;
2740 };
2741 
2742 struct bpf_trace_run_ctx {
2743 	struct bpf_run_ctx run_ctx;
2744 	u64 bpf_cookie;
2745 	bool is_uprobe;
2746 };
2747 
2748 struct bpf_tramp_run_ctx {
2749 	struct bpf_run_ctx run_ctx;
2750 	u64 bpf_cookie;
2751 	int retval;
2752 	struct bpf_run_ctx *saved_run_ctx;
2753 };
2754 
2755 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
2756 {
2757 	struct bpf_run_ctx *old_ctx = NULL;
2758 
2759 #ifdef CONFIG_BPF_SYSCALL
2760 	old_ctx = current->bpf_ctx;
2761 	current->bpf_ctx = new_ctx;
2762 #endif
2763 	return old_ctx;
2764 }
2765 
2766 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
2767 {
2768 #ifdef CONFIG_BPF_SYSCALL
2769 	current->bpf_ctx = old_ctx;
2770 #endif
2771 }
2772 
2773 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
2774 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE			(1 << 0)
2775 /* BPF program asks to set CN on the packet. */
2776 #define BPF_RET_SET_CN						(1 << 0)
2777 
2778 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);
2779 
2780 static __always_inline u32
2781 bpf_prog_run_array(const struct bpf_prog_array *array,
2782 		   const void *ctx, bpf_prog_run_fn run_prog)
2783 {
2784 	const struct bpf_prog_array_item *item;
2785 	const struct bpf_prog *prog;
2786 	struct bpf_run_ctx *old_run_ctx;
2787 	struct bpf_trace_run_ctx run_ctx;
2788 	u32 ret = 1;
2789 
2790 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
2791 
2792 	if (unlikely(!array))
2793 		return ret;
2794 
2795 	run_ctx.is_uprobe = false;
2796 
2797 	migrate_disable();
2798 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2799 	item = &array->items[0];
2800 	while ((prog = READ_ONCE(item->prog))) {
2801 		run_ctx.bpf_cookie = item->bpf_cookie;
2802 		ret &= run_prog(prog, ctx);
2803 		item++;
2804 	}
2805 	bpf_reset_run_ctx(old_run_ctx);
2806 	migrate_enable();
2807 	return ret;
2808 }
2809 
2810 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
2811  *
2812  * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
2813  * overall. As a result, we must use the bpf_prog_array_free_sleepable
2814  * in order to use the tasks_trace rcu grace period.
2815  *
2816  * When a non-sleepable program is inside the array, we take the rcu read
2817  * section and disable preemption for that program alone, so it can access
2818  * rcu-protected dynamically sized maps.
2819  */
2820 static __always_inline u32
2821 bpf_prog_run_array_uprobe(const struct bpf_prog_array *array,
2822 			  const void *ctx, bpf_prog_run_fn run_prog)
2823 {
2824 	const struct bpf_prog_array_item *item;
2825 	const struct bpf_prog *prog;
2826 	struct bpf_run_ctx *old_run_ctx;
2827 	struct bpf_trace_run_ctx run_ctx;
2828 	u32 ret = 1;
2829 
2830 	might_fault();
2831 	RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held");
2832 
2833 	if (unlikely(!array))
2834 		return ret;
2835 
2836 	migrate_disable();
2837 
2838 	run_ctx.is_uprobe = true;
2839 
2840 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2841 	item = &array->items[0];
2842 	while ((prog = READ_ONCE(item->prog))) {
2843 		if (!prog->sleepable)
2844 			rcu_read_lock();
2845 
2846 		run_ctx.bpf_cookie = item->bpf_cookie;
2847 		ret &= run_prog(prog, ctx);
2848 		item++;
2849 
2850 		if (!prog->sleepable)
2851 			rcu_read_unlock();
2852 	}
2853 	bpf_reset_run_ctx(old_run_ctx);
2854 	migrate_enable();
2855 	return ret;
2856 }
2857 
2858 bool bpf_jit_bypass_spec_v1(void);
2859 bool bpf_jit_bypass_spec_v4(void);
2860 
2861 #define bpf_rcu_lock_held() \
2862 	(rcu_read_lock_held() || rcu_read_lock_trace_held() || rcu_read_lock_bh_held())
2863 
2864 #ifdef CONFIG_BPF_SYSCALL
2865 DECLARE_PER_CPU(int, bpf_prog_active);
2866 extern struct mutex bpf_stats_enabled_mutex;
2867 
2868 /*
2869  * Block execution of BPF programs attached to instrumentation (perf,
2870  * kprobes, tracepoints) to prevent deadlocks on map operations as any of
2871  * these events can happen inside a region which holds a map bucket lock
2872  * and can deadlock on it.
2873  */
2874 static inline void bpf_disable_instrumentation(void)
2875 {
2876 	migrate_disable();
2877 	this_cpu_inc(bpf_prog_active);
2878 }
2879 
2880 static inline void bpf_enable_instrumentation(void)
2881 {
2882 	this_cpu_dec(bpf_prog_active);
2883 	migrate_enable();
2884 }
2885 
2886 extern const struct super_operations bpf_super_ops;
2887 extern const struct file_operations bpf_map_fops;
2888 extern const struct file_operations bpf_prog_fops;
2889 extern const struct file_operations bpf_iter_fops;
2890 extern const struct file_operations bpf_token_fops;
2891 
2892 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2893 	extern const struct bpf_prog_ops _name ## _prog_ops; \
2894 	extern const struct bpf_verifier_ops _name ## _verifier_ops;
2895 #define BPF_MAP_TYPE(_id, _ops) \
2896 	extern const struct bpf_map_ops _ops;
2897 #define BPF_LINK_TYPE(_id, _name)
2898 #include <linux/bpf_types.h>
2899 #undef BPF_PROG_TYPE
2900 #undef BPF_MAP_TYPE
2901 #undef BPF_LINK_TYPE
2902 
2903 extern const struct bpf_prog_ops bpf_offload_prog_ops;
2904 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
2905 extern const struct bpf_verifier_ops xdp_analyzer_ops;
2906 
2907 struct bpf_prog *bpf_prog_get(u32 ufd);
2908 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2909 				       bool attach_drv);
2910 void bpf_prog_add(struct bpf_prog *prog, int i);
2911 void bpf_prog_sub(struct bpf_prog *prog, int i);
2912 void bpf_prog_inc(struct bpf_prog *prog);
2913 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
2914 void bpf_prog_put(struct bpf_prog *prog);
2915 
2916 void bpf_prog_free_id(struct bpf_prog *prog);
2917 void bpf_map_free_id(struct bpf_map *map);
2918 
2919 struct btf_field *btf_record_find(const struct btf_record *rec,
2920 				  u32 offset, u32 field_mask);
2921 void btf_record_free(struct btf_record *rec);
2922 void bpf_map_free_record(struct bpf_map *map);
2923 struct btf_record *btf_record_dup(const struct btf_record *rec);
2924 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
2925 void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
2926 void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
2927 void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);
2928 void bpf_obj_cancel_fields(struct bpf_map *map, void *obj);
2929 void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
2930 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
2931 
2932 struct bpf_map *bpf_map_get(u32 ufd);
2933 struct bpf_map *bpf_map_get_with_uref(u32 ufd);
2934 
2935 /*
2936  * The __bpf_map_get() and __btf_get_by_fd() functions parse a file
2937  * descriptor and return a corresponding map or btf object.
2938  * Their names are double underscored to emphasize the fact that they
2939  * do not increase refcnt. To also increase refcnt use corresponding
2940  * bpf_map_get() and btf_get_by_fd() functions.
2941  */
2942 
2943 static inline struct bpf_map *__bpf_map_get(struct fd f)
2944 {
2945 	if (fd_empty(f))
2946 		return ERR_PTR(-EBADF);
2947 	if (unlikely(fd_file(f)->f_op != &bpf_map_fops))
2948 		return ERR_PTR(-EINVAL);
2949 	return fd_file(f)->private_data;
2950 }
2951 
2952 static inline struct btf *__btf_get_by_fd(struct fd f)
2953 {
2954 	if (fd_empty(f))
2955 		return ERR_PTR(-EBADF);
2956 	if (unlikely(fd_file(f)->f_op != &btf_fops))
2957 		return ERR_PTR(-EINVAL);
2958 	return fd_file(f)->private_data;
2959 }
2960 
2961 void bpf_map_inc(struct bpf_map *map);
2962 void bpf_map_inc_with_uref(struct bpf_map *map);
2963 struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
2964 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
2965 void bpf_map_put_with_uref(struct bpf_map *map);
2966 void bpf_map_put(struct bpf_map *map);
2967 void *bpf_map_area_alloc(u64 size, int numa_node);
2968 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
2969 void bpf_map_area_free(void *base);
2970 bool bpf_map_write_active(const struct bpf_map *map);
2971 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
2972 int  generic_map_lookup_batch(struct bpf_map *map,
2973 			      const union bpf_attr *attr,
2974 			      union bpf_attr __user *uattr);
2975 int  generic_map_update_batch(struct bpf_map *map, struct file *map_file,
2976 			      const union bpf_attr *attr,
2977 			      union bpf_attr __user *uattr);
2978 int  generic_map_delete_batch(struct bpf_map *map,
2979 			      const union bpf_attr *attr,
2980 			      union bpf_attr __user *uattr);
2981 struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
2982 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
2983 
2984 
2985 struct page *bpf_alloc_page(int nid, bool sleepable);
2986 int bpf_alloc_pages(int nid, unsigned long nr_pages,
2987 		    struct llist_head *pages, bool sleepable);
2988 void bpf_free_pages(struct llist_head *pages);
2989 #ifdef CONFIG_MEMCG
2990 void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg,
2991 			 struct mem_cgroup **new_memcg);
2992 void bpf_map_memcg_exit(struct mem_cgroup *old_memcg,
2993 			struct mem_cgroup *memcg);
2994 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2995 			   int node);
2996 void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
2997 			     int node);
2998 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
2999 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
3000 		       gfp_t flags);
3001 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
3002 				    size_t align, gfp_t flags);
3003 #else
3004 /*
3005  * These specialized allocators have to be macros for their allocations to be
3006  * accounted separately (to have separate alloc_tag).
3007  */
3008 #define bpf_map_kmalloc_node(_map, _size, _flags, _node)	\
3009 		kmalloc_node(_size, _flags, _node)
3010 #define bpf_map_kmalloc_nolock(_map, _size, _flags, _node)	\
3011 		kmalloc_nolock(_size, _flags, _node)
3012 #define bpf_map_kzalloc(_map, _size, _flags)			\
3013 		kzalloc(_size, _flags)
3014 #define bpf_map_kvcalloc(_map, _n, _size, _flags)		\
3015 		kvcalloc(_n, _size, _flags)
3016 #define bpf_map_alloc_percpu(_map, _size, _align, _flags)	\
3017 		__alloc_percpu_gfp(_size, _align, _flags)
3018 static inline void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg,
3019 				       struct mem_cgroup **new_memcg)
3020 {
3021 	*new_memcg = NULL;
3022 	*old_memcg = NULL;
3023 }
3024 
3025 static inline void bpf_map_memcg_exit(struct mem_cgroup *old_memcg,
3026 				      struct mem_cgroup *memcg)
3027 {
3028 }
3029 #endif
3030 
3031 static inline int
3032 bpf_map_init_elem_count(struct bpf_map *map)
3033 {
3034 	size_t size = sizeof(*map->elem_count), align = size;
3035 	gfp_t flags = GFP_USER | __GFP_NOWARN;
3036 
3037 	map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
3038 	if (!map->elem_count)
3039 		return -ENOMEM;
3040 
3041 	return 0;
3042 }
3043 
3044 static inline void
3045 bpf_map_free_elem_count(struct bpf_map *map)
3046 {
3047 	free_percpu(map->elem_count);
3048 }
3049 
3050 static inline void bpf_map_inc_elem_count(struct bpf_map *map)
3051 {
3052 	this_cpu_inc(*map->elem_count);
3053 }
3054 
3055 static inline void bpf_map_dec_elem_count(struct bpf_map *map)
3056 {
3057 	this_cpu_dec(*map->elem_count);
3058 }
3059 
3060 s64 bpf_map_sum_elem_count(const struct bpf_map *map);
3061 
3062 extern int sysctl_unprivileged_bpf_disabled;
3063 
3064 bool bpf_token_capable(const struct bpf_token *token, int cap);
3065 
3066 static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)
3067 {
3068 	return bpf_token_capable(token, CAP_PERFMON);
3069 }
3070 
3071 static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
3072 {
3073 	return bpf_token_capable(token, CAP_PERFMON);
3074 }
3075 
3076 static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
3077 {
3078 	return bpf_jit_bypass_spec_v1() ||
3079 		cpu_mitigations_off() ||
3080 		bpf_token_capable(token, CAP_PERFMON);
3081 }
3082 
3083 static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
3084 {
3085 	return bpf_jit_bypass_spec_v4() ||
3086 		cpu_mitigations_off() ||
3087 		bpf_token_capable(token, CAP_PERFMON);
3088 }
3089 
3090 int bpf_map_new_fd(struct bpf_map *map, int flags);
3091 int bpf_prog_new_fd(struct bpf_prog *prog);
3092 
3093 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
3094 		   const struct bpf_link_ops *ops, struct bpf_prog *prog,
3095 		   enum bpf_attach_type attach_type);
3096 void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
3097 			     const struct bpf_link_ops *ops, struct bpf_prog *prog,
3098 			     enum bpf_attach_type attach_type, bool sleepable);
3099 void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type,
3100 			 const struct bpf_link_ops *ops, struct bpf_prog *prog,
3101 			 enum bpf_attach_type attach_type, u64 cookie);
3102 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
3103 int bpf_link_settle(struct bpf_link_primer *primer);
3104 void bpf_link_cleanup(struct bpf_link_primer *primer);
3105 void bpf_link_inc(struct bpf_link *link);
3106 struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link);
3107 void bpf_link_put(struct bpf_link *link);
3108 int bpf_link_new_fd(struct bpf_link *link);
3109 struct bpf_link *bpf_link_get_from_fd(u32 ufd);
3110 struct bpf_link *bpf_link_get_curr_or_next(u32 *id);
3111 
3112 void bpf_token_inc(struct bpf_token *token);
3113 void bpf_token_put(struct bpf_token *token);
3114 int bpf_token_create(union bpf_attr *attr);
3115 struct bpf_token *bpf_token_get_from_fd(u32 ufd);
3116 int bpf_token_get_info_by_fd(struct bpf_token *token,
3117 			     const union bpf_attr *attr,
3118 			     union bpf_attr __user *uattr);
3119 
3120 bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
3121 bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
3122 bool bpf_token_allow_prog_type(const struct bpf_token *token,
3123 			       enum bpf_prog_type prog_type,
3124 			       enum bpf_attach_type attach_type);
3125 
3126 int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
3127 int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
3128 struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,
3129 			    umode_t mode);
3130 
3131 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
3132 #define DEFINE_BPF_ITER_FUNC(target, args...)			\
3133 	extern int bpf_iter_ ## target(args);			\
3134 	int __init bpf_iter_ ## target(args) { return 0; }
3135 
3136 /*
3137  * The task type of iterators.
3138  *
3139  * For BPF task iterators, they can be parameterized with various
3140  * parameters to visit only some of tasks.
3141  *
3142  * BPF_TASK_ITER_ALL (default)
3143  *	Iterate over resources of every task.
3144  *
3145  * BPF_TASK_ITER_TID
3146  *	Iterate over resources of a task/tid.
3147  *
3148  * BPF_TASK_ITER_TGID
3149  *	Iterate over resources of every task of a process / task group.
3150  */
3151 enum bpf_iter_task_type {
3152 	BPF_TASK_ITER_ALL = 0,
3153 	BPF_TASK_ITER_TID,
3154 	BPF_TASK_ITER_TGID,
3155 };
3156 
3157 struct bpf_iter_aux_info {
3158 	/* for map_elem iter */
3159 	struct bpf_map *map;
3160 
3161 	/* for cgroup iter */
3162 	struct {
3163 		struct cgroup *start; /* starting cgroup */
3164 		enum bpf_cgroup_iter_order order;
3165 	} cgroup;
3166 	struct {
3167 		enum bpf_iter_task_type	type;
3168 		u32 pid;
3169 	} task;
3170 };
3171 
3172 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
3173 					union bpf_iter_link_info *linfo,
3174 					struct bpf_iter_aux_info *aux);
3175 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
3176 typedef int (*bpf_iter_check_prog_t)(const struct bpf_prog *prog,
3177 				     const struct bpf_iter_aux_info *aux);
3178 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
3179 					struct seq_file *seq);
3180 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
3181 					 struct bpf_link_info *info);
3182 typedef const struct bpf_func_proto *
3183 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
3184 			     const struct bpf_prog *prog);
3185 
3186 enum bpf_iter_feature {
3187 	BPF_ITER_RESCHED	= BIT(0),	/* may cond_resched() between objects */
3188 	BPF_ITER_NOSLEEP	= BIT(1),	/* ->show() can't run sleepable progs */
3189 };
3190 
3191 #define BPF_ITER_CTX_ARG_MAX 2
3192 struct bpf_iter_reg {
3193 	const char *target;
3194 	bpf_iter_attach_target_t attach_target;
3195 	bpf_iter_detach_target_t detach_target;
3196 	/* may the program run on what the link is attached to? */
3197 	bpf_iter_check_prog_t check_prog;
3198 	bpf_iter_show_fdinfo_t show_fdinfo;
3199 	bpf_iter_fill_link_info_t fill_link_info;
3200 	bpf_iter_get_func_proto_t get_func_proto;
3201 	u32 ctx_arg_info_size;
3202 	u32 feature;
3203 	struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
3204 	const struct bpf_iter_seq_info *seq_info;
3205 };
3206 
3207 struct bpf_iter_meta {
3208 	__bpf_md_ptr(struct seq_file *, seq);
3209 	u64 session_id;
3210 	u64 seq_num;
3211 };
3212 
3213 struct bpf_iter__bpf_map_elem {
3214 	__bpf_md_ptr(struct bpf_iter_meta *, meta);
3215 	__bpf_md_ptr(struct bpf_map *, map);
3216 	__bpf_md_ptr(void *, key);
3217 	__bpf_md_ptr(void *, value);
3218 };
3219 
3220 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
3221 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
3222 int bpf_iter_prog_supported(struct bpf_prog *prog);
3223 const struct bpf_func_proto *
3224 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
3225 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
3226 int bpf_iter_new_fd(struct bpf_link *link);
3227 bool bpf_link_is_iter(struct bpf_link *link);
3228 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
3229 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
3230 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
3231 			      struct seq_file *seq);
3232 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
3233 				struct bpf_link_info *info);
3234 int bpf_iter_map_check_value_write(const struct bpf_prog *prog,
3235 				   const struct bpf_map *map);
3236 
3237 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
3238 				   struct bpf_func_state *caller,
3239 				   struct bpf_func_state *callee);
3240 
3241 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 flags);
3242 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value, u64 flags);
3243 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
3244 			   u64 flags);
3245 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
3246 			    u64 flags);
3247 
3248 int bpf_stackmap_extract(struct bpf_map *map, void *key, void *value, bool delete);
3249 
3250 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
3251 				 void *key, void *value, u64 map_flags);
3252 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
3253 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
3254 				void *key, void *value, u64 map_flags);
3255 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
3256 
3257 int bpf_get_file_flag(int flags);
3258 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
3259 			     size_t actual_size);
3260 
3261 /* verify correctness of eBPF program */
3262 struct bpf_log_attr;
3263 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr,
3264 	      struct bpf_log_attr *attr_log);
3265 
3266 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
3267 int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
3268 s32 bpf_call_args_imm(s16 idx);
3269 #else
3270 static inline s32 bpf_call_args_imm(s16 idx)
3271 {
3272 	return 0;
3273 }
3274 #endif
3275 
3276 struct btf *bpf_get_btf_vmlinux(void);
3277 
3278 /* Map specifics */
3279 struct xdp_frame;
3280 struct sk_buff;
3281 struct bpf_dtab_netdev;
3282 struct bpf_cpu_map_entry;
3283 
3284 void __dev_flush(struct list_head *flush_list);
3285 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
3286 		    struct net_device *dev_rx);
3287 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
3288 		    struct net_device *dev_rx);
3289 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
3290 			  struct bpf_map *map, bool exclude_ingress);
3291 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
3292 			     const struct bpf_prog *xdp_prog);
3293 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
3294 			   const struct bpf_prog *xdp_prog,
3295 			   struct bpf_map *map, bool exclude_ingress);
3296 
3297 void __cpu_map_flush(struct list_head *flush_list);
3298 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
3299 		    struct net_device *dev_rx);
3300 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
3301 			     struct sk_buff *skb);
3302 
3303 /* Return map's numa specified by userspace */
3304 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
3305 {
3306 	return (attr->map_flags & BPF_F_NUMA_NODE) ?
3307 		attr->numa_node : NUMA_NO_NODE;
3308 }
3309 
3310 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
3311 int array_map_alloc_check(union bpf_attr *attr);
3312 
3313 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
3314 			  union bpf_attr __user *uattr);
3315 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
3316 			  union bpf_attr __user *uattr);
3317 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
3318 			      const union bpf_attr *kattr,
3319 			      union bpf_attr __user *uattr);
3320 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
3321 				     const union bpf_attr *kattr,
3322 				     union bpf_attr __user *uattr);
3323 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
3324 			     const union bpf_attr *kattr,
3325 			     union bpf_attr __user *uattr);
3326 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
3327 				const union bpf_attr *kattr,
3328 				union bpf_attr __user *uattr);
3329 int bpf_prog_test_run_nf(struct bpf_prog *prog,
3330 			 const union bpf_attr *kattr,
3331 			 union bpf_attr __user *uattr);
3332 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
3333 		    const struct bpf_prog *prog,
3334 		    struct bpf_insn_access_aux *info);
3335 
3336 static inline bool bpf_tracing_ctx_access(int off, int size,
3337 					  enum bpf_access_type type)
3338 {
3339 	if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
3340 		return false;
3341 	if (type != BPF_READ)
3342 		return false;
3343 	if (off % size != 0)
3344 		return false;
3345 	return true;
3346 }
3347 
3348 static inline bool bpf_tracing_btf_ctx_access(int off, int size,
3349 					      enum bpf_access_type type,
3350 					      const struct bpf_prog *prog,
3351 					      struct bpf_insn_access_aux *info)
3352 {
3353 	if (!bpf_tracing_ctx_access(off, size, type))
3354 		return false;
3355 	return btf_ctx_access(off, size, type, prog, info);
3356 }
3357 
3358 int btf_struct_access(struct bpf_verifier_log *log,
3359 		      const struct bpf_reg_state *reg,
3360 		      int off, int size, enum bpf_access_type atype,
3361 		      u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
3362 bool btf_struct_ids_match(struct bpf_verifier_log *log,
3363 			  const struct btf *btf, u32 id, int off,
3364 			  const struct btf *need_btf, u32 need_type_id,
3365 			  bool strict, bool walk_flex_arrays);
3366 
3367 int btf_distill_func_proto(struct bpf_verifier_log *log,
3368 			   struct btf *btf,
3369 			   const struct btf_type *func_proto,
3370 			   const char *func_name,
3371 			   struct btf_func_model *m);
3372 
3373 struct bpf_reg_state;
3374 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
3375 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
3376 			 struct btf *btf, const struct btf_type *t);
3377 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
3378 				    int comp_idx, const char *tag_key);
3379 int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,
3380 			   int comp_idx, const char *tag_key, int last_id);
3381 
3382 struct bpf_prog *bpf_prog_by_id(u32 id);
3383 struct bpf_link *bpf_link_by_id(u32 id);
3384 
3385 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
3386 						 const struct bpf_prog *prog);
3387 void bpf_task_storage_free(struct task_struct *task);
3388 void bpf_cgrp_storage_free(struct cgroup *cgroup);
3389 const struct btf_func_model *
3390 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
3391 			 const struct bpf_insn *insn);
3392 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
3393 		       u16 desc_idx, u8 **func_addr);
3394 
3395 struct bpf_core_ctx {
3396 	struct bpf_verifier_log *log;
3397 	const struct btf *btf;
3398 };
3399 
3400 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
3401 				const struct bpf_reg_state *reg,
3402 				const char *field_name, u32 btf_id, const char *suffix);
3403 
3404 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
3405 			       const struct btf *reg_btf, u32 reg_id,
3406 			       const struct btf *arg_btf, u32 arg_id);
3407 
3408 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
3409 		   int relo_idx, void *insn);
3410 
3411 static inline bool unprivileged_ebpf_enabled(void)
3412 {
3413 	return !sysctl_unprivileged_bpf_disabled;
3414 }
3415 
3416 /* Not all bpf prog type has the bpf_ctx.
3417  * For the bpf prog type that has initialized the bpf_ctx,
3418  * this function can be used to decide if a kernel function
3419  * is called by a bpf program.
3420  */
3421 static inline bool has_current_bpf_ctx(void)
3422 {
3423 	return !!current->bpf_ctx;
3424 }
3425 
3426 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
3427 
3428 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
3429 		     enum bpf_dynptr_type type, u32 offset, u32 size);
3430 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
3431 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
3432 void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip);
3433 
3434 static __always_inline u32
3435 bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
3436 			     const void *ctx, bpf_prog_run_fn run_prog)
3437 {
3438 	const struct bpf_prog_array_item *item;
3439 	struct bpf_prog *prog;
3440 	struct bpf_run_ctx *old_run_ctx;
3441 	struct bpf_trace_run_ctx run_ctx;
3442 	u32 ret = 1;
3443 
3444 	if (unlikely(!array))
3445 		return ret;
3446 
3447 	migrate_disable();
3448 
3449 	run_ctx.is_uprobe = false;
3450 
3451 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
3452 	item = &array->items[0];
3453 	while ((prog = READ_ONCE(item->prog))) {
3454 		/* Skip dummy_bpf_prog placeholder (len == 0) */
3455 		if (unlikely(!prog->len)) {
3456 			item++;
3457 			continue;
3458 		}
3459 
3460 		if (unlikely(!bpf_prog_get_recursion_context(prog))) {
3461 			bpf_prog_inc_misses_counter(prog);
3462 			bpf_prog_put_recursion_context(prog);
3463 			item++;
3464 			continue;
3465 		}
3466 
3467 		run_ctx.bpf_cookie = item->bpf_cookie;
3468 
3469 		if (!prog->sleepable) {
3470 			guard(rcu)();
3471 			ret &= run_prog(prog, ctx);
3472 		} else {
3473 			ret &= run_prog(prog, ctx);
3474 		}
3475 
3476 		bpf_prog_put_recursion_context(prog);
3477 		item++;
3478 	}
3479 	bpf_reset_run_ctx(old_run_ctx);
3480 	migrate_enable();
3481 	return ret;
3482 }
3483 
3484 #else /* !CONFIG_BPF_SYSCALL */
3485 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
3486 {
3487 	return ERR_PTR(-EOPNOTSUPP);
3488 }
3489 
3490 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
3491 						     enum bpf_prog_type type,
3492 						     bool attach_drv)
3493 {
3494 	return ERR_PTR(-EOPNOTSUPP);
3495 }
3496 
3497 static inline void bpf_prog_add(struct bpf_prog *prog, int i)
3498 {
3499 }
3500 
3501 static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
3502 {
3503 }
3504 
3505 static inline void bpf_prog_put(struct bpf_prog *prog)
3506 {
3507 }
3508 
3509 static inline void bpf_prog_inc(struct bpf_prog *prog)
3510 {
3511 }
3512 
3513 static inline struct bpf_prog *__must_check
3514 bpf_prog_inc_not_zero(struct bpf_prog *prog)
3515 {
3516 	return ERR_PTR(-EOPNOTSUPP);
3517 }
3518 
3519 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
3520 				 const struct bpf_link_ops *ops,
3521 				 struct bpf_prog *prog, enum bpf_attach_type attach_type)
3522 {
3523 }
3524 
3525 static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type,
3526 					   const struct bpf_link_ops *ops, struct bpf_prog *prog,
3527 					   enum bpf_attach_type attach_type, bool sleepable)
3528 {
3529 }
3530 
3531 static inline void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type,
3532 				       const struct bpf_link_ops *ops, struct bpf_prog *prog,
3533 				       enum bpf_attach_type attach_type, u64 cookie)
3534 {
3535 }
3536 
3537 static inline int bpf_link_prime(struct bpf_link *link,
3538 				 struct bpf_link_primer *primer)
3539 {
3540 	return -EOPNOTSUPP;
3541 }
3542 
3543 static inline int bpf_link_settle(struct bpf_link_primer *primer)
3544 {
3545 	return -EOPNOTSUPP;
3546 }
3547 
3548 static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
3549 {
3550 }
3551 
3552 static inline void bpf_link_inc(struct bpf_link *link)
3553 {
3554 }
3555 
3556 static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link)
3557 {
3558 	return NULL;
3559 }
3560 
3561 static inline void bpf_link_put(struct bpf_link *link)
3562 {
3563 }
3564 
3565 static inline int bpf_obj_get_user(const char __user *pathname, int flags)
3566 {
3567 	return -EOPNOTSUPP;
3568 }
3569 
3570 static inline bool bpf_token_capable(const struct bpf_token *token, int cap)
3571 {
3572 	return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));
3573 }
3574 
3575 static inline void bpf_token_inc(struct bpf_token *token)
3576 {
3577 }
3578 
3579 static inline void bpf_token_put(struct bpf_token *token)
3580 {
3581 }
3582 
3583 static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
3584 {
3585 	return ERR_PTR(-EOPNOTSUPP);
3586 }
3587 
3588 static inline int bpf_token_get_info_by_fd(struct bpf_token *token,
3589 					   const union bpf_attr *attr,
3590 					   union bpf_attr __user *uattr)
3591 {
3592 	return -EOPNOTSUPP;
3593 }
3594 
3595 static inline void __dev_flush(struct list_head *flush_list)
3596 {
3597 }
3598 
3599 struct xdp_frame;
3600 struct bpf_dtab_netdev;
3601 struct bpf_cpu_map_entry;
3602 
3603 static inline
3604 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
3605 		    struct net_device *dev_rx)
3606 {
3607 	return 0;
3608 }
3609 
3610 static inline
3611 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
3612 		    struct net_device *dev_rx)
3613 {
3614 	return 0;
3615 }
3616 
3617 static inline
3618 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
3619 			  struct bpf_map *map, bool exclude_ingress)
3620 {
3621 	return 0;
3622 }
3623 
3624 struct sk_buff;
3625 
3626 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
3627 					   struct sk_buff *skb,
3628 					   const struct bpf_prog *xdp_prog)
3629 {
3630 	return 0;
3631 }
3632 
3633 static inline
3634 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
3635 			   const struct bpf_prog *xdp_prog,
3636 			   struct bpf_map *map, bool exclude_ingress)
3637 {
3638 	return 0;
3639 }
3640 
3641 static inline void __cpu_map_flush(struct list_head *flush_list)
3642 {
3643 }
3644 
3645 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
3646 				  struct xdp_frame *xdpf,
3647 				  struct net_device *dev_rx)
3648 {
3649 	return 0;
3650 }
3651 
3652 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
3653 					   struct sk_buff *skb)
3654 {
3655 	return -EOPNOTSUPP;
3656 }
3657 
3658 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
3659 				enum bpf_prog_type type)
3660 {
3661 	return ERR_PTR(-EOPNOTSUPP);
3662 }
3663 
3664 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
3665 					const union bpf_attr *kattr,
3666 					union bpf_attr __user *uattr)
3667 {
3668 	return -ENOTSUPP;
3669 }
3670 
3671 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
3672 					const union bpf_attr *kattr,
3673 					union bpf_attr __user *uattr)
3674 {
3675 	return -ENOTSUPP;
3676 }
3677 
3678 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
3679 					    const union bpf_attr *kattr,
3680 					    union bpf_attr __user *uattr)
3681 {
3682 	return -ENOTSUPP;
3683 }
3684 
3685 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
3686 						   const union bpf_attr *kattr,
3687 						   union bpf_attr __user *uattr)
3688 {
3689 	return -ENOTSUPP;
3690 }
3691 
3692 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
3693 					      const union bpf_attr *kattr,
3694 					      union bpf_attr __user *uattr)
3695 {
3696 	return -ENOTSUPP;
3697 }
3698 
3699 static inline void bpf_map_put(struct bpf_map *map)
3700 {
3701 }
3702 
3703 static inline struct bpf_prog *bpf_prog_by_id(u32 id)
3704 {
3705 	return ERR_PTR(-ENOTSUPP);
3706 }
3707 
3708 static inline int btf_struct_access(struct bpf_verifier_log *log,
3709 				    const struct bpf_reg_state *reg,
3710 				    int off, int size, enum bpf_access_type atype,
3711 				    u32 *next_btf_id, enum bpf_type_flag *flag,
3712 				    const char **field_name)
3713 {
3714 	return -EACCES;
3715 }
3716 
3717 static inline const struct bpf_func_proto *
3718 bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
3719 {
3720 	return NULL;
3721 }
3722 
3723 static inline void bpf_task_storage_free(struct task_struct *task)
3724 {
3725 }
3726 
3727 static inline const struct btf_func_model *
3728 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
3729 			 const struct bpf_insn *insn)
3730 {
3731 	return NULL;
3732 }
3733 
3734 static inline int
3735 bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
3736 		   u16 desc_idx, u8 **func_addr)
3737 {
3738 	return -ENOTSUPP;
3739 }
3740 
3741 static inline bool unprivileged_ebpf_enabled(void)
3742 {
3743 	return false;
3744 }
3745 
3746 static inline bool has_current_bpf_ctx(void)
3747 {
3748 	return false;
3749 }
3750 
3751 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
3752 {
3753 }
3754 
3755 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
3756 {
3757 }
3758 
3759 static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
3760 				   enum bpf_dynptr_type type, u32 offset, u32 size)
3761 {
3762 }
3763 
3764 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
3765 {
3766 }
3767 
3768 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
3769 {
3770 }
3771 
3772 static inline void bpf_prog_report_arena_violation(bool write, unsigned long addr,
3773 						   unsigned long fault_ip)
3774 {
3775 }
3776 #endif /* CONFIG_BPF_SYSCALL */
3777 
3778 static inline bool bpf_net_capable(void)
3779 {
3780 	return capable(CAP_NET_ADMIN) || capable(CAP_SYS_ADMIN);
3781 }
3782 
3783 static __always_inline int
3784 bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
3785 {
3786 	int ret = -EFAULT;
3787 
3788 	if (IS_ENABLED(CONFIG_BPF_EVENTS))
3789 		ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
3790 	if (unlikely(ret < 0))
3791 		memset(dst, 0, size);
3792 	return ret;
3793 }
3794 
3795 void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len);
3796 
3797 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
3798 						 enum bpf_prog_type type)
3799 {
3800 	return bpf_prog_get_type_dev(ufd, type, false);
3801 }
3802 
3803 void __bpf_free_used_maps(struct bpf_prog_aux *aux,
3804 			  struct bpf_map **used_maps, u32 len);
3805 
3806 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
3807 
3808 int bpf_prog_offload_compile(struct bpf_prog *prog);
3809 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
3810 int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
3811 			       struct bpf_prog *prog);
3812 
3813 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
3814 
3815 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
3816 int bpf_map_offload_update_elem(struct bpf_map *map,
3817 				void *key, void *value, u64 flags);
3818 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
3819 int bpf_map_offload_get_next_key(struct bpf_map *map,
3820 				 void *key, void *next_key);
3821 
3822 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
3823 
3824 struct bpf_offload_dev *
3825 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
3826 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
3827 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
3828 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
3829 				    struct net_device *netdev);
3830 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
3831 				       struct net_device *netdev);
3832 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
3833 
3834 void unpriv_ebpf_notify(int new_state);
3835 
3836 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
3837 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3838 			      struct bpf_prog_aux *prog_aux);
3839 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
3840 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
3841 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
3842 void bpf_prog_dev_bound_moved(struct bpf_prog *prog);
3843 void bpf_dev_bound_netdev_unregister(struct net_device *dev);
3844 
3845 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3846 {
3847 	return aux->dev_bound;
3848 }
3849 
3850 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
3851 {
3852 	return aux->offload_requested;
3853 }
3854 
3855 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);
3856 
3857 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3858 {
3859 	return unlikely(map->ops == &bpf_map_offload_ops);
3860 }
3861 
3862 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
3863 void bpf_map_offload_map_free(struct bpf_map *map);
3864 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
3865 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3866 			      const union bpf_attr *kattr,
3867 			      union bpf_attr __user *uattr);
3868 
3869 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
3870 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
3871 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
3872 int sock_map_bpf_prog_query(const union bpf_attr *attr,
3873 			    union bpf_attr __user *uattr);
3874 int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog);
3875 
3876 void sock_map_unhash(struct sock *sk);
3877 void sock_map_destroy(struct sock *sk);
3878 void sock_map_close(struct sock *sk, long timeout);
3879 #else
3880 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3881 					    struct bpf_prog_aux *prog_aux)
3882 {
3883 	return -EOPNOTSUPP;
3884 }
3885 
3886 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
3887 						u32 func_id)
3888 {
3889 	return NULL;
3890 }
3891 
3892 static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
3893 					  union bpf_attr *attr)
3894 {
3895 	return -EOPNOTSUPP;
3896 }
3897 
3898 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
3899 					     struct bpf_prog *old_prog)
3900 {
3901 	return -EOPNOTSUPP;
3902 }
3903 
3904 static inline void bpf_prog_dev_bound_moved(struct bpf_prog *prog)
3905 {
3906 }
3907 
3908 static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
3909 {
3910 }
3911 
3912 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3913 {
3914 	return false;
3915 }
3916 
3917 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
3918 {
3919 	return false;
3920 }
3921 
3922 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
3923 {
3924 	return false;
3925 }
3926 
3927 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3928 {
3929 	return false;
3930 }
3931 
3932 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
3933 {
3934 	return ERR_PTR(-EOPNOTSUPP);
3935 }
3936 
3937 static inline void bpf_map_offload_map_free(struct bpf_map *map)
3938 {
3939 }
3940 
3941 static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
3942 {
3943 	return 0;
3944 }
3945 
3946 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3947 					    const union bpf_attr *kattr,
3948 					    union bpf_attr __user *uattr)
3949 {
3950 	return -ENOTSUPP;
3951 }
3952 
3953 #ifdef CONFIG_BPF_SYSCALL
3954 static inline int sock_map_get_from_fd(const union bpf_attr *attr,
3955 				       struct bpf_prog *prog)
3956 {
3957 	return -EINVAL;
3958 }
3959 
3960 static inline int sock_map_prog_detach(const union bpf_attr *attr,
3961 				       enum bpf_prog_type ptype)
3962 {
3963 	return -EOPNOTSUPP;
3964 }
3965 
3966 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
3967 					   u64 flags)
3968 {
3969 	return -EOPNOTSUPP;
3970 }
3971 
3972 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
3973 					  union bpf_attr __user *uattr)
3974 {
3975 	return -EINVAL;
3976 }
3977 
3978 static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog)
3979 {
3980 	return -EOPNOTSUPP;
3981 }
3982 #endif /* CONFIG_BPF_SYSCALL */
3983 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
3984 
3985 static __always_inline void
3986 bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
3987 {
3988 	const struct bpf_prog_array_item *item;
3989 	struct bpf_prog *prog;
3990 
3991 	if (unlikely(!array))
3992 		return;
3993 
3994 	item = &array->items[0];
3995 	while ((prog = READ_ONCE(item->prog))) {
3996 		bpf_prog_inc_misses_counter(prog);
3997 		item++;
3998 	}
3999 }
4000 
4001 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
4002 void bpf_sk_reuseport_detach(struct sock *sk);
4003 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
4004 				       void *value);
4005 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
4006 				       void *value, u64 map_flags);
4007 #else
4008 static inline void bpf_sk_reuseport_detach(struct sock *sk)
4009 {
4010 }
4011 
4012 #ifdef CONFIG_BPF_SYSCALL
4013 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
4014 						     void *key, void *value)
4015 {
4016 	return -EOPNOTSUPP;
4017 }
4018 
4019 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
4020 						     void *key, void *value,
4021 						     u64 map_flags)
4022 {
4023 	return -EOPNOTSUPP;
4024 }
4025 #endif /* CONFIG_BPF_SYSCALL */
4026 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
4027 
4028 #ifdef CONFIG_KEYS
4029 struct bpf_key {
4030 	struct key *key;
4031 	bool has_ref;
4032 };
4033 #endif /* CONFIG_KEYS */
4034 
4035 #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL)
4036 struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags);
4037 struct bpf_key *bpf_lookup_system_key(u64 id);
4038 struct bpf_key *bpf_lookup_keyring(void);
4039 bool bpf_keyring_enforced(void);
4040 void bpf_key_put(struct bpf_key *bkey);
4041 int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
4042 			       const struct bpf_dynptr *sig_p,
4043 			       struct bpf_key *trusted_keyring);
4044 
4045 static inline s32 bpf_key_serial(const struct bpf_key *key)
4046 {
4047 	return key->has_ref ? key->key->serial : 0;
4048 }
4049 #else
4050 static inline struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags)
4051 {
4052 	return NULL;
4053 }
4054 
4055 static inline struct bpf_key *bpf_lookup_system_key(u64 id)
4056 {
4057 	return NULL;
4058 }
4059 
4060 static inline struct bpf_key *bpf_lookup_keyring(void)
4061 {
4062 	return NULL;
4063 }
4064 
4065 static inline bool bpf_keyring_enforced(void)
4066 {
4067 	return false;
4068 }
4069 
4070 static inline void bpf_key_put(struct bpf_key *bkey)
4071 {
4072 }
4073 
4074 static inline int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
4075 					     const struct bpf_dynptr *sig_p,
4076 					     struct bpf_key *trusted_keyring)
4077 {
4078 	return -EOPNOTSUPP;
4079 }
4080 
4081 static inline s32 bpf_key_serial(const struct bpf_key *key)
4082 {
4083 	return 0;
4084 }
4085 #endif /* defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) */
4086 
4087 /* verifier prototypes for helper functions called from eBPF programs */
4088 extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
4089 extern const struct bpf_func_proto bpf_map_update_elem_proto;
4090 extern const struct bpf_func_proto bpf_map_delete_elem_proto;
4091 extern const struct bpf_func_proto bpf_map_push_elem_proto;
4092 extern const struct bpf_func_proto bpf_map_pop_elem_proto;
4093 extern const struct bpf_func_proto bpf_map_peek_elem_proto;
4094 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;
4095 
4096 extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
4097 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
4098 extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
4099 extern const struct bpf_func_proto bpf_tail_call_proto;
4100 extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
4101 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
4102 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
4103 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
4104 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
4105 extern const struct bpf_func_proto bpf_get_current_comm_proto;
4106 extern const struct bpf_func_proto bpf_get_stackid_proto;
4107 extern const struct bpf_func_proto bpf_get_stack_proto;
4108 extern const struct bpf_func_proto bpf_get_stack_sleepable_proto;
4109 extern const struct bpf_func_proto bpf_get_task_stack_proto;
4110 extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto;
4111 extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
4112 extern const struct bpf_func_proto bpf_get_stack_proto_pe;
4113 extern const struct bpf_func_proto bpf_sock_map_update_proto;
4114 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
4115 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
4116 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
4117 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
4118 extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto;
4119 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
4120 extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
4121 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
4122 extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
4123 extern const struct bpf_func_proto bpf_spin_lock_proto;
4124 extern const struct bpf_func_proto bpf_spin_unlock_proto;
4125 extern const struct bpf_func_proto bpf_get_local_storage_proto;
4126 extern const struct bpf_func_proto bpf_strtol_proto;
4127 extern const struct bpf_func_proto bpf_strtoul_proto;
4128 extern const struct bpf_func_proto bpf_tcp_sock_proto;
4129 extern const struct bpf_func_proto bpf_jiffies64_proto;
4130 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
4131 extern const struct bpf_func_proto bpf_event_output_data_proto;
4132 extern const struct bpf_func_proto bpf_ringbuf_output_proto;
4133 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
4134 extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
4135 extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
4136 extern const struct bpf_func_proto bpf_ringbuf_query_proto;
4137 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
4138 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
4139 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
4140 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
4141 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
4142 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
4143 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
4144 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
4145 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
4146 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
4147 extern const struct bpf_func_proto bpf_copy_from_user_proto;
4148 extern const struct bpf_func_proto bpf_snprintf_btf_proto;
4149 extern const struct bpf_func_proto bpf_snprintf_proto;
4150 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
4151 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
4152 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
4153 extern const struct bpf_func_proto bpf_sock_from_file_proto;
4154 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
4155 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
4156 extern const struct bpf_func_proto bpf_task_storage_get_proto;
4157 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
4158 extern const struct bpf_func_proto bpf_task_storage_delete_proto;
4159 extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
4160 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
4161 extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
4162 extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
4163 extern const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto;
4164 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
4165 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
4166 extern const struct bpf_func_proto bpf_find_vma_proto;
4167 extern const struct bpf_func_proto bpf_loop_proto;
4168 extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
4169 extern const struct bpf_func_proto bpf_set_retval_proto;
4170 extern const struct bpf_func_proto bpf_get_retval_proto;
4171 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
4172 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
4173 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
4174 
4175 const struct bpf_func_proto *tracing_prog_func_proto(
4176   enum bpf_func_id func_id, const struct bpf_prog *prog);
4177 
4178 /* Shared helpers among cBPF and eBPF. */
4179 void bpf_user_rnd_init_once(void);
4180 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
4181 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
4182 
4183 #if defined(CONFIG_NET)
4184 bool bpf_sock_common_is_valid_access(int off, int size,
4185 				     enum bpf_access_type type,
4186 				     struct bpf_insn_access_aux *info);
4187 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
4188 			      struct bpf_insn_access_aux *info);
4189 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
4190 				const struct bpf_insn *si,
4191 				struct bpf_insn *insn_buf,
4192 				struct bpf_prog *prog,
4193 				u32 *target_size);
4194 int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
4195 			       struct bpf_dynptr *ptr);
4196 #else
4197 static inline bool bpf_sock_common_is_valid_access(int off, int size,
4198 						   enum bpf_access_type type,
4199 						   struct bpf_insn_access_aux *info)
4200 {
4201 	return false;
4202 }
4203 static inline bool bpf_sock_is_valid_access(int off, int size,
4204 					    enum bpf_access_type type,
4205 					    struct bpf_insn_access_aux *info)
4206 {
4207 	return false;
4208 }
4209 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
4210 					      const struct bpf_insn *si,
4211 					      struct bpf_insn *insn_buf,
4212 					      struct bpf_prog *prog,
4213 					      u32 *target_size)
4214 {
4215 	return 0;
4216 }
4217 static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
4218 					     struct bpf_dynptr *ptr)
4219 {
4220 	return -EOPNOTSUPP;
4221 }
4222 #endif
4223 
4224 #ifdef CONFIG_INET
4225 struct sk_reuseport_kern {
4226 	struct sk_buff *skb;
4227 	struct sock *sk;
4228 	struct sock *selected_sk;
4229 	struct sock *migrating_sk;
4230 	void *data_end;
4231 	u32 hash;
4232 	u32 reuseport_id;
4233 	bool bind_inany;
4234 };
4235 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
4236 				  struct bpf_insn_access_aux *info);
4237 
4238 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
4239 				    const struct bpf_insn *si,
4240 				    struct bpf_insn *insn_buf,
4241 				    struct bpf_prog *prog,
4242 				    u32 *target_size);
4243 
4244 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
4245 				  struct bpf_insn_access_aux *info);
4246 
4247 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
4248 				    const struct bpf_insn *si,
4249 				    struct bpf_insn *insn_buf,
4250 				    struct bpf_prog *prog,
4251 				    u32 *target_size);
4252 #else
4253 static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
4254 						enum bpf_access_type type,
4255 						struct bpf_insn_access_aux *info)
4256 {
4257 	return false;
4258 }
4259 
4260 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
4261 						  const struct bpf_insn *si,
4262 						  struct bpf_insn *insn_buf,
4263 						  struct bpf_prog *prog,
4264 						  u32 *target_size)
4265 {
4266 	return 0;
4267 }
4268 static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
4269 						enum bpf_access_type type,
4270 						struct bpf_insn_access_aux *info)
4271 {
4272 	return false;
4273 }
4274 
4275 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
4276 						  const struct bpf_insn *si,
4277 						  struct bpf_insn *insn_buf,
4278 						  struct bpf_prog *prog,
4279 						  u32 *target_size)
4280 {
4281 	return 0;
4282 }
4283 #endif /* CONFIG_INET */
4284 
4285 enum bpf_text_poke_type {
4286 	BPF_MOD_NOP,
4287 	BPF_MOD_CALL,
4288 	BPF_MOD_JUMP,
4289 };
4290 
4291 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t,
4292 		       enum bpf_text_poke_type new_t, void *old_addr,
4293 		       void *new_addr);
4294 
4295 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
4296 			       struct bpf_prog *new, struct bpf_prog *old);
4297 
4298 void *bpf_arch_text_copy(void *dst, void *src, size_t len);
4299 int bpf_arch_text_invalidate(void *dst, size_t len);
4300 
4301 struct btf_id_set;
4302 bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
4303 
4304 #define MAX_BPRINTF_VARARGS		12
4305 #define MAX_BPRINTF_BUF			1024
4306 
4307 /* Per-cpu temp buffers used by printf-like helpers to store the bprintf binary
4308  * arguments representation.
4309  */
4310 #define MAX_BPRINTF_BIN_ARGS	512
4311 
4312 struct bpf_bprintf_buffers {
4313 	char bin_args[MAX_BPRINTF_BIN_ARGS];
4314 	char buf[MAX_BPRINTF_BUF];
4315 };
4316 
4317 struct bpf_bprintf_data {
4318 	u32 *bin_args;
4319 	char *buf;
4320 	bool get_bin_args;
4321 	bool get_buf;
4322 };
4323 
4324 int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args,
4325 			u32 num_args, struct bpf_bprintf_data *data);
4326 void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
4327 int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs);
4328 void bpf_put_buffers(void);
4329 
4330 int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags);
4331 void bpf_prog_stream_free(struct bpf_prog *prog);
4332 int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len);
4333 int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags);
4334 void bpf_stream_stage_init(struct bpf_stream_stage *ss);
4335 void bpf_stream_stage_free(struct bpf_stream_stage *ss);
4336 __printf(2, 3)
4337 int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...);
4338 int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
4339 			    enum bpf_stream_id stream_id);
4340 int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss);
4341 
4342 #define bpf_stream_printk(ss, ...) bpf_stream_stage_printk(&ss, __VA_ARGS__)
4343 #define bpf_stream_dump_stack(ss) bpf_stream_stage_dump_stack(&ss)
4344 
4345 #define bpf_stream_stage(ss, prog, stream_id, expr)            \
4346 	({                                                     \
4347 		bpf_stream_stage_init(&ss);                    \
4348 		(expr);                                        \
4349 		bpf_stream_stage_commit(&ss, prog, stream_id); \
4350 		bpf_stream_stage_free(&ss);                    \
4351 	})
4352 
4353 #ifdef CONFIG_BPF_LSM
4354 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
4355 void bpf_cgroup_atype_put(int cgroup_atype);
4356 #else
4357 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
4358 static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
4359 #endif /* CONFIG_BPF_LSM */
4360 
4361 static inline bool type_is_alloc(u32 type)
4362 {
4363 	return type & MEM_ALLOC;
4364 }
4365 
4366 static inline gfp_t bpf_memcg_flags(gfp_t flags)
4367 {
4368 	if (memcg_bpf_enabled())
4369 		return flags | __GFP_ACCOUNT;
4370 	return flags;
4371 }
4372 
4373 static inline bool bpf_is_subprog(const struct bpf_prog *prog)
4374 {
4375 	return prog->aux->func_idx != 0;
4376 }
4377 
4378 const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off);
4379 struct bpf_linfo_source {
4380 	const char *file;
4381 	const char *line;
4382 	u32 file_name_off;
4383 	int line_num;
4384 	int line_col;
4385 };
4386 
4387 void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo,
4388 			  struct bpf_linfo_source *src);
4389 int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
4390 			   const char **linep, int *nump);
4391 struct bpf_prog *bpf_prog_find_from_stack(void);
4392 
4393 int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog);
4394 int bpf_insn_array_ready(struct bpf_map *map);
4395 void bpf_insn_array_release(struct bpf_map *map);
4396 void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len);
4397 void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len);
4398 
4399 #ifdef CONFIG_BPF_SYSCALL
4400 void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image);
4401 #else
4402 static inline void
4403 bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image)
4404 {
4405 }
4406 #endif
4407 
4408 static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type)
4409 {
4410 	switch (map_type) {
4411 	case BPF_MAP_TYPE_PERCPU_ARRAY:
4412 	case BPF_MAP_TYPE_PERCPU_HASH:
4413 	case BPF_MAP_TYPE_LRU_PERCPU_HASH:
4414 	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
4415 		return true;
4416 	default:
4417 		return false;
4418 	}
4419 }
4420 
4421 #define BPF_EXIST_FLAGS		(BPF_EXIST | BPF_NOEXIST)
4422 #define BPF_CPU_FLAGS		(BPF_F_CPU | BPF_F_ALL_CPUS)
4423 
4424 static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags)
4425 {
4426 	u32 cpu;
4427 
4428 	if ((u32)flags & ~allowed_flags)
4429 		return -EINVAL;
4430 
4431 	if ((flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
4432 		return -EINVAL;
4433 
4434 	if (!(flags & BPF_F_CPU) && flags >> 32)
4435 		return -EINVAL;
4436 
4437 	if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) {
4438 		if (!bpf_map_is_percpu_map(map->map_type))
4439 			return -EINVAL;
4440 		if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS))
4441 			return -EINVAL;
4442 
4443 		cpu = flags >> 32;
4444 		if ((flags & BPF_F_CPU) && (cpu >= nr_cpu_ids || !cpu_possible(cpu)))
4445 			return -ERANGE;
4446 	}
4447 
4448 	return 0;
4449 }
4450 
4451 #endif /* _LINUX_BPF_H */
4452