xref: /linux/net/core/filter.c (revision 84afebd853437e44cb71f718be11d90e4c1a59dd)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Linux Socket Filter - Kernel level socket filtering
4  *
5  * Based on the design of the Berkeley Packet Filter. The new
6  * internal format has been designed by PLUMgrid:
7  *
8  *	Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
9  *
10  * Authors:
11  *
12  *	Jay Schulist <jschlst@samba.org>
13  *	Alexei Starovoitov <ast@plumgrid.com>
14  *	Daniel Borkmann <dborkman@redhat.com>
15  *
16  * Andi Kleen - Fix a few bad bugs and races.
17  * Kris Katterjohn - Added many additional checks in bpf_check_classic()
18  */
19 
20 #include <linux/atomic.h>
21 #include <linux/bpf_verifier.h>
22 #include <linux/module.h>
23 #include <linux/types.h>
24 #include <linux/mm.h>
25 #include <linux/fcntl.h>
26 #include <linux/socket.h>
27 #include <linux/sock_diag.h>
28 #include <linux/in.h>
29 #include <linux/inet.h>
30 #include <linux/netdevice.h>
31 #include <linux/if_packet.h>
32 #include <linux/if_arp.h>
33 #include <linux/gfp.h>
34 #include <net/inet_common.h>
35 #include <net/ip.h>
36 #include <net/protocol.h>
37 #include <net/netlink.h>
38 #include <linux/skbuff.h>
39 #include <linux/skmsg.h>
40 #include <net/sock.h>
41 #include <net/flow_dissector.h>
42 #include <linux/errno.h>
43 #include <linux/timer.h>
44 #include <linux/uaccess.h>
45 #include <linux/unaligned.h>
46 #include <linux/filter.h>
47 #include <linux/ratelimit.h>
48 #include <linux/seccomp.h>
49 #include <linux/if_vlan.h>
50 #include <linux/bpf.h>
51 #include <linux/btf.h>
52 #include <net/sch_generic.h>
53 #include <net/cls_cgroup.h>
54 #include <net/dst_metadata.h>
55 #include <net/dst.h>
56 #include <net/sock_reuseport.h>
57 #include <net/busy_poll.h>
58 #include <net/tcp.h>
59 #include <net/gre.h>
60 #include <net/xfrm.h>
61 #include <net/udp.h>
62 #include <linux/bpf_trace.h>
63 #include <net/xdp_sock.h>
64 #include <linux/inetdevice.h>
65 #include <net/inet_hashtables.h>
66 #include <net/inet6_hashtables.h>
67 #include <net/ip_fib.h>
68 #include <net/nexthop.h>
69 #include <net/flow.h>
70 #include <net/arp.h>
71 #include <net/ipv6.h>
72 #include <net/net_namespace.h>
73 #include <linux/seg6_local.h>
74 #include <net/seg6.h>
75 #include <net/seg6_local.h>
76 #include <net/lwtunnel.h>
77 #include <net/bpf_sk_storage.h>
78 #include <net/transp_v6.h>
79 #include <linux/btf_ids.h>
80 #include <net/tls.h>
81 #include <net/xdp.h>
82 #include <net/mptcp.h>
83 #include <net/netfilter/nf_conntrack_bpf.h>
84 #include <net/netkit.h>
85 #include <linux/un.h>
86 #include <net/xdp_sock_drv.h>
87 #include <net/inet_dscp.h>
88 #include <linux/icmpv6.h>
89 #include <net/icmp.h>
90 #include <net/ip6_route.h>
91 
92 #include "dev.h"
93 
94 /* Keep the struct bpf_fib_lookup small so that it fits into a cacheline */
95 static_assert(sizeof(struct bpf_fib_lookup) == 64, "struct bpf_fib_lookup size check");
96 
97 static const struct bpf_func_proto *
98 bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
99 
100 int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len)
101 {
102 	if (in_compat_syscall()) {
103 		struct compat_sock_fprog f32;
104 
105 		if (len != sizeof(f32))
106 			return -EINVAL;
107 		if (copy_from_sockptr(&f32, src, sizeof(f32)))
108 			return -EFAULT;
109 		memset(dst, 0, sizeof(*dst));
110 		dst->len = f32.len;
111 		dst->filter = compat_ptr(f32.filter);
112 	} else {
113 		if (len != sizeof(*dst))
114 			return -EINVAL;
115 		if (copy_from_sockptr(dst, src, sizeof(*dst)))
116 			return -EFAULT;
117 	}
118 
119 	return 0;
120 }
121 EXPORT_SYMBOL_GPL(copy_bpf_fprog_from_user);
122 
123 /**
124  *	sk_filter_trim_cap - run a packet through a socket filter
125  *	@sk: sock associated with &sk_buff
126  *	@skb: buffer to filter
127  *	@cap: limit on how short the eBPF program may trim the packet
128  *
129  * Run the eBPF program and then cut skb->data to correct size returned by
130  * the program. If pkt_len is 0 we toss packet. If skb->len is smaller
131  * than pkt_len we keep whole skb->data. This is the socket level
132  * wrapper to bpf_prog_run. It returns 0 if the packet should
133  * be accepted or a drop_reason if the packet should be tossed.
134  *
135  */
136 enum skb_drop_reason
137 sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
138 {
139 	enum skb_drop_reason drop_reason;
140 	struct sk_filter *filter;
141 	int err;
142 
143 	/*
144 	 * If the skb was allocated from pfmemalloc reserves, only
145 	 * allow SOCK_MEMALLOC sockets to use it as this socket is
146 	 * helping free memory
147 	 */
148 	if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC)) {
149 		NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
150 		return SKB_DROP_REASON_PFMEMALLOC;
151 	}
152 	err = BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb);
153 	if (err)
154 		return SKB_DROP_REASON_SOCKET_FILTER;
155 
156 	err = security_sock_rcv_skb(sk, skb);
157 	if (err)
158 		return SKB_DROP_REASON_SECURITY_HOOK;
159 
160 	drop_reason = 0;
161 	rcu_read_lock();
162 	filter = rcu_dereference(sk->sk_filter);
163 	if (filter) {
164 		struct sock *save_sk = skb->sk;
165 		unsigned int pkt_len;
166 
167 		skb->sk = sk;
168 		pkt_len = bpf_prog_run_save_cb(filter->prog, skb);
169 		skb->sk = save_sk;
170 		err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
171 		if (err)
172 			drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
173 	}
174 	rcu_read_unlock();
175 
176 	return drop_reason;
177 }
178 EXPORT_SYMBOL(sk_filter_trim_cap);
179 
180 BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb)
181 {
182 	return skb_get_poff(skb);
183 }
184 
185 BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
186 {
187 	struct nlattr *nla;
188 
189 	if (skb_is_nonlinear(skb))
190 		return 0;
191 
192 	if (skb->len < sizeof(struct nlattr))
193 		return 0;
194 
195 	if (a > skb->len - sizeof(struct nlattr))
196 		return 0;
197 
198 	nla = nla_find((struct nlattr *) &skb->data[a], skb->len - a, x);
199 	if (nla)
200 		return (void *) nla - (void *) skb->data;
201 
202 	return 0;
203 }
204 
205 BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
206 {
207 	struct nlattr *nla;
208 
209 	if (skb_is_nonlinear(skb))
210 		return 0;
211 
212 	if (skb->len < sizeof(struct nlattr))
213 		return 0;
214 
215 	if (a > skb->len - sizeof(struct nlattr))
216 		return 0;
217 
218 	nla = (struct nlattr *) &skb->data[a];
219 	if (!nla_ok(nla, skb->len - a))
220 		return 0;
221 
222 	nla = nla_find_nested(nla, x);
223 	if (nla)
224 		return (void *) nla - (void *) skb->data;
225 
226 	return 0;
227 }
228 
229 static int bpf_skb_load_helper_convert_offset(const struct sk_buff *skb, int offset)
230 {
231 	if (likely(offset >= 0))
232 		return offset;
233 
234 	if (offset >= SKF_NET_OFF)
235 		return offset - SKF_NET_OFF + skb_network_offset(skb);
236 
237 	if (offset >= SKF_LL_OFF && skb_mac_header_was_set(skb))
238 		return offset - SKF_LL_OFF + skb_mac_offset(skb);
239 
240 	return INT_MIN;
241 }
242 
243 BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *,
244 	   data, int, headlen, int, offset)
245 {
246 	u8 tmp;
247 	const int len = sizeof(tmp);
248 
249 	offset = bpf_skb_load_helper_convert_offset(skb, offset);
250 	if (offset == INT_MIN)
251 		return -EFAULT;
252 
253 	if (headlen - offset >= len)
254 		return *(u8 *)(data + offset);
255 	if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
256 		return tmp;
257 	else
258 		return -EFAULT;
259 }
260 
261 BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb,
262 	   int, offset)
263 {
264 	return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len,
265 					 offset);
266 }
267 
268 BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *,
269 	   data, int, headlen, int, offset)
270 {
271 	__be16 tmp;
272 	const int len = sizeof(tmp);
273 
274 	offset = bpf_skb_load_helper_convert_offset(skb, offset);
275 	if (offset == INT_MIN)
276 		return -EFAULT;
277 
278 	if (headlen - offset >= len)
279 		return get_unaligned_be16(data + offset);
280 	if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
281 		return be16_to_cpu(tmp);
282 	else
283 		return -EFAULT;
284 }
285 
286 BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb,
287 	   int, offset)
288 {
289 	return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len,
290 					  offset);
291 }
292 
293 BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *,
294 	   data, int, headlen, int, offset)
295 {
296 	__be32 tmp;
297 	const int len = sizeof(tmp);
298 
299 	offset = bpf_skb_load_helper_convert_offset(skb, offset);
300 	if (offset == INT_MIN)
301 		return -EFAULT;
302 
303 	if (headlen - offset >= len)
304 		return get_unaligned_be32(data + offset);
305 	if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
306 		return be32_to_cpu(tmp);
307 	else
308 		return -EFAULT;
309 }
310 
311 BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb,
312 	   int, offset)
313 {
314 	return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len,
315 					  offset);
316 }
317 
318 static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
319 			      struct bpf_insn *insn_buf)
320 {
321 	struct bpf_insn *insn = insn_buf;
322 
323 	switch (skb_field) {
324 	case SKF_AD_MARK:
325 		BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);
326 
327 		*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
328 				      offsetof(struct sk_buff, mark));
329 		break;
330 
331 	case SKF_AD_PKTTYPE:
332 		*insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET);
333 		*insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
334 #ifdef __BIG_ENDIAN_BITFIELD
335 		*insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
336 #endif
337 		break;
338 
339 	case SKF_AD_QUEUE:
340 		BUILD_BUG_ON(sizeof_field(struct sk_buff, queue_mapping) != 2);
341 
342 		*insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
343 				      offsetof(struct sk_buff, queue_mapping));
344 		break;
345 
346 	case SKF_AD_VLAN_TAG:
347 		BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);
348 
349 		/* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */
350 		*insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
351 				      offsetof(struct sk_buff, vlan_tci));
352 		break;
353 	case SKF_AD_VLAN_TAG_PRESENT:
354 		BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_all) != 4);
355 		*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
356 				      offsetof(struct sk_buff, vlan_all));
357 		*insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1);
358 		*insn++ = BPF_ALU32_IMM(BPF_MOV, dst_reg, 1);
359 		break;
360 	}
361 
362 	return insn - insn_buf;
363 }
364 
365 static bool convert_bpf_extensions(struct sock_filter *fp,
366 				   struct bpf_insn **insnp)
367 {
368 	struct bpf_insn *insn = *insnp;
369 	u32 cnt;
370 
371 	switch (fp->k) {
372 	case SKF_AD_OFF + SKF_AD_PROTOCOL:
373 		BUILD_BUG_ON(sizeof_field(struct sk_buff, protocol) != 2);
374 
375 		/* A = *(u16 *) (CTX + offsetof(protocol)) */
376 		*insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
377 				      offsetof(struct sk_buff, protocol));
378 		/* A = ntohs(A) [emitting a nop or swap16] */
379 		*insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
380 		break;
381 
382 	case SKF_AD_OFF + SKF_AD_PKTTYPE:
383 		cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn);
384 		insn += cnt - 1;
385 		break;
386 
387 	case SKF_AD_OFF + SKF_AD_IFINDEX:
388 	case SKF_AD_OFF + SKF_AD_HATYPE:
389 		BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4);
390 		BUILD_BUG_ON(sizeof_field(struct net_device, type) != 2);
391 
392 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
393 				      BPF_REG_TMP, BPF_REG_CTX,
394 				      offsetof(struct sk_buff, dev));
395 		/* if (tmp != 0) goto pc + 1 */
396 		*insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1);
397 		*insn++ = BPF_EXIT_INSN();
398 		if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX)
399 			*insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP,
400 					    offsetof(struct net_device, ifindex));
401 		else
402 			*insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP,
403 					    offsetof(struct net_device, type));
404 		break;
405 
406 	case SKF_AD_OFF + SKF_AD_MARK:
407 		cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn);
408 		insn += cnt - 1;
409 		break;
410 
411 	case SKF_AD_OFF + SKF_AD_RXHASH:
412 		BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);
413 
414 		*insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
415 				    offsetof(struct sk_buff, hash));
416 		break;
417 
418 	case SKF_AD_OFF + SKF_AD_QUEUE:
419 		cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn);
420 		insn += cnt - 1;
421 		break;
422 
423 	case SKF_AD_OFF + SKF_AD_VLAN_TAG:
424 		cnt = convert_skb_access(SKF_AD_VLAN_TAG,
425 					 BPF_REG_A, BPF_REG_CTX, insn);
426 		insn += cnt - 1;
427 		break;
428 
429 	case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
430 		cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT,
431 					 BPF_REG_A, BPF_REG_CTX, insn);
432 		insn += cnt - 1;
433 		break;
434 
435 	case SKF_AD_OFF + SKF_AD_VLAN_TPID:
436 		BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_proto) != 2);
437 
438 		/* A = *(u16 *) (CTX + offsetof(vlan_proto)) */
439 		*insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
440 				      offsetof(struct sk_buff, vlan_proto));
441 		/* A = ntohs(A) [emitting a nop or swap16] */
442 		*insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
443 		break;
444 
445 	case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
446 	case SKF_AD_OFF + SKF_AD_NLATTR:
447 	case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
448 	case SKF_AD_OFF + SKF_AD_CPU:
449 	case SKF_AD_OFF + SKF_AD_RANDOM:
450 		/* arg1 = CTX */
451 		*insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
452 		/* arg2 = A */
453 		*insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A);
454 		/* arg3 = X */
455 		*insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X);
456 		/* Emit call(arg1=CTX, arg2=A, arg3=X) */
457 		switch (fp->k) {
458 		case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
459 			*insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset);
460 			break;
461 		case SKF_AD_OFF + SKF_AD_NLATTR:
462 			*insn = BPF_EMIT_CALL(bpf_skb_get_nlattr);
463 			break;
464 		case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
465 			*insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest);
466 			break;
467 		case SKF_AD_OFF + SKF_AD_CPU:
468 			*insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id);
469 			break;
470 		case SKF_AD_OFF + SKF_AD_RANDOM:
471 			*insn = BPF_EMIT_CALL(bpf_user_rnd_u32);
472 			bpf_user_rnd_init_once();
473 			break;
474 		}
475 		break;
476 
477 	case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
478 		/* A ^= X */
479 		*insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X);
480 		break;
481 
482 	default:
483 		/* This is just a dummy call to avoid letting the compiler
484 		 * evict __bpf_call_base() as an optimization. Placed here
485 		 * where no-one bothers.
486 		 */
487 		BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
488 		return false;
489 	}
490 
491 	*insnp = insn;
492 	return true;
493 }
494 
495 static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
496 {
497 	const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS);
498 	int size = bpf_size_to_bytes(BPF_SIZE(fp->code));
499 	bool endian = BPF_SIZE(fp->code) == BPF_H ||
500 		      BPF_SIZE(fp->code) == BPF_W;
501 	bool indirect = BPF_MODE(fp->code) == BPF_IND;
502 	const int ip_align = NET_IP_ALIGN;
503 	struct bpf_insn *insn = *insnp;
504 	int offset = fp->k;
505 
506 	if (!indirect &&
507 	    ((unaligned_ok && offset >= 0) ||
508 	     (!unaligned_ok && offset >= 0 &&
509 	      offset + ip_align >= 0 &&
510 	      (offset + ip_align) % size == 0))) {
511 		bool ldx_off_ok = offset <= S16_MAX;
512 
513 		*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
514 		if (offset)
515 			*insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
516 		*insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
517 				      size, 2 + endian + (!ldx_off_ok * 2));
518 		if (ldx_off_ok) {
519 			*insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
520 					      BPF_REG_D, offset);
521 		} else {
522 			*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D);
523 			*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset);
524 			*insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
525 					      BPF_REG_TMP, 0);
526 		}
527 		if (endian)
528 			*insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8);
529 		*insn++ = BPF_JMP_A(8);
530 	}
531 
532 	*insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
533 	*insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D);
534 	*insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H);
535 	if (!indirect) {
536 		*insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset);
537 	} else {
538 		*insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X);
539 		if (fp->k)
540 			*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset);
541 	}
542 
543 	switch (BPF_SIZE(fp->code)) {
544 	case BPF_B:
545 		*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8);
546 		break;
547 	case BPF_H:
548 		*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16);
549 		break;
550 	case BPF_W:
551 		*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32);
552 		break;
553 	default:
554 		return false;
555 	}
556 
557 	*insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2);
558 	*insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
559 	*insn   = BPF_EXIT_INSN();
560 
561 	*insnp = insn;
562 	return true;
563 }
564 
565 /**
566  *	bpf_convert_filter - convert filter program
567  *	@prog: the user passed filter program
568  *	@len: the length of the user passed filter program
569  *	@new_prog: allocated 'struct bpf_prog' or NULL
570  *	@new_len: pointer to store length of converted program
571  *	@seen_ld_abs: bool whether we've seen ld_abs/ind
572  *
573  * Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn'
574  * style extended BPF (eBPF).
575  * Conversion workflow:
576  *
577  * 1) First pass for calculating the new program length:
578  *   bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs)
579  *
580  * 2) 2nd pass to remap in two passes: 1st pass finds new
581  *    jump offsets, 2nd pass remapping:
582  *   bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs)
583  */
584 static int bpf_convert_filter(struct sock_filter *prog, int len,
585 			      struct bpf_prog *new_prog, int *new_len,
586 			      bool *seen_ld_abs)
587 {
588 	int new_flen = 0, pass = 0, target, i, stack_off;
589 	struct bpf_insn *new_insn, *first_insn = NULL;
590 	struct sock_filter *fp;
591 	int *addrs = NULL;
592 	u8 bpf_src;
593 
594 	BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
595 	BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
596 
597 	if (len <= 0 || len > BPF_MAXINSNS)
598 		return -EINVAL;
599 
600 	if (new_prog) {
601 		first_insn = new_prog->insnsi;
602 		addrs = kzalloc_objs(*addrs, len, GFP_KERNEL | __GFP_NOWARN);
603 		if (!addrs)
604 			return -ENOMEM;
605 	}
606 
607 do_pass:
608 	new_insn = first_insn;
609 	fp = prog;
610 
611 	/* Classic BPF related prologue emission. */
612 	if (new_prog) {
613 		/* Classic BPF expects A and X to be reset first. These need
614 		 * to be guaranteed to be the first two instructions.
615 		 */
616 		*new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
617 		*new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X);
618 
619 		/* All programs must keep CTX in callee saved BPF_REG_CTX.
620 		 * In eBPF case it's done by the compiler, here we need to
621 		 * do this ourself. Initial CTX is present in BPF_REG_ARG1.
622 		 */
623 		*new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1);
624 		if (*seen_ld_abs) {
625 			/* For packet access in classic BPF, cache skb->data
626 			 * in callee-saved BPF R8 and skb->len - skb->data_len
627 			 * (headlen) in BPF R9. Since classic BPF is read-only
628 			 * on CTX, we only need to cache it once.
629 			 */
630 			*new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
631 						  BPF_REG_D, BPF_REG_CTX,
632 						  offsetof(struct sk_buff, data));
633 			*new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX,
634 						  offsetof(struct sk_buff, len));
635 			*new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX,
636 						  offsetof(struct sk_buff, data_len));
637 			*new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP);
638 		}
639 	} else {
640 		new_insn += 3;
641 	}
642 
643 	for (i = 0; i < len; fp++, i++) {
644 		struct bpf_insn tmp_insns[32] = { };
645 		struct bpf_insn *insn = tmp_insns;
646 
647 		if (addrs)
648 			addrs[i] = new_insn - first_insn;
649 
650 		switch (fp->code) {
651 		/* All arithmetic insns and skb loads map as-is. */
652 		case BPF_ALU | BPF_ADD | BPF_X:
653 		case BPF_ALU | BPF_ADD | BPF_K:
654 		case BPF_ALU | BPF_SUB | BPF_X:
655 		case BPF_ALU | BPF_SUB | BPF_K:
656 		case BPF_ALU | BPF_AND | BPF_X:
657 		case BPF_ALU | BPF_AND | BPF_K:
658 		case BPF_ALU | BPF_OR | BPF_X:
659 		case BPF_ALU | BPF_OR | BPF_K:
660 		case BPF_ALU | BPF_LSH | BPF_X:
661 		case BPF_ALU | BPF_LSH | BPF_K:
662 		case BPF_ALU | BPF_RSH | BPF_X:
663 		case BPF_ALU | BPF_RSH | BPF_K:
664 		case BPF_ALU | BPF_XOR | BPF_X:
665 		case BPF_ALU | BPF_XOR | BPF_K:
666 		case BPF_ALU | BPF_MUL | BPF_X:
667 		case BPF_ALU | BPF_MUL | BPF_K:
668 		case BPF_ALU | BPF_DIV | BPF_X:
669 		case BPF_ALU | BPF_DIV | BPF_K:
670 		case BPF_ALU | BPF_MOD | BPF_X:
671 		case BPF_ALU | BPF_MOD | BPF_K:
672 		case BPF_ALU | BPF_NEG:
673 		case BPF_LD | BPF_ABS | BPF_W:
674 		case BPF_LD | BPF_ABS | BPF_H:
675 		case BPF_LD | BPF_ABS | BPF_B:
676 		case BPF_LD | BPF_IND | BPF_W:
677 		case BPF_LD | BPF_IND | BPF_H:
678 		case BPF_LD | BPF_IND | BPF_B:
679 			/* Check for overloaded BPF extension and
680 			 * directly convert it if found, otherwise
681 			 * just move on with mapping.
682 			 */
683 			if (BPF_CLASS(fp->code) == BPF_LD &&
684 			    BPF_MODE(fp->code) == BPF_ABS &&
685 			    convert_bpf_extensions(fp, &insn))
686 				break;
687 			if (BPF_CLASS(fp->code) == BPF_LD &&
688 			    convert_bpf_ld_abs(fp, &insn)) {
689 				*seen_ld_abs = true;
690 				break;
691 			}
692 
693 			if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) ||
694 			    fp->code == (BPF_ALU | BPF_MOD | BPF_X)) {
695 				*insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X);
696 				/* Error with exception code on div/mod by 0.
697 				 * For cBPF programs, this was always return 0.
698 				 */
699 				*insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2);
700 				*insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
701 				*insn++ = BPF_EXIT_INSN();
702 			}
703 
704 			*insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k);
705 			break;
706 
707 		/* Jump transformation cannot use BPF block macros
708 		 * everywhere as offset calculation and target updates
709 		 * require a bit more work than the rest, i.e. jump
710 		 * opcodes map as-is, but offsets need adjustment.
711 		 */
712 
713 #define BPF_EMIT_JMP							\
714 	do {								\
715 		const s32 off_min = S16_MIN, off_max = S16_MAX;		\
716 		s32 off;						\
717 									\
718 		if (target >= len || target < 0)			\
719 			goto err;					\
720 		off = addrs ? addrs[target] - addrs[i] - 1 : 0;		\
721 		/* Adjust pc relative offset for 2nd or 3rd insn. */	\
722 		off -= insn - tmp_insns;				\
723 		/* Reject anything not fitting into insn->off. */	\
724 		if (off < off_min || off > off_max)			\
725 			goto err;					\
726 		insn->off = off;					\
727 	} while (0)
728 
729 		case BPF_JMP | BPF_JA:
730 			target = i + fp->k + 1;
731 			insn->code = fp->code;
732 			BPF_EMIT_JMP;
733 			break;
734 
735 		case BPF_JMP | BPF_JEQ | BPF_K:
736 		case BPF_JMP | BPF_JEQ | BPF_X:
737 		case BPF_JMP | BPF_JSET | BPF_K:
738 		case BPF_JMP | BPF_JSET | BPF_X:
739 		case BPF_JMP | BPF_JGT | BPF_K:
740 		case BPF_JMP | BPF_JGT | BPF_X:
741 		case BPF_JMP | BPF_JGE | BPF_K:
742 		case BPF_JMP | BPF_JGE | BPF_X:
743 			if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
744 				/* BPF immediates are signed, zero extend
745 				 * immediate into tmp register and use it
746 				 * in compare insn.
747 				 */
748 				*insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k);
749 
750 				insn->dst_reg = BPF_REG_A;
751 				insn->src_reg = BPF_REG_TMP;
752 				bpf_src = BPF_X;
753 			} else {
754 				insn->dst_reg = BPF_REG_A;
755 				insn->imm = fp->k;
756 				bpf_src = BPF_SRC(fp->code);
757 				insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0;
758 			}
759 
760 			/* Common case where 'jump_false' is next insn. */
761 			if (fp->jf == 0) {
762 				insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
763 				target = i + fp->jt + 1;
764 				BPF_EMIT_JMP;
765 				break;
766 			}
767 
768 			/* Convert some jumps when 'jump_true' is next insn. */
769 			if (fp->jt == 0) {
770 				switch (BPF_OP(fp->code)) {
771 				case BPF_JEQ:
772 					insn->code = BPF_JMP | BPF_JNE | bpf_src;
773 					break;
774 				case BPF_JGT:
775 					insn->code = BPF_JMP | BPF_JLE | bpf_src;
776 					break;
777 				case BPF_JGE:
778 					insn->code = BPF_JMP | BPF_JLT | bpf_src;
779 					break;
780 				default:
781 					goto jmp_rest;
782 				}
783 
784 				target = i + fp->jf + 1;
785 				BPF_EMIT_JMP;
786 				break;
787 			}
788 jmp_rest:
789 			/* Other jumps are mapped into two insns: Jxx and JA. */
790 			target = i + fp->jt + 1;
791 			insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
792 			BPF_EMIT_JMP;
793 			insn++;
794 
795 			insn->code = BPF_JMP | BPF_JA;
796 			target = i + fp->jf + 1;
797 			BPF_EMIT_JMP;
798 			break;
799 
800 		/* ldxb 4 * ([14] & 0xf) is remapped into 6 insns. */
801 		case BPF_LDX | BPF_MSH | BPF_B: {
802 			struct sock_filter tmp = {
803 				.code	= BPF_LD | BPF_ABS | BPF_B,
804 				.k	= fp->k,
805 			};
806 
807 			*seen_ld_abs = true;
808 
809 			/* X = A */
810 			*insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
811 			/* A = BPF_R0 = *(u8 *) (skb->data + K) */
812 			convert_bpf_ld_abs(&tmp, &insn);
813 			insn++;
814 			/* A &= 0xf */
815 			*insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf);
816 			/* A <<= 2 */
817 			*insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2);
818 			/* tmp = X */
819 			*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X);
820 			/* X = A */
821 			*insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
822 			/* A = tmp */
823 			*insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP);
824 			break;
825 		}
826 		/* RET_K is remapped into 2 insns. RET_A case doesn't need an
827 		 * extra mov as BPF_REG_0 is already mapped into BPF_REG_A.
828 		 */
829 		case BPF_RET | BPF_A:
830 		case BPF_RET | BPF_K:
831 			if (BPF_RVAL(fp->code) == BPF_K)
832 				*insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0,
833 							0, fp->k);
834 			*insn = BPF_EXIT_INSN();
835 			break;
836 
837 		/* Store to stack. */
838 		case BPF_ST:
839 		case BPF_STX:
840 			stack_off = fp->k * 4  + 4;
841 			*insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) ==
842 					    BPF_ST ? BPF_REG_A : BPF_REG_X,
843 					    -stack_off);
844 			/* check_load_and_stores() verifies that classic BPF can
845 			 * load from stack only after write, so tracking
846 			 * stack_depth for ST|STX insns is enough
847 			 */
848 			if (new_prog && new_prog->aux->stack_depth < stack_off)
849 				new_prog->aux->stack_depth = stack_off;
850 			break;
851 
852 		/* Load from stack. */
853 		case BPF_LD | BPF_MEM:
854 		case BPF_LDX | BPF_MEM:
855 			stack_off = fp->k * 4  + 4;
856 			*insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD  ?
857 					    BPF_REG_A : BPF_REG_X, BPF_REG_FP,
858 					    -stack_off);
859 			break;
860 
861 		/* A = K or X = K */
862 		case BPF_LD | BPF_IMM:
863 		case BPF_LDX | BPF_IMM:
864 			*insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ?
865 					      BPF_REG_A : BPF_REG_X, fp->k);
866 			break;
867 
868 		/* X = A */
869 		case BPF_MISC | BPF_TAX:
870 			*insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
871 			break;
872 
873 		/* A = X */
874 		case BPF_MISC | BPF_TXA:
875 			*insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X);
876 			break;
877 
878 		/* A = skb->len or X = skb->len */
879 		case BPF_LD | BPF_W | BPF_LEN:
880 		case BPF_LDX | BPF_W | BPF_LEN:
881 			*insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
882 					    BPF_REG_A : BPF_REG_X, BPF_REG_CTX,
883 					    offsetof(struct sk_buff, len));
884 			break;
885 
886 		/* Access seccomp_data fields. */
887 		case BPF_LDX | BPF_ABS | BPF_W:
888 			/* A = *(u32 *) (ctx + K) */
889 			*insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k);
890 			break;
891 
892 		/* Unknown instruction. */
893 		default:
894 			goto err;
895 		}
896 
897 		insn++;
898 		if (new_prog)
899 			memcpy(new_insn, tmp_insns,
900 			       sizeof(*insn) * (insn - tmp_insns));
901 		new_insn += insn - tmp_insns;
902 	}
903 
904 	if (!new_prog) {
905 		/* Only calculating new length. */
906 		*new_len = new_insn - first_insn;
907 		if (*seen_ld_abs)
908 			*new_len += 4; /* Prologue bits. */
909 		return 0;
910 	}
911 
912 	pass++;
913 	if (new_flen != new_insn - first_insn) {
914 		new_flen = new_insn - first_insn;
915 		if (pass > 2)
916 			goto err;
917 		goto do_pass;
918 	}
919 
920 	kfree(addrs);
921 	BUG_ON(*new_len != new_flen);
922 	return 0;
923 err:
924 	kfree(addrs);
925 	return -EINVAL;
926 }
927 
928 /* Security:
929  *
930  * As we dont want to clear mem[] array for each packet going through
931  * __bpf_prog_run(), we check that filter loaded by user never try to read
932  * a cell if not previously written, and we check all branches to be sure
933  * a malicious user doesn't try to abuse us.
934  */
935 static int check_load_and_stores(const struct sock_filter *filter, int flen)
936 {
937 	u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */
938 	int pc, ret = 0;
939 
940 	BUILD_BUG_ON(BPF_MEMWORDS > 16);
941 
942 	masks = kmalloc_objs(*masks, flen);
943 	if (!masks)
944 		return -ENOMEM;
945 
946 	memset(masks, 0xff, flen * sizeof(*masks));
947 
948 	for (pc = 0; pc < flen; pc++) {
949 		memvalid &= masks[pc];
950 
951 		switch (filter[pc].code) {
952 		case BPF_ST:
953 		case BPF_STX:
954 			memvalid |= (1 << filter[pc].k);
955 			break;
956 		case BPF_LD | BPF_MEM:
957 		case BPF_LDX | BPF_MEM:
958 			if (!(memvalid & (1 << filter[pc].k))) {
959 				ret = -EINVAL;
960 				goto error;
961 			}
962 			break;
963 		case BPF_JMP | BPF_JA:
964 			/* A jump must set masks on target */
965 			masks[pc + 1 + filter[pc].k] &= memvalid;
966 			memvalid = ~0;
967 			break;
968 		case BPF_JMP | BPF_JEQ | BPF_K:
969 		case BPF_JMP | BPF_JEQ | BPF_X:
970 		case BPF_JMP | BPF_JGE | BPF_K:
971 		case BPF_JMP | BPF_JGE | BPF_X:
972 		case BPF_JMP | BPF_JGT | BPF_K:
973 		case BPF_JMP | BPF_JGT | BPF_X:
974 		case BPF_JMP | BPF_JSET | BPF_K:
975 		case BPF_JMP | BPF_JSET | BPF_X:
976 			/* A jump must set masks on targets */
977 			masks[pc + 1 + filter[pc].jt] &= memvalid;
978 			masks[pc + 1 + filter[pc].jf] &= memvalid;
979 			memvalid = ~0;
980 			break;
981 		}
982 	}
983 error:
984 	kfree(masks);
985 	return ret;
986 }
987 
988 static bool chk_code_allowed(u16 code_to_probe)
989 {
990 	static const bool codes[] = {
991 		/* 32 bit ALU operations */
992 		[BPF_ALU | BPF_ADD | BPF_K] = true,
993 		[BPF_ALU | BPF_ADD | BPF_X] = true,
994 		[BPF_ALU | BPF_SUB | BPF_K] = true,
995 		[BPF_ALU | BPF_SUB | BPF_X] = true,
996 		[BPF_ALU | BPF_MUL | BPF_K] = true,
997 		[BPF_ALU | BPF_MUL | BPF_X] = true,
998 		[BPF_ALU | BPF_DIV | BPF_K] = true,
999 		[BPF_ALU | BPF_DIV | BPF_X] = true,
1000 		[BPF_ALU | BPF_MOD | BPF_K] = true,
1001 		[BPF_ALU | BPF_MOD | BPF_X] = true,
1002 		[BPF_ALU | BPF_AND | BPF_K] = true,
1003 		[BPF_ALU | BPF_AND | BPF_X] = true,
1004 		[BPF_ALU | BPF_OR | BPF_K] = true,
1005 		[BPF_ALU | BPF_OR | BPF_X] = true,
1006 		[BPF_ALU | BPF_XOR | BPF_K] = true,
1007 		[BPF_ALU | BPF_XOR | BPF_X] = true,
1008 		[BPF_ALU | BPF_LSH | BPF_K] = true,
1009 		[BPF_ALU | BPF_LSH | BPF_X] = true,
1010 		[BPF_ALU | BPF_RSH | BPF_K] = true,
1011 		[BPF_ALU | BPF_RSH | BPF_X] = true,
1012 		[BPF_ALU | BPF_NEG] = true,
1013 		/* Load instructions */
1014 		[BPF_LD | BPF_W | BPF_ABS] = true,
1015 		[BPF_LD | BPF_H | BPF_ABS] = true,
1016 		[BPF_LD | BPF_B | BPF_ABS] = true,
1017 		[BPF_LD | BPF_W | BPF_LEN] = true,
1018 		[BPF_LD | BPF_W | BPF_IND] = true,
1019 		[BPF_LD | BPF_H | BPF_IND] = true,
1020 		[BPF_LD | BPF_B | BPF_IND] = true,
1021 		[BPF_LD | BPF_IMM] = true,
1022 		[BPF_LD | BPF_MEM] = true,
1023 		[BPF_LDX | BPF_W | BPF_LEN] = true,
1024 		[BPF_LDX | BPF_B | BPF_MSH] = true,
1025 		[BPF_LDX | BPF_IMM] = true,
1026 		[BPF_LDX | BPF_MEM] = true,
1027 		/* Store instructions */
1028 		[BPF_ST] = true,
1029 		[BPF_STX] = true,
1030 		/* Misc instructions */
1031 		[BPF_MISC | BPF_TAX] = true,
1032 		[BPF_MISC | BPF_TXA] = true,
1033 		/* Return instructions */
1034 		[BPF_RET | BPF_K] = true,
1035 		[BPF_RET | BPF_A] = true,
1036 		/* Jump instructions */
1037 		[BPF_JMP | BPF_JA] = true,
1038 		[BPF_JMP | BPF_JEQ | BPF_K] = true,
1039 		[BPF_JMP | BPF_JEQ | BPF_X] = true,
1040 		[BPF_JMP | BPF_JGE | BPF_K] = true,
1041 		[BPF_JMP | BPF_JGE | BPF_X] = true,
1042 		[BPF_JMP | BPF_JGT | BPF_K] = true,
1043 		[BPF_JMP | BPF_JGT | BPF_X] = true,
1044 		[BPF_JMP | BPF_JSET | BPF_K] = true,
1045 		[BPF_JMP | BPF_JSET | BPF_X] = true,
1046 	};
1047 
1048 	if (code_to_probe >= ARRAY_SIZE(codes))
1049 		return false;
1050 
1051 	return codes[code_to_probe];
1052 }
1053 
1054 static bool bpf_check_basics_ok(const struct sock_filter *filter,
1055 				unsigned int flen)
1056 {
1057 	if (filter == NULL)
1058 		return false;
1059 	if (flen == 0 || flen > BPF_MAXINSNS)
1060 		return false;
1061 
1062 	return true;
1063 }
1064 
1065 /**
1066  *	bpf_check_classic - verify socket filter code
1067  *	@filter: filter to verify
1068  *	@flen: length of filter
1069  *
1070  * Check the user's filter code. If we let some ugly
1071  * filter code slip through kaboom! The filter must contain
1072  * no references or jumps that are out of range, no illegal
1073  * instructions, and must end with a RET instruction.
1074  *
1075  * All jumps are forward as they are not signed.
1076  *
1077  * Returns 0 if the rule set is legal or -EINVAL if not.
1078  */
1079 static int bpf_check_classic(const struct sock_filter *filter,
1080 			     unsigned int flen)
1081 {
1082 	bool anc_found;
1083 	int pc;
1084 
1085 	/* Check the filter code now */
1086 	for (pc = 0; pc < flen; pc++) {
1087 		const struct sock_filter *ftest = &filter[pc];
1088 
1089 		/* May we actually operate on this code? */
1090 		if (!chk_code_allowed(ftest->code))
1091 			return -EINVAL;
1092 
1093 		/* Some instructions need special checks */
1094 		switch (ftest->code) {
1095 		case BPF_ALU | BPF_DIV | BPF_K:
1096 		case BPF_ALU | BPF_MOD | BPF_K:
1097 			/* Check for division by zero */
1098 			if (ftest->k == 0)
1099 				return -EINVAL;
1100 			break;
1101 		case BPF_ALU | BPF_LSH | BPF_K:
1102 		case BPF_ALU | BPF_RSH | BPF_K:
1103 			if (ftest->k >= 32)
1104 				return -EINVAL;
1105 			break;
1106 		case BPF_LD | BPF_MEM:
1107 		case BPF_LDX | BPF_MEM:
1108 		case BPF_ST:
1109 		case BPF_STX:
1110 			/* Check for invalid memory addresses */
1111 			if (ftest->k >= BPF_MEMWORDS)
1112 				return -EINVAL;
1113 			break;
1114 		case BPF_JMP | BPF_JA:
1115 			/* Note, the large ftest->k might cause loops.
1116 			 * Compare this with conditional jumps below,
1117 			 * where offsets are limited. --ANK (981016)
1118 			 */
1119 			if (ftest->k >= (unsigned int)(flen - pc - 1))
1120 				return -EINVAL;
1121 			break;
1122 		case BPF_JMP | BPF_JEQ | BPF_K:
1123 		case BPF_JMP | BPF_JEQ | BPF_X:
1124 		case BPF_JMP | BPF_JGE | BPF_K:
1125 		case BPF_JMP | BPF_JGE | BPF_X:
1126 		case BPF_JMP | BPF_JGT | BPF_K:
1127 		case BPF_JMP | BPF_JGT | BPF_X:
1128 		case BPF_JMP | BPF_JSET | BPF_K:
1129 		case BPF_JMP | BPF_JSET | BPF_X:
1130 			/* Both conditionals must be safe */
1131 			if (pc + ftest->jt + 1 >= flen ||
1132 			    pc + ftest->jf + 1 >= flen)
1133 				return -EINVAL;
1134 			break;
1135 		case BPF_LD | BPF_W | BPF_ABS:
1136 		case BPF_LD | BPF_H | BPF_ABS:
1137 		case BPF_LD | BPF_B | BPF_ABS:
1138 			anc_found = false;
1139 			if (bpf_anc_helper(ftest) & BPF_ANC)
1140 				anc_found = true;
1141 			/* Ancillary operation unknown or unsupported */
1142 			if (anc_found == false && ftest->k >= SKF_AD_OFF)
1143 				return -EINVAL;
1144 		}
1145 	}
1146 
1147 	/* Last instruction must be a RET code */
1148 	switch (filter[flen - 1].code) {
1149 	case BPF_RET | BPF_K:
1150 	case BPF_RET | BPF_A:
1151 		return check_load_and_stores(filter, flen);
1152 	}
1153 
1154 	return -EINVAL;
1155 }
1156 
1157 static int bpf_prog_store_orig_filter(struct bpf_prog *fp,
1158 				      const struct sock_fprog *fprog)
1159 {
1160 	unsigned int fsize = bpf_classic_proglen(fprog);
1161 	struct sock_fprog_kern *fkprog;
1162 
1163 	fp->orig_prog = kmalloc_obj(*fkprog);
1164 	if (!fp->orig_prog)
1165 		return -ENOMEM;
1166 
1167 	fkprog = fp->orig_prog;
1168 	fkprog->len = fprog->len;
1169 
1170 	fkprog->filter = kmemdup(fp->insns, fsize,
1171 				 GFP_KERNEL | __GFP_NOWARN);
1172 	if (!fkprog->filter) {
1173 		kfree(fp->orig_prog);
1174 		return -ENOMEM;
1175 	}
1176 
1177 	return 0;
1178 }
1179 
1180 static void bpf_release_orig_filter(struct bpf_prog *fp)
1181 {
1182 	struct sock_fprog_kern *fprog = fp->orig_prog;
1183 
1184 	if (fprog) {
1185 		kfree(fprog->filter);
1186 		kfree(fprog);
1187 	}
1188 }
1189 
1190 static void __bpf_prog_release(struct bpf_prog *prog)
1191 {
1192 	if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) {
1193 		bpf_prog_put(prog);
1194 	} else {
1195 		bpf_release_orig_filter(prog);
1196 		bpf_prog_free(prog);
1197 	}
1198 }
1199 
1200 static void __sk_filter_release(struct sk_filter *fp)
1201 {
1202 	__bpf_prog_release(fp->prog);
1203 	kfree(fp);
1204 }
1205 
1206 /**
1207  * 	sk_filter_release_rcu - Release a socket filter by rcu_head
1208  *	@rcu: rcu_head that contains the sk_filter to free
1209  */
1210 static void sk_filter_release_rcu(struct rcu_head *rcu)
1211 {
1212 	struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);
1213 
1214 	__sk_filter_release(fp);
1215 }
1216 
1217 /**
1218  *	sk_filter_release - release a socket filter
1219  *	@fp: filter to remove
1220  *
1221  *	Remove a filter from a socket and release its resources.
1222  */
1223 static void sk_filter_release(struct sk_filter *fp)
1224 {
1225 	if (refcount_dec_and_test(&fp->refcnt))
1226 		call_rcu(&fp->rcu, sk_filter_release_rcu);
1227 }
1228 
1229 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1230 {
1231 	u32 filter_size = bpf_prog_size(fp->prog->len);
1232 
1233 	atomic_sub(filter_size, &sk->sk_omem_alloc);
1234 	sk_filter_release(fp);
1235 }
1236 
1237 /* try to charge the socket memory if there is space available
1238  * return true on success
1239  */
1240 static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1241 {
1242 	int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
1243 	u32 filter_size = bpf_prog_size(fp->prog->len);
1244 
1245 	/* same check as in sock_kmalloc() */
1246 	if (filter_size <= optmem_max &&
1247 	    atomic_read(&sk->sk_omem_alloc) + filter_size < optmem_max) {
1248 		atomic_add(filter_size, &sk->sk_omem_alloc);
1249 		return true;
1250 	}
1251 	return false;
1252 }
1253 
1254 bool sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1255 {
1256 	if (!refcount_inc_not_zero(&fp->refcnt))
1257 		return false;
1258 
1259 	if (!__sk_filter_charge(sk, fp)) {
1260 		sk_filter_release(fp);
1261 		return false;
1262 	}
1263 	return true;
1264 }
1265 
1266 static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
1267 {
1268 	struct sock_filter *old_prog;
1269 	struct bpf_prog *old_fp;
1270 	int err, new_len, old_len = fp->len;
1271 	bool seen_ld_abs = false;
1272 
1273 	/* We are free to overwrite insns et al right here as it won't be used at
1274 	 * this point in time anymore internally after the migration to the eBPF
1275 	 * instruction representation.
1276 	 */
1277 	BUILD_BUG_ON(sizeof(struct sock_filter) !=
1278 		     sizeof(struct bpf_insn));
1279 
1280 	/* Conversion cannot happen on overlapping memory areas,
1281 	 * so we need to keep the user BPF around until the 2nd
1282 	 * pass. At this time, the user BPF is stored in fp->insns.
1283 	 */
1284 	old_prog = kmemdup_array(fp->insns, old_len, sizeof(struct sock_filter),
1285 				 GFP_KERNEL | __GFP_NOWARN);
1286 	if (!old_prog) {
1287 		err = -ENOMEM;
1288 		goto out_err;
1289 	}
1290 
1291 	/* 1st pass: calculate the new program length. */
1292 	err = bpf_convert_filter(old_prog, old_len, NULL, &new_len,
1293 				 &seen_ld_abs);
1294 	if (err)
1295 		goto out_err_free;
1296 
1297 	/* Expand fp for appending the new filter representation. */
1298 	old_fp = fp;
1299 	fp = bpf_prog_realloc(old_fp, bpf_prog_size(new_len), 0);
1300 	if (!fp) {
1301 		/* The old_fp is still around in case we couldn't
1302 		 * allocate new memory, so uncharge on that one.
1303 		 */
1304 		fp = old_fp;
1305 		err = -ENOMEM;
1306 		goto out_err_free;
1307 	}
1308 
1309 	fp->len = new_len;
1310 
1311 	/* 2nd pass: remap sock_filter insns into bpf_insn insns. */
1312 	err = bpf_convert_filter(old_prog, old_len, fp, &new_len,
1313 				 &seen_ld_abs);
1314 	if (err)
1315 		/* 2nd bpf_convert_filter() can fail only if it fails
1316 		 * to allocate memory, remapping must succeed. Note,
1317 		 * that at this time old_fp has already been released
1318 		 * by krealloc().
1319 		 */
1320 		goto out_err_free;
1321 
1322 	fp = bpf_prog_select_runtime(fp, &err);
1323 	if (err)
1324 		goto out_err_free;
1325 
1326 	kfree(old_prog);
1327 	return fp;
1328 
1329 out_err_free:
1330 	kfree(old_prog);
1331 out_err:
1332 	__bpf_prog_release(fp);
1333 	return ERR_PTR(err);
1334 }
1335 
1336 static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp,
1337 					   bpf_aux_classic_check_t trans)
1338 {
1339 	int err;
1340 
1341 	fp->bpf_func = NULL;
1342 	fp->jited = 0;
1343 
1344 	err = bpf_check_classic(fp->insns, fp->len);
1345 	if (err) {
1346 		__bpf_prog_release(fp);
1347 		return ERR_PTR(err);
1348 	}
1349 
1350 	/* There might be additional checks and transformations
1351 	 * needed on classic filters, f.e. in case of seccomp.
1352 	 */
1353 	if (trans) {
1354 		err = trans(fp->insns, fp->len);
1355 		if (err) {
1356 			__bpf_prog_release(fp);
1357 			return ERR_PTR(err);
1358 		}
1359 	}
1360 
1361 	/* Probe if we can JIT compile the filter and if so, do
1362 	 * the compilation of the filter.
1363 	 */
1364 	bpf_jit_compile(fp);
1365 
1366 	/* JIT compiler couldn't process this filter, so do the eBPF translation
1367 	 * for the optimized interpreter.
1368 	 */
1369 	if (!fp->jited)
1370 		fp = bpf_migrate_filter(fp);
1371 
1372 	return fp;
1373 }
1374 
1375 /**
1376  *	bpf_prog_create - create an unattached filter
1377  *	@pfp: the unattached filter that is created
1378  *	@fprog: the filter program
1379  *
1380  * Create a filter independent of any socket. We first run some
1381  * sanity checks on it to make sure it does not explode on us later.
1382  * If an error occurs or there is insufficient memory for the filter
1383  * a negative errno code is returned. On success the return is zero.
1384  */
1385 int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
1386 {
1387 	unsigned int fsize = bpf_classic_proglen(fprog);
1388 	struct bpf_prog *fp;
1389 
1390 	/* Make sure new filter is there and in the right amounts. */
1391 	if (!bpf_check_basics_ok(fprog->filter, fprog->len))
1392 		return -EINVAL;
1393 
1394 	fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1395 	if (!fp)
1396 		return -ENOMEM;
1397 
1398 	memcpy(fp->insns, fprog->filter, fsize);
1399 
1400 	fp->len = fprog->len;
1401 	/* Since unattached filters are not copied back to user
1402 	 * space through sk_get_filter(), we do not need to hold
1403 	 * a copy here, and can spare us the work.
1404 	 */
1405 	fp->orig_prog = NULL;
1406 
1407 	/* bpf_prepare_filter() already takes care of freeing
1408 	 * memory in case something goes wrong.
1409 	 */
1410 	fp = bpf_prepare_filter(fp, NULL);
1411 	if (IS_ERR(fp))
1412 		return PTR_ERR(fp);
1413 
1414 	*pfp = fp;
1415 	return 0;
1416 }
1417 EXPORT_SYMBOL_GPL(bpf_prog_create);
1418 
1419 /**
1420  *	bpf_prog_create_from_user - create an unattached filter from user buffer
1421  *	@pfp: the unattached filter that is created
1422  *	@fprog: the filter program
1423  *	@trans: post-classic verifier transformation handler
1424  *	@save_orig: save classic BPF program
1425  *
1426  * This function effectively does the same as bpf_prog_create(), only
1427  * that it builds up its insns buffer from user space provided buffer.
1428  * It also allows for passing a bpf_aux_classic_check_t handler.
1429  */
1430 int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
1431 			      bpf_aux_classic_check_t trans, bool save_orig)
1432 {
1433 	unsigned int fsize = bpf_classic_proglen(fprog);
1434 	struct bpf_prog *fp;
1435 	int err;
1436 
1437 	/* Make sure new filter is there and in the right amounts. */
1438 	if (!bpf_check_basics_ok(fprog->filter, fprog->len))
1439 		return -EINVAL;
1440 
1441 	fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1442 	if (!fp)
1443 		return -ENOMEM;
1444 
1445 	if (copy_from_user(fp->insns, fprog->filter, fsize)) {
1446 		__bpf_prog_free(fp);
1447 		return -EFAULT;
1448 	}
1449 
1450 	fp->len = fprog->len;
1451 	fp->orig_prog = NULL;
1452 
1453 	if (save_orig) {
1454 		err = bpf_prog_store_orig_filter(fp, fprog);
1455 		if (err) {
1456 			__bpf_prog_free(fp);
1457 			return -ENOMEM;
1458 		}
1459 	}
1460 
1461 	/* bpf_prepare_filter() already takes care of freeing
1462 	 * memory in case something goes wrong.
1463 	 */
1464 	fp = bpf_prepare_filter(fp, trans);
1465 	if (IS_ERR(fp))
1466 		return PTR_ERR(fp);
1467 
1468 	*pfp = fp;
1469 	return 0;
1470 }
1471 EXPORT_SYMBOL_GPL(bpf_prog_create_from_user);
1472 
1473 void bpf_prog_destroy(struct bpf_prog *fp)
1474 {
1475 	__bpf_prog_release(fp);
1476 }
1477 EXPORT_SYMBOL_GPL(bpf_prog_destroy);
1478 
1479 static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk)
1480 {
1481 	struct sk_filter *fp, *old_fp;
1482 
1483 	fp = kmalloc_obj(*fp);
1484 	if (!fp)
1485 		return -ENOMEM;
1486 
1487 	fp->prog = prog;
1488 
1489 	if (!__sk_filter_charge(sk, fp)) {
1490 		kfree(fp);
1491 		return -ENOMEM;
1492 	}
1493 	refcount_set(&fp->refcnt, 1);
1494 
1495 	old_fp = rcu_dereference_protected(sk->sk_filter,
1496 					   lockdep_sock_is_held(sk));
1497 	rcu_assign_pointer(sk->sk_filter, fp);
1498 
1499 	if (old_fp)
1500 		sk_filter_uncharge(sk, old_fp);
1501 
1502 	return 0;
1503 }
1504 
1505 static
1506 struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
1507 {
1508 	unsigned int fsize = bpf_classic_proglen(fprog);
1509 	struct bpf_prog *prog;
1510 	int err;
1511 
1512 	if (sock_flag(sk, SOCK_FILTER_LOCKED))
1513 		return ERR_PTR(-EPERM);
1514 
1515 	/* Make sure new filter is there and in the right amounts. */
1516 	if (!bpf_check_basics_ok(fprog->filter, fprog->len))
1517 		return ERR_PTR(-EINVAL);
1518 
1519 	prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
1520 	if (!prog)
1521 		return ERR_PTR(-ENOMEM);
1522 
1523 	if (copy_from_user(prog->insns, fprog->filter, fsize)) {
1524 		__bpf_prog_free(prog);
1525 		return ERR_PTR(-EFAULT);
1526 	}
1527 
1528 	prog->len = fprog->len;
1529 
1530 	err = bpf_prog_store_orig_filter(prog, fprog);
1531 	if (err) {
1532 		__bpf_prog_free(prog);
1533 		return ERR_PTR(-ENOMEM);
1534 	}
1535 
1536 	/* bpf_prepare_filter() already takes care of freeing
1537 	 * memory in case something goes wrong.
1538 	 */
1539 	return bpf_prepare_filter(prog, NULL);
1540 }
1541 
1542 /**
1543  *	sk_attach_filter - attach a socket filter
1544  *	@fprog: the filter program
1545  *	@sk: the socket to use
1546  *
1547  * Attach the user's filter code. We first run some sanity checks on
1548  * it to make sure it does not explode on us later. If an error
1549  * occurs or there is insufficient memory for the filter a negative
1550  * errno code is returned. On success the return is zero.
1551  */
1552 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1553 {
1554 	struct bpf_prog *prog = __get_filter(fprog, sk);
1555 	int err;
1556 
1557 	if (IS_ERR(prog))
1558 		return PTR_ERR(prog);
1559 
1560 	err = __sk_attach_prog(prog, sk);
1561 	if (err < 0) {
1562 		__bpf_prog_release(prog);
1563 		return err;
1564 	}
1565 
1566 	return 0;
1567 }
1568 EXPORT_SYMBOL_GPL(sk_attach_filter);
1569 
1570 int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk)
1571 {
1572 	struct bpf_prog *prog = __get_filter(fprog, sk);
1573 	int err, optmem_max;
1574 
1575 	if (IS_ERR(prog))
1576 		return PTR_ERR(prog);
1577 
1578 	optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
1579 	if (bpf_prog_size(prog->len) > optmem_max)
1580 		err = -ENOMEM;
1581 	else
1582 		err = reuseport_attach_prog(sk, prog);
1583 
1584 	if (err)
1585 		__bpf_prog_release(prog);
1586 
1587 	return err;
1588 }
1589 
1590 static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk)
1591 {
1592 	if (sock_flag(sk, SOCK_FILTER_LOCKED))
1593 		return ERR_PTR(-EPERM);
1594 
1595 	return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
1596 }
1597 
1598 int sk_attach_bpf(u32 ufd, struct sock *sk)
1599 {
1600 	struct bpf_prog *prog = __get_bpf(ufd, sk);
1601 	int err;
1602 
1603 	if (IS_ERR(prog))
1604 		return PTR_ERR(prog);
1605 
1606 	err = __sk_attach_prog(prog, sk);
1607 	if (err < 0) {
1608 		bpf_prog_put(prog);
1609 		return err;
1610 	}
1611 
1612 	return 0;
1613 }
1614 
1615 int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
1616 {
1617 	struct bpf_prog *prog;
1618 	int err, optmem_max;
1619 
1620 	if (sock_flag(sk, SOCK_FILTER_LOCKED))
1621 		return -EPERM;
1622 
1623 	prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
1624 	if (PTR_ERR(prog) == -EINVAL)
1625 		prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SK_REUSEPORT);
1626 	if (IS_ERR(prog))
1627 		return PTR_ERR(prog);
1628 
1629 	if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) {
1630 		/* Like other non BPF_PROG_TYPE_SOCKET_FILTER
1631 		 * bpf prog (e.g. sockmap).  It depends on the
1632 		 * limitation imposed by bpf_prog_load().
1633 		 * Hence, sysctl_optmem_max is not checked.
1634 		 */
1635 		if ((sk->sk_type != SOCK_STREAM &&
1636 		     sk->sk_type != SOCK_DGRAM) ||
1637 		    (sk->sk_protocol != IPPROTO_UDP &&
1638 		     sk->sk_protocol != IPPROTO_TCP) ||
1639 		    (sk->sk_family != AF_INET &&
1640 		     sk->sk_family != AF_INET6)) {
1641 			err = -ENOTSUPP;
1642 			goto err_prog_put;
1643 		}
1644 	} else {
1645 		/* BPF_PROG_TYPE_SOCKET_FILTER */
1646 		optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
1647 		if (bpf_prog_size(prog->len) > optmem_max) {
1648 			err = -ENOMEM;
1649 			goto err_prog_put;
1650 		}
1651 	}
1652 
1653 	err = reuseport_attach_prog(sk, prog);
1654 err_prog_put:
1655 	if (err)
1656 		bpf_prog_put(prog);
1657 
1658 	return err;
1659 }
1660 
1661 static void sk_reuseport_prog_free_rcu(struct rcu_head *rcu)
1662 {
1663 	struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1664 	struct bpf_prog *prog = aux->prog;
1665 
1666 	bpf_release_orig_filter(prog);
1667 	bpf_prog_free(prog);
1668 }
1669 
1670 void sk_reuseport_prog_free(struct bpf_prog *prog)
1671 {
1672 	if (!prog)
1673 		return;
1674 
1675 	if (bpf_prog_was_classic(prog))
1676 		call_rcu(&prog->aux->rcu, sk_reuseport_prog_free_rcu);
1677 	else
1678 		bpf_prog_put(prog);
1679 }
1680 
1681 static inline int __bpf_try_make_writable(struct sk_buff *skb,
1682 					  unsigned int write_len)
1683 {
1684 #ifdef CONFIG_DEBUG_NET
1685 	/* Avoid a splat in pskb_may_pull_reason() */
1686 	if (write_len > INT_MAX)
1687 		return -EINVAL;
1688 #endif
1689 	return skb_ensure_writable(skb, write_len);
1690 }
1691 
1692 static inline int bpf_try_make_writable(struct sk_buff *skb,
1693 					unsigned int write_len)
1694 {
1695 	int err = __bpf_try_make_writable(skb, write_len);
1696 
1697 	bpf_compute_data_pointers(skb);
1698 	return err;
1699 }
1700 
1701 static int bpf_try_make_head_writable(struct sk_buff *skb)
1702 {
1703 	return bpf_try_make_writable(skb, skb_headlen(skb));
1704 }
1705 
1706 static inline void bpf_push_mac_rcsum(struct sk_buff *skb)
1707 {
1708 	if (skb_at_tc_ingress(skb))
1709 		skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len);
1710 }
1711 
1712 static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
1713 {
1714 	if (skb_at_tc_ingress(skb))
1715 		skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
1716 }
1717 
1718 /*
1719  * The source of a write through a dynptr may be a slice of the same skb.
1720  * Making the skb writable can move its head: follow the move.
1721  */
1722 static int bpf_try_make_writable_from(struct sk_buff *skb, u32 write_len,
1723 				      const void **from)
1724 {
1725 	const void *head = skb->head;
1726 	bool in_head = *from >= head &&
1727 		       *from < (const void *)skb_end_pointer(skb);
1728 	int err;
1729 
1730 	err = bpf_try_make_writable(skb, write_len);
1731 	if (!err && in_head)
1732 		*from = skb->head + (*from - head);
1733 	return err;
1734 }
1735 
1736 BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
1737 	   const void *, from, u32, len, u64, flags)
1738 {
1739 	void *ptr;
1740 
1741 	if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH)))
1742 		return -EINVAL;
1743 	if (unlikely(offset > INT_MAX))
1744 		return -EFAULT;
1745 	if (unlikely(bpf_try_make_writable_from(skb, offset + len, &from)))
1746 		return -EFAULT;
1747 
1748 	ptr = skb->data + offset;
1749 	if (flags & BPF_F_RECOMPUTE_CSUM)
1750 		__skb_postpull_rcsum(skb, ptr, len, offset);
1751 
1752 	/* through a dynptr the source may be an overlapping slice of the skb */
1753 	memmove(ptr, from, len);
1754 
1755 	if (flags & BPF_F_RECOMPUTE_CSUM)
1756 		__skb_postpush_rcsum(skb, ptr, len, offset);
1757 	if (flags & BPF_F_INVALIDATE_HASH)
1758 		skb_clear_hash(skb);
1759 
1760 	return 0;
1761 }
1762 
1763 static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
1764 	.func		= bpf_skb_store_bytes,
1765 	.gpl_only	= false,
1766 	.ret_type	= RET_INTEGER,
1767 	.arg1_type	= ARG_PTR_TO_CTX,
1768 	.arg2_type	= ARG_ANYTHING,
1769 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
1770 	.arg4_type	= ARG_MEM_SIZE,
1771 	.arg5_type	= ARG_ANYTHING,
1772 };
1773 
1774 int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
1775 			  u32 len, u64 flags)
1776 {
1777 	return ____bpf_skb_store_bytes(skb, offset, from, len, flags);
1778 }
1779 
1780 BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
1781 	   void *, to, u32, len)
1782 {
1783 	void *ptr;
1784 
1785 	if (unlikely(offset > INT_MAX))
1786 		goto err_clear;
1787 
1788 	ptr = skb_header_pointer(skb, offset, len, to);
1789 	if (unlikely(!ptr))
1790 		goto err_clear;
1791 	if (ptr != to)
1792 		memmove(to, ptr, len);
1793 
1794 	return 0;
1795 err_clear:
1796 	memset(to, 0, len);
1797 	return -EFAULT;
1798 }
1799 
1800 static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
1801 	.func		= bpf_skb_load_bytes,
1802 	.gpl_only	= false,
1803 	.ret_type	= RET_INTEGER,
1804 	.arg1_type	= ARG_PTR_TO_CTX,
1805 	.arg2_type	= ARG_ANYTHING,
1806 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
1807 	.arg4_type	= ARG_MEM_SIZE,
1808 };
1809 
1810 int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len)
1811 {
1812 	return ____bpf_skb_load_bytes(skb, offset, to, len);
1813 }
1814 
1815 BPF_CALL_4(bpf_flow_dissector_load_bytes,
1816 	   const struct bpf_flow_dissector *, ctx, u32, offset,
1817 	   void *, to, u32, len)
1818 {
1819 	void *ptr;
1820 
1821 	if (unlikely(offset > 0xffff))
1822 		goto err_clear;
1823 
1824 	if (unlikely(!ctx->skb))
1825 		goto err_clear;
1826 
1827 	ptr = skb_header_pointer(ctx->skb, offset, len, to);
1828 	if (unlikely(!ptr))
1829 		goto err_clear;
1830 	if (ptr != to)
1831 		memcpy(to, ptr, len);
1832 
1833 	return 0;
1834 err_clear:
1835 	memset(to, 0, len);
1836 	return -EFAULT;
1837 }
1838 
1839 static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = {
1840 	.func		= bpf_flow_dissector_load_bytes,
1841 	.gpl_only	= false,
1842 	.ret_type	= RET_INTEGER,
1843 	.arg1_type	= ARG_PTR_TO_CTX,
1844 	.arg2_type	= ARG_ANYTHING,
1845 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
1846 	.arg4_type	= ARG_MEM_SIZE,
1847 };
1848 
1849 BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb,
1850 	   u32, offset, void *, to, u32, len, u32, start_header)
1851 {
1852 	u8 *end = skb_tail_pointer(skb);
1853 	u8 *start, *ptr;
1854 
1855 	if (unlikely(offset > 0xffff))
1856 		goto err_clear;
1857 
1858 	switch (start_header) {
1859 	case BPF_HDR_START_MAC:
1860 		if (unlikely(!skb_mac_header_was_set(skb)))
1861 			goto err_clear;
1862 		start = skb_mac_header(skb);
1863 		break;
1864 	case BPF_HDR_START_NET:
1865 		start = skb_network_header(skb);
1866 		break;
1867 	default:
1868 		goto err_clear;
1869 	}
1870 
1871 	ptr = start + offset;
1872 
1873 	if (likely(ptr + len <= end)) {
1874 		memmove(to, ptr, len);
1875 		return 0;
1876 	}
1877 
1878 err_clear:
1879 	memset(to, 0, len);
1880 	return -EFAULT;
1881 }
1882 
1883 static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = {
1884 	.func		= bpf_skb_load_bytes_relative,
1885 	.gpl_only	= false,
1886 	.ret_type	= RET_INTEGER,
1887 	.arg1_type	= ARG_PTR_TO_CTX,
1888 	.arg2_type	= ARG_ANYTHING,
1889 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
1890 	.arg4_type	= ARG_MEM_SIZE,
1891 	.arg5_type	= ARG_ANYTHING,
1892 };
1893 
1894 BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len)
1895 {
1896 	/* Idea is the following: should the needed direct read/write
1897 	 * test fail during runtime, we can pull in more data and redo
1898 	 * again, since implicitly, we invalidate previous checks here.
1899 	 *
1900 	 * Or, since we know how much we need to make read/writeable,
1901 	 * this can be done once at the program beginning for direct
1902 	 * access case. By this we overcome limitations of only current
1903 	 * headroom being accessible.
1904 	 */
1905 	return bpf_try_make_writable(skb, len ? : skb_headlen(skb));
1906 }
1907 
1908 static const struct bpf_func_proto bpf_skb_pull_data_proto = {
1909 	.func		= bpf_skb_pull_data,
1910 	.gpl_only	= false,
1911 	.ret_type	= RET_INTEGER,
1912 	.arg1_type	= ARG_PTR_TO_CTX,
1913 	.arg2_type	= ARG_ANYTHING,
1914 };
1915 
1916 BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
1917 {
1918 	return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
1919 }
1920 
1921 static const struct bpf_func_proto bpf_sk_fullsock_proto = {
1922 	.func		= bpf_sk_fullsock,
1923 	.gpl_only	= false,
1924 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
1925 	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
1926 };
1927 
1928 static inline int sk_skb_try_make_writable(struct sk_buff *skb,
1929 					   unsigned int write_len)
1930 {
1931 	return __bpf_try_make_writable(skb, write_len);
1932 }
1933 
1934 BPF_CALL_2(sk_skb_pull_data, struct sk_buff *, skb, u32, len)
1935 {
1936 	/* Idea is the following: should the needed direct read/write
1937 	 * test fail during runtime, we can pull in more data and redo
1938 	 * again, since implicitly, we invalidate previous checks here.
1939 	 *
1940 	 * Or, since we know how much we need to make read/writeable,
1941 	 * this can be done once at the program beginning for direct
1942 	 * access case. By this we overcome limitations of only current
1943 	 * headroom being accessible.
1944 	 */
1945 	return sk_skb_try_make_writable(skb, len ? : skb_headlen(skb));
1946 }
1947 
1948 static const struct bpf_func_proto sk_skb_pull_data_proto = {
1949 	.func		= sk_skb_pull_data,
1950 	.gpl_only	= false,
1951 	.ret_type	= RET_INTEGER,
1952 	.arg1_type	= ARG_PTR_TO_CTX,
1953 	.arg2_type	= ARG_ANYTHING,
1954 };
1955 
1956 BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset,
1957 	   u64, from, u64, to, u64, flags)
1958 {
1959 	__sum16 *ptr;
1960 
1961 	if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK)))
1962 		return -EINVAL;
1963 	if (unlikely(offset > 0xffff || offset & 1))
1964 		return -EFAULT;
1965 	if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
1966 		return -EFAULT;
1967 
1968 	ptr = (__sum16 *)(skb->data + offset);
1969 	switch (flags & BPF_F_HDR_FIELD_MASK) {
1970 	case 0:
1971 		if (unlikely(from != 0))
1972 			return -EINVAL;
1973 
1974 		csum_replace_by_diff(ptr, to);
1975 		break;
1976 	case 2:
1977 		csum_replace2(ptr, from, to);
1978 		break;
1979 	case 4:
1980 		csum_replace4(ptr, from, to);
1981 		break;
1982 	default:
1983 		return -EINVAL;
1984 	}
1985 
1986 	return 0;
1987 }
1988 
1989 static const struct bpf_func_proto bpf_l3_csum_replace_proto = {
1990 	.func		= bpf_l3_csum_replace,
1991 	.gpl_only	= false,
1992 	.ret_type	= RET_INTEGER,
1993 	.arg1_type	= ARG_PTR_TO_CTX,
1994 	.arg2_type	= ARG_ANYTHING,
1995 	.arg3_type	= ARG_ANYTHING,
1996 	.arg4_type	= ARG_ANYTHING,
1997 	.arg5_type	= ARG_ANYTHING,
1998 };
1999 
2000 BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset,
2001 	   u64, from, u64, to, u64, flags)
2002 {
2003 	bool is_pseudo = flags & BPF_F_PSEUDO_HDR;
2004 	bool is_mmzero = flags & BPF_F_MARK_MANGLED_0;
2005 	bool do_mforce = flags & BPF_F_MARK_ENFORCE;
2006 	bool is_ipv6   = flags & BPF_F_IPV6;
2007 	__sum16 *ptr;
2008 
2009 	if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE |
2010 			       BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK | BPF_F_IPV6)))
2011 		return -EINVAL;
2012 	if (unlikely(offset > 0xffff || offset & 1))
2013 		return -EFAULT;
2014 	if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
2015 		return -EFAULT;
2016 
2017 	ptr = (__sum16 *)(skb->data + offset);
2018 	if (is_mmzero && !do_mforce && !*ptr)
2019 		return 0;
2020 
2021 	switch (flags & BPF_F_HDR_FIELD_MASK) {
2022 	case 0:
2023 		if (unlikely(from != 0))
2024 			return -EINVAL;
2025 
2026 		inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo, is_ipv6);
2027 		break;
2028 	case 2:
2029 		inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo);
2030 		break;
2031 	case 4:
2032 		inet_proto_csum_replace4(ptr, skb, from, to, is_pseudo);
2033 		break;
2034 	default:
2035 		return -EINVAL;
2036 	}
2037 
2038 	if (is_mmzero && !*ptr)
2039 		*ptr = CSUM_MANGLED_0;
2040 	return 0;
2041 }
2042 
2043 static const struct bpf_func_proto bpf_l4_csum_replace_proto = {
2044 	.func		= bpf_l4_csum_replace,
2045 	.gpl_only	= false,
2046 	.ret_type	= RET_INTEGER,
2047 	.arg1_type	= ARG_PTR_TO_CTX,
2048 	.arg2_type	= ARG_ANYTHING,
2049 	.arg3_type	= ARG_ANYTHING,
2050 	.arg4_type	= ARG_ANYTHING,
2051 	.arg5_type	= ARG_ANYTHING,
2052 };
2053 
2054 BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size,
2055 	   __be32 *, to, u32, to_size, __wsum, seed)
2056 {
2057 	/* This is quite flexible, some examples:
2058 	 *
2059 	 * from_size == 0, to_size > 0,  seed := csum --> pushing data
2060 	 * from_size > 0,  to_size == 0, seed := csum --> pulling data
2061 	 * from_size > 0,  to_size > 0,  seed := 0    --> diffing data
2062 	 *
2063 	 * Even for diffing, from_size and to_size don't need to be equal.
2064 	 */
2065 
2066 	__wsum ret = seed;
2067 
2068 	if (from_size && to_size)
2069 		ret = csum_sub(csum_partial(to, to_size, ret),
2070 			       csum_partial(from, from_size, 0));
2071 	else if (to_size)
2072 		ret = csum_partial(to, to_size, ret);
2073 
2074 	else if (from_size)
2075 		ret = ~csum_partial(from, from_size, ~ret);
2076 
2077 	return csum_from32to16((__force unsigned int)ret);
2078 }
2079 
2080 static const struct bpf_func_proto bpf_csum_diff_proto = {
2081 	.func		= bpf_csum_diff,
2082 	.gpl_only	= false,
2083 	.pkt_access	= true,
2084 	.ret_type	= RET_INTEGER,
2085 	.arg1_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
2086 	.arg2_type	= ARG_MEM_SIZE_OR_ZERO,
2087 	.arg3_type	= ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
2088 	.arg4_type	= ARG_MEM_SIZE_OR_ZERO,
2089 	.arg5_type	= ARG_ANYTHING,
2090 };
2091 
2092 BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum)
2093 {
2094 	/* The interface is to be used in combination with bpf_csum_diff()
2095 	 * for direct packet writes. csum rotation for alignment as well
2096 	 * as emulating csum_sub() can be done from the eBPF program.
2097 	 */
2098 	if (skb->ip_summed == CHECKSUM_COMPLETE)
2099 		return (skb->csum = csum_add(skb->csum, csum));
2100 
2101 	return -ENOTSUPP;
2102 }
2103 
2104 static const struct bpf_func_proto bpf_csum_update_proto = {
2105 	.func		= bpf_csum_update,
2106 	.gpl_only	= false,
2107 	.ret_type	= RET_INTEGER,
2108 	.arg1_type	= ARG_PTR_TO_CTX,
2109 	.arg2_type	= ARG_ANYTHING,
2110 };
2111 
2112 BPF_CALL_2(bpf_csum_level, struct sk_buff *, skb, u64, level)
2113 {
2114 	/* The interface is to be used in combination with bpf_skb_adjust_room()
2115 	 * for encap/decap of packet headers when BPF_F_ADJ_ROOM_NO_CSUM_RESET
2116 	 * is passed as flags, for example.
2117 	 */
2118 	switch (level) {
2119 	case BPF_CSUM_LEVEL_INC:
2120 		__skb_incr_checksum_unnecessary(skb);
2121 		break;
2122 	case BPF_CSUM_LEVEL_DEC:
2123 		__skb_decr_checksum_unnecessary(skb);
2124 		break;
2125 	case BPF_CSUM_LEVEL_RESET:
2126 		__skb_reset_checksum_unnecessary(skb);
2127 		break;
2128 	case BPF_CSUM_LEVEL_QUERY:
2129 		return skb->ip_summed == CHECKSUM_UNNECESSARY ?
2130 		       skb->csum_level : -EACCES;
2131 	default:
2132 		return -EINVAL;
2133 	}
2134 
2135 	return 0;
2136 }
2137 
2138 static const struct bpf_func_proto bpf_csum_level_proto = {
2139 	.func		= bpf_csum_level,
2140 	.gpl_only	= false,
2141 	.ret_type	= RET_INTEGER,
2142 	.arg1_type	= ARG_PTR_TO_CTX,
2143 	.arg2_type	= ARG_ANYTHING,
2144 };
2145 
2146 static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
2147 {
2148 	return dev_forward_skb_nomtu(dev, skb);
2149 }
2150 
2151 static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
2152 				      struct sk_buff *skb)
2153 {
2154 	int ret = ____dev_forward_skb(dev, skb, false);
2155 
2156 	if (likely(!ret)) {
2157 		skb->dev = dev;
2158 		ret = netif_rx(skb);
2159 	}
2160 
2161 	return ret;
2162 }
2163 
2164 static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
2165 {
2166 	int ret;
2167 
2168 	if (dev_xmit_recursion()) {
2169 		net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2170 		kfree_skb(skb);
2171 		return -ENETDOWN;
2172 	}
2173 
2174 	skb->dev = dev;
2175 	skb_set_redirected_noclear(skb, skb_at_tc_ingress(skb));
2176 	skb_clear_tstamp(skb);
2177 
2178 	dev_xmit_recursion_inc();
2179 	ret = dev_queue_xmit(skb);
2180 	dev_xmit_recursion_dec();
2181 
2182 	return ret;
2183 }
2184 
2185 static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev,
2186 				 u32 flags)
2187 {
2188 	unsigned int mlen = skb_network_offset(skb);
2189 
2190 	if (unlikely(skb->len <= mlen)) {
2191 		kfree_skb(skb);
2192 		return -ERANGE;
2193 	}
2194 
2195 	if (mlen) {
2196 		__skb_pull(skb, mlen);
2197 
2198 		/* At ingress, the mac header has already been pulled once.
2199 		 * At egress, skb_pospull_rcsum has to be done in case that
2200 		 * the skb is originated from ingress (i.e. a forwarded skb)
2201 		 * to ensure that rcsum starts at net header.
2202 		 */
2203 		if (!skb_at_tc_ingress(skb))
2204 			skb_postpull_rcsum(skb, skb_mac_header(skb), mlen);
2205 	}
2206 	skb_pop_mac_header(skb);
2207 	skb_reset_mac_len(skb);
2208 	return flags & BPF_F_INGRESS ?
2209 	       __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb);
2210 }
2211 
2212 static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev,
2213 				 u32 flags)
2214 {
2215 	/* Verify that a link layer header is carried */
2216 	if (unlikely(skb->mac_header >= skb->network_header || skb->len == 0)) {
2217 		kfree_skb(skb);
2218 		return -ERANGE;
2219 	}
2220 
2221 	bpf_push_mac_rcsum(skb);
2222 	return flags & BPF_F_INGRESS ?
2223 	       __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
2224 }
2225 
2226 static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev,
2227 			  u32 flags)
2228 {
2229 	if (dev_is_mac_header_xmit(dev))
2230 		return __bpf_redirect_common(skb, dev, flags);
2231 	else
2232 		return __bpf_redirect_no_mac(skb, dev, flags);
2233 }
2234 
2235 #if IS_ENABLED(CONFIG_IPV6)
2236 static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
2237 			    struct net_device *dev, struct bpf_nh_params *nh)
2238 {
2239 	u32 hh_len = LL_RESERVED_SPACE(dev);
2240 	const struct in6_addr *nexthop;
2241 	struct dst_entry *dst = NULL;
2242 	struct neighbour *neigh;
2243 
2244 	if (dev_xmit_recursion()) {
2245 		net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2246 		goto out_drop;
2247 	}
2248 
2249 	skb->dev = dev;
2250 	skb_clear_tstamp(skb);
2251 
2252 	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
2253 		skb = skb_expand_head(skb, hh_len);
2254 		if (!skb)
2255 			return -ENOMEM;
2256 	}
2257 
2258 	if (unlikely(!ipv6_mod_enabled()))
2259 		goto out_drop;
2260 
2261 	rcu_read_lock();
2262 	if (!nh) {
2263 		dst = skb_dst(skb);
2264 		nexthop = rt6_nexthop(dst_rt6_info(dst),
2265 				      &ipv6_hdr(skb)->daddr);
2266 	} else {
2267 		nexthop = &nh->ipv6_nh;
2268 	}
2269 	neigh = ip_neigh_gw6(dev, nexthop);
2270 	if (likely(!IS_ERR(neigh))) {
2271 		int ret;
2272 
2273 		sock_confirm_neigh(skb, neigh);
2274 		local_bh_disable();
2275 		dev_xmit_recursion_inc();
2276 		ret = neigh_output(neigh, skb, false);
2277 		dev_xmit_recursion_dec();
2278 		local_bh_enable();
2279 		rcu_read_unlock();
2280 		return ret;
2281 	}
2282 	rcu_read_unlock();
2283 	if (dst)
2284 		IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
2285 out_drop:
2286 	kfree_skb(skb);
2287 	return -ENETDOWN;
2288 }
2289 
2290 static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
2291 				   struct bpf_nh_params *nh)
2292 {
2293 	struct net *net = dev_net(dev);
2294 	int err, ret = NET_XMIT_DROP;
2295 	const struct ipv6hdr *ip6h;
2296 
2297 	if (unlikely(!pskb_may_pull(skb, sizeof(*ip6h))))
2298 		goto out_drop;
2299 	ip6h = ipv6_hdr(skb);
2300 
2301 	if (!nh) {
2302 		struct dst_entry *dst;
2303 		struct flowi6 fl6 = {
2304 			.flowi6_flags = FLOWI_FLAG_ANYSRC,
2305 			.flowi6_mark  = skb->mark,
2306 			.flowlabel    = ip6_flowinfo(ip6h),
2307 			.flowi6_oif   = dev->ifindex,
2308 			.flowi6_proto = ip6h->nexthdr,
2309 			.daddr	      = ip6h->daddr,
2310 			.saddr	      = ip6h->saddr,
2311 		};
2312 
2313 		dst = ip6_dst_lookup_flow(net, NULL, &fl6, NULL);
2314 		if (IS_ERR(dst))
2315 			goto out_drop;
2316 
2317 		skb_dst_drop(skb);
2318 		skb_dst_set(skb, dst);
2319 	} else if (nh->nh_family != AF_INET6) {
2320 		goto out_drop;
2321 	}
2322 
2323 	err = bpf_out_neigh_v6(net, skb, dev, nh);
2324 	if (unlikely(net_xmit_eval(err)))
2325 		dev_core_stats_tx_dropped_inc(dev);
2326 	else
2327 		ret = NET_XMIT_SUCCESS;
2328 	goto out_xmit;
2329 out_drop:
2330 	dev_core_stats_tx_dropped_inc(dev);
2331 	kfree_skb(skb);
2332 out_xmit:
2333 	return ret;
2334 }
2335 #else
2336 static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
2337 				   struct bpf_nh_params *nh)
2338 {
2339 	kfree_skb(skb);
2340 	return NET_XMIT_DROP;
2341 }
2342 #endif /* CONFIG_IPV6 */
2343 
2344 #if IS_ENABLED(CONFIG_INET)
2345 static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
2346 			    struct net_device *dev, struct bpf_nh_params *nh)
2347 {
2348 	u32 hh_len = LL_RESERVED_SPACE(dev);
2349 	struct neighbour *neigh;
2350 	bool is_v6gw = false;
2351 
2352 	if (dev_xmit_recursion()) {
2353 		net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
2354 		goto out_drop;
2355 	}
2356 
2357 	skb->dev = dev;
2358 	skb_clear_tstamp(skb);
2359 
2360 	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
2361 		skb = skb_expand_head(skb, hh_len);
2362 		if (!skb)
2363 			return -ENOMEM;
2364 	}
2365 
2366 	rcu_read_lock();
2367 	if (!nh) {
2368 		struct rtable *rt = skb_rtable(skb);
2369 
2370 		neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
2371 	} else if (nh->nh_family == AF_INET6) {
2372 		if (unlikely(!ipv6_mod_enabled())) {
2373 			rcu_read_unlock();
2374 			goto out_drop;
2375 		}
2376 		neigh = ip_neigh_gw6(dev, &nh->ipv6_nh);
2377 		is_v6gw = true;
2378 	} else if (nh->nh_family == AF_INET) {
2379 		neigh = ip_neigh_gw4(dev, nh->ipv4_nh);
2380 	} else {
2381 		rcu_read_unlock();
2382 		goto out_drop;
2383 	}
2384 
2385 	if (likely(!IS_ERR(neigh))) {
2386 		int ret;
2387 
2388 		sock_confirm_neigh(skb, neigh);
2389 		local_bh_disable();
2390 		dev_xmit_recursion_inc();
2391 		ret = neigh_output(neigh, skb, is_v6gw);
2392 		dev_xmit_recursion_dec();
2393 		local_bh_enable();
2394 		rcu_read_unlock();
2395 		return ret;
2396 	}
2397 	rcu_read_unlock();
2398 out_drop:
2399 	kfree_skb(skb);
2400 	return -ENETDOWN;
2401 }
2402 
2403 static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
2404 				   struct bpf_nh_params *nh)
2405 {
2406 	struct net *net = dev_net(dev);
2407 	int err, ret = NET_XMIT_DROP;
2408 	const struct iphdr *ip4h;
2409 
2410 	if (unlikely(!pskb_may_pull(skb, sizeof(*ip4h))))
2411 		goto out_drop;
2412 	ip4h = ip_hdr(skb);
2413 
2414 	if (!nh) {
2415 		struct flowi4 fl4 = {
2416 			.flowi4_flags = FLOWI_FLAG_ANYSRC,
2417 			.flowi4_mark  = skb->mark,
2418 			.flowi4_dscp  = ip4h_dscp(ip4h),
2419 			.flowi4_oif   = dev->ifindex,
2420 			.flowi4_proto = ip4h->protocol,
2421 			.daddr	      = ip4h->daddr,
2422 			.saddr	      = ip4h->saddr,
2423 		};
2424 		struct rtable *rt;
2425 
2426 		rt = ip_route_output_flow(net, &fl4, NULL);
2427 		if (IS_ERR(rt))
2428 			goto out_drop;
2429 		if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
2430 			ip_rt_put(rt);
2431 			goto out_drop;
2432 		}
2433 
2434 		skb_dst_drop(skb);
2435 		skb_dst_set(skb, &rt->dst);
2436 	}
2437 
2438 	err = bpf_out_neigh_v4(net, skb, dev, nh);
2439 	if (unlikely(net_xmit_eval(err)))
2440 		dev_core_stats_tx_dropped_inc(dev);
2441 	else
2442 		ret = NET_XMIT_SUCCESS;
2443 	goto out_xmit;
2444 out_drop:
2445 	dev_core_stats_tx_dropped_inc(dev);
2446 	kfree_skb(skb);
2447 out_xmit:
2448 	return ret;
2449 }
2450 #else
2451 static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
2452 				   struct bpf_nh_params *nh)
2453 {
2454 	kfree_skb(skb);
2455 	return NET_XMIT_DROP;
2456 }
2457 #endif /* CONFIG_INET */
2458 
2459 static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev,
2460 				struct bpf_nh_params *nh)
2461 {
2462 	if (unlikely(skb_network_offset(skb) < ETH_HLEN ||
2463 		     !pskb_may_pull(skb, ETH_HLEN)))
2464 		goto out;
2465 	bpf_push_mac_rcsum(skb);
2466 	if (is_multicast_ether_addr(eth_hdr(skb)->h_dest))
2467 		goto out;
2468 
2469 	skb_pull(skb, ETH_HLEN);
2470 	skb_unset_mac_header(skb);
2471 	skb_reset_network_header(skb);
2472 
2473 	if (skb->protocol == htons(ETH_P_IP))
2474 		return __bpf_redirect_neigh_v4(skb, dev, nh);
2475 	else if (skb->protocol == htons(ETH_P_IPV6))
2476 		return __bpf_redirect_neigh_v6(skb, dev, nh);
2477 out:
2478 	kfree_skb(skb);
2479 	return -ENOTSUPP;
2480 }
2481 
2482 /* Internal, non-exposed redirect flags. */
2483 enum {
2484 	BPF_F_NEIGH	= (1ULL << 16),
2485 	BPF_F_PEER	= (1ULL << 17),
2486 	BPF_F_NEXTHOP	= (1ULL << 18),
2487 #define BPF_F_REDIRECT_INTERNAL	(BPF_F_NEIGH | BPF_F_PEER | BPF_F_NEXTHOP)
2488 };
2489 
2490 BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
2491 {
2492 	struct net_device *dev;
2493 	struct sk_buff *clone;
2494 	int ret;
2495 
2496 	BUILD_BUG_ON(BPF_F_REDIRECT_INTERNAL & BPF_F_REDIRECT_FLAGS);
2497 
2498 	if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
2499 		return -EINVAL;
2500 
2501 	/* BPF test infra's convert___skb_to_skb() can create type-less
2502 	 * GSO packets. gso_features_check() will detect this as a bad
2503 	 * offload. However, lets not leak them out in the first place.
2504 	 */
2505 	if (unlikely(skb_is_gso(skb) && !skb_shinfo(skb)->gso_type))
2506 		return -EBADMSG;
2507 
2508 	dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex);
2509 	if (unlikely(!dev))
2510 		return -EINVAL;
2511 
2512 	clone = skb_clone(skb, GFP_ATOMIC);
2513 	if (unlikely(!clone))
2514 		return -ENOMEM;
2515 
2516 	/* For direct write, we need to keep the invariant that the skbs
2517 	 * we're dealing with need to be uncloned. Should uncloning fail
2518 	 * here, we need to free the just generated clone to unclone once
2519 	 * again.
2520 	 */
2521 	ret = bpf_try_make_head_writable(skb);
2522 	if (unlikely(ret)) {
2523 		kfree_skb(clone);
2524 		return -ENOMEM;
2525 	}
2526 
2527 	return __bpf_redirect(clone, dev, flags);
2528 }
2529 
2530 static const struct bpf_func_proto bpf_clone_redirect_proto = {
2531 	.func           = bpf_clone_redirect,
2532 	.gpl_only       = false,
2533 	.ret_type       = RET_INTEGER,
2534 	.arg1_type      = ARG_PTR_TO_CTX,
2535 	.arg2_type      = ARG_ANYTHING,
2536 	.arg3_type      = ARG_ANYTHING,
2537 };
2538 
2539 static struct net_device *skb_get_peer_dev(struct net_device *dev)
2540 {
2541 	const struct net_device_ops *ops = dev->netdev_ops;
2542 
2543 	if (likely(ops->ndo_get_peer_dev))
2544 		return INDIRECT_CALL_1(ops->ndo_get_peer_dev,
2545 				       netkit_peer_dev, dev);
2546 	return NULL;
2547 }
2548 
2549 int skb_do_redirect(struct sk_buff *skb)
2550 {
2551 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
2552 	struct net *net = dev_net(skb->dev);
2553 	struct net_device *dev;
2554 	u32 flags = ri->flags;
2555 
2556 	dev = dev_get_by_index_rcu(net, ri->tgt_index);
2557 	ri->tgt_index = 0;
2558 	ri->flags = 0;
2559 	if (unlikely(!dev))
2560 		goto out_drop;
2561 	if (flags & BPF_F_PEER) {
2562 		dev = skb_get_peer_dev(dev);
2563 		if (unlikely(!dev ||
2564 			     !(dev->flags & IFF_UP) ||
2565 			     net_eq(net, dev_net(dev))))
2566 			goto out_drop;
2567 		skb_scrub_packet(skb, false);
2568 		if (flags & BPF_F_EGRESS)
2569 			return __bpf_redirect(skb, dev, 0);
2570 		if (unlikely(!skb_at_tc_ingress(skb)))
2571 			goto out_drop;
2572 		skb->dev = dev;
2573 		dev_sw_netstats_rx_add(dev, skb->len);
2574 		return -EAGAIN;
2575 	}
2576 	return flags & BPF_F_NEIGH ?
2577 	       __bpf_redirect_neigh(skb, dev, flags & BPF_F_NEXTHOP ?
2578 				    &ri->nh : NULL) :
2579 	       __bpf_redirect(skb, dev, flags);
2580 out_drop:
2581 	kfree_skb(skb);
2582 	return -EINVAL;
2583 }
2584 
2585 BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
2586 {
2587 	struct bpf_redirect_info *ri;
2588 
2589 	if (unlikely(!bpf_net_ctx_get() ||
2590 		     (flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))))
2591 		return TC_ACT_SHOT;
2592 
2593 	ri = bpf_net_ctx_get_ri();
2594 	ri->flags = flags;
2595 	ri->tgt_index = ifindex;
2596 
2597 	return TC_ACT_REDIRECT;
2598 }
2599 
2600 static const struct bpf_func_proto bpf_redirect_proto = {
2601 	.func           = bpf_redirect,
2602 	.gpl_only       = false,
2603 	.ret_type       = RET_INTEGER,
2604 	.arg1_type      = ARG_ANYTHING,
2605 	.arg2_type      = ARG_ANYTHING,
2606 };
2607 
2608 BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
2609 {
2610 	struct bpf_redirect_info *ri;
2611 
2612 	if (unlikely(!bpf_net_ctx_get() || (flags & ~BPF_F_EGRESS)))
2613 		return TC_ACT_SHOT;
2614 
2615 	ri = bpf_net_ctx_get_ri();
2616 	ri->flags = BPF_F_PEER | flags;
2617 	ri->tgt_index = ifindex;
2618 
2619 	return TC_ACT_REDIRECT;
2620 }
2621 
2622 static const struct bpf_func_proto bpf_redirect_peer_proto = {
2623 	.func           = bpf_redirect_peer,
2624 	.gpl_only       = false,
2625 	.ret_type       = RET_INTEGER,
2626 	.arg1_type      = ARG_ANYTHING,
2627 	.arg2_type      = ARG_ANYTHING,
2628 };
2629 
2630 BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params,
2631 	   int, plen, u64, flags)
2632 {
2633 	struct bpf_redirect_info *ri;
2634 
2635 	if (unlikely((plen && plen < sizeof(*params)) ||
2636 		     !bpf_net_ctx_get() || flags))
2637 		return TC_ACT_SHOT;
2638 
2639 	ri = bpf_net_ctx_get_ri();
2640 	ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0);
2641 	ri->tgt_index = ifindex;
2642 
2643 	BUILD_BUG_ON(sizeof(struct bpf_redir_neigh) != sizeof(struct bpf_nh_params));
2644 	if (plen)
2645 		memcpy(&ri->nh, params, sizeof(ri->nh));
2646 
2647 	return TC_ACT_REDIRECT;
2648 }
2649 
2650 static const struct bpf_func_proto bpf_redirect_neigh_proto = {
2651 	.func		= bpf_redirect_neigh,
2652 	.gpl_only	= false,
2653 	.ret_type	= RET_INTEGER,
2654 	.arg1_type	= ARG_ANYTHING,
2655 	.arg2_type      = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
2656 	.arg3_type      = ARG_MEM_SIZE_OR_ZERO,
2657 	.arg4_type	= ARG_ANYTHING,
2658 };
2659 
2660 BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes)
2661 {
2662 	msg->apply_bytes = bytes;
2663 	return 0;
2664 }
2665 
2666 static const struct bpf_func_proto bpf_msg_apply_bytes_proto = {
2667 	.func           = bpf_msg_apply_bytes,
2668 	.gpl_only       = false,
2669 	.ret_type       = RET_INTEGER,
2670 	.arg1_type	= ARG_PTR_TO_CTX,
2671 	.arg2_type      = ARG_ANYTHING,
2672 };
2673 
2674 BPF_CALL_2(bpf_msg_cork_bytes, struct sk_msg *, msg, u32, bytes)
2675 {
2676 	msg->cork_bytes = bytes;
2677 	return 0;
2678 }
2679 
2680 static bool sk_msg_elem_is_copy(const struct sk_msg *msg, u32 i)
2681 {
2682 	return test_bit(i, msg->sg.copy);
2683 }
2684 
2685 static void sk_msg_clear_elem_copy(struct sk_msg *msg, u32 i)
2686 {
2687 	__clear_bit(i, msg->sg.copy);
2688 }
2689 
2690 static void sk_msg_set_elem_copy(struct sk_msg *msg, u32 i, bool sg_copy)
2691 {
2692 	__assign_bit(i, msg->sg.copy, sg_copy);
2693 }
2694 
2695 static void sk_msg_clear_copy_range(struct sk_msg *msg, u32 start, u32 end)
2696 {
2697 	while (start != end) {
2698 		sk_msg_clear_elem_copy(msg, start);
2699 		sk_msg_iter_var_next(start);
2700 	}
2701 }
2702 
2703 static void sk_msg_sg_move(struct sk_msg *msg, u32 dst, u32 src)
2704 {
2705 	msg->sg.data[dst] = msg->sg.data[src];
2706 
2707 	sk_msg_set_elem_copy(msg, dst,
2708 		sk_msg_elem_is_copy(msg, src));
2709 }
2710 
2711 static const struct bpf_func_proto bpf_msg_cork_bytes_proto = {
2712 	.func           = bpf_msg_cork_bytes,
2713 	.gpl_only       = false,
2714 	.ret_type       = RET_INTEGER,
2715 	.arg1_type	= ARG_PTR_TO_CTX,
2716 	.arg2_type      = ARG_ANYTHING,
2717 };
2718 
2719 BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start,
2720 	   u32, end, u64, flags)
2721 {
2722 	u32 len = 0, offset = 0, copy = 0, poffset = 0, bytes = end - start;
2723 	u32 first_sge, last_sge, i, shift, bytes_sg_total;
2724 	struct scatterlist *sge;
2725 	u8 *raw, *to, *from;
2726 	struct page *page;
2727 
2728 	if (unlikely(flags || end <= start))
2729 		return -EINVAL;
2730 
2731 	/* First find the starting scatterlist element */
2732 	i = msg->sg.start;
2733 	do {
2734 		offset += len;
2735 		len = sk_msg_elem(msg, i)->length;
2736 		if (start < offset + len)
2737 			break;
2738 		sk_msg_iter_var_next(i);
2739 	} while (i != msg->sg.end);
2740 
2741 	if (unlikely(start >= offset + len))
2742 		return -EINVAL;
2743 
2744 	first_sge = i;
2745 	/* The start may point into the sg element so we need to also
2746 	 * account for the headroom.
2747 	 */
2748 	bytes_sg_total = start - offset + bytes;
2749 	if (!sk_msg_elem_is_copy(msg, i) && bytes_sg_total <= len)
2750 		goto out;
2751 
2752 	/* At this point we need to linearize multiple scatterlist
2753 	 * elements or a single shared page. Either way we need to
2754 	 * copy into a linear buffer exclusively owned by BPF. Then
2755 	 * place the buffer in the scatterlist and fixup the original
2756 	 * entries by removing the entries now in the linear buffer
2757 	 * and shifting the remaining entries. For now we do not try
2758 	 * to copy partial entries to avoid complexity of running out
2759 	 * of sg_entry slots. The downside is reading a single byte
2760 	 * will copy the entire sg entry.
2761 	 */
2762 	do {
2763 		copy += sk_msg_elem(msg, i)->length;
2764 		sk_msg_iter_var_next(i);
2765 		if (bytes_sg_total <= copy)
2766 			break;
2767 	} while (i != msg->sg.end);
2768 	last_sge = i;
2769 
2770 	if (unlikely(bytes_sg_total > copy))
2771 		return -EINVAL;
2772 
2773 	page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
2774 			   get_order(copy));
2775 	if (unlikely(!page))
2776 		return -ENOMEM;
2777 
2778 	raw = page_address(page);
2779 	i = first_sge;
2780 	do {
2781 		sge = sk_msg_elem(msg, i);
2782 		from = sg_virt(sge);
2783 		len = sge->length;
2784 		to = raw + poffset;
2785 
2786 		memcpy(to, from, len);
2787 		poffset += len;
2788 		sge->length = 0;
2789 		put_page(sg_page(sge));
2790 		sk_msg_clear_elem_copy(msg, i);
2791 
2792 		sk_msg_iter_var_next(i);
2793 	} while (i != last_sge);
2794 
2795 	sg_set_page(&msg->sg.data[first_sge], page, copy, 0);
2796 	sk_msg_clear_elem_copy(msg, first_sge);
2797 
2798 	/* To repair sg ring we need to shift entries. If we only
2799 	 * had a single entry though we can just replace it and
2800 	 * be done. Otherwise walk the ring and shift the entries.
2801 	 */
2802 	WARN_ON_ONCE(last_sge == first_sge);
2803 	shift = last_sge > first_sge ?
2804 		last_sge - first_sge - 1 :
2805 		NR_MSG_FRAG_IDS - first_sge + last_sge - 1;
2806 	if (!shift) {
2807 		sk_msg_clear_elem_copy(msg, msg->sg.end);
2808 		goto out;
2809 	}
2810 
2811 	i = first_sge;
2812 	sk_msg_iter_var_next(i);
2813 	sk_msg_clear_copy_range(msg, i, last_sge);
2814 
2815 	i = first_sge;
2816 	sk_msg_iter_var_next(i);
2817 	do {
2818 		u32 move_from;
2819 
2820 		if (i + shift >= NR_MSG_FRAG_IDS)
2821 			move_from = i + shift - NR_MSG_FRAG_IDS;
2822 		else
2823 			move_from = i + shift;
2824 		if (move_from == msg->sg.end)
2825 			break;
2826 
2827 		sk_msg_sg_move(msg, i, move_from);
2828 		msg->sg.data[move_from].length = 0;
2829 		msg->sg.data[move_from].page_link = 0;
2830 		msg->sg.data[move_from].offset = 0;
2831 		sk_msg_clear_elem_copy(msg, move_from);
2832 		sk_msg_iter_var_next(i);
2833 	} while (1);
2834 
2835 	msg->sg.end = msg->sg.end - shift > msg->sg.end ?
2836 		      msg->sg.end - shift + NR_MSG_FRAG_IDS :
2837 		      msg->sg.end - shift;
2838 	sk_msg_clear_elem_copy(msg, msg->sg.end);
2839 out:
2840 	sk_msg_reset_curr(msg);
2841 	msg->data = sg_virt(&msg->sg.data[first_sge]) + start - offset;
2842 	msg->data_end = msg->data + bytes;
2843 	return 0;
2844 }
2845 
2846 static const struct bpf_func_proto bpf_msg_pull_data_proto = {
2847 	.func		= bpf_msg_pull_data,
2848 	.gpl_only	= false,
2849 	.ret_type	= RET_INTEGER,
2850 	.arg1_type	= ARG_PTR_TO_CTX,
2851 	.arg2_type	= ARG_ANYTHING,
2852 	.arg3_type	= ARG_ANYTHING,
2853 	.arg4_type	= ARG_ANYTHING,
2854 };
2855 
2856 BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start,
2857 	   u32, len, u64, flags)
2858 {
2859 	bool sge_copy = false, nsge_copy = false, nnsge_copy = false;
2860 	struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge;
2861 	u32 new, i = 0, l = 0, space, copy = 0, offset = 0;
2862 	bool rsge_copy = false;
2863 	u8 *raw, *to, *from;
2864 	struct page *page;
2865 
2866 	if (unlikely(flags))
2867 		return -EINVAL;
2868 
2869 	if (unlikely(len == 0))
2870 		return 0;
2871 
2872 	/* First find the starting scatterlist element */
2873 	i = msg->sg.start;
2874 	do {
2875 		offset += l;
2876 		l = sk_msg_elem(msg, i)->length;
2877 
2878 		if (start < offset + l)
2879 			break;
2880 		sk_msg_iter_var_next(i);
2881 	} while (i != msg->sg.end);
2882 
2883 	if (start > offset + l)
2884 		return -EINVAL;
2885 
2886 	/* A push at the end of the msg works on the last element */
2887 	if (i == msg->sg.end && start != offset)
2888 		sk_msg_iter_var_prev(i);
2889 
2890 	space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
2891 
2892 	/* If no space available will fallback to copy, we need at
2893 	 * least one scatterlist elem available to push data into
2894 	 * when start aligns to the beginning of an element or two
2895 	 * when it falls inside an element. We handle the start equals
2896 	 * offset case because its the common case for inserting a
2897 	 * header.
2898 	 */
2899 	if (!space || (space == 1 && start != offset && start < offset + l))
2900 		copy = msg->sg.data[i].length;
2901 
2902 	if (unlikely(copy + len < copy))
2903 		return -EINVAL;
2904 
2905 	page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
2906 			   get_order(copy + len));
2907 	if (unlikely(!page))
2908 		return -ENOMEM;
2909 
2910 	if (copy) {
2911 		int front, back;
2912 
2913 		raw = page_address(page);
2914 
2915 		psge = sk_msg_elem(msg, i);
2916 		front = start - offset;
2917 		back = psge->length - front;
2918 		from = sg_virt(psge);
2919 
2920 		if (front)
2921 			memcpy(raw, from, front);
2922 
2923 		if (back) {
2924 			from += front;
2925 			to = raw + front + len;
2926 
2927 			memcpy(to, from, back);
2928 		}
2929 
2930 		put_page(sg_page(psge));
2931 		new = i;
2932 		goto place_new;
2933 	}
2934 
2935 	if (start - offset) {
2936 		psge = sk_msg_elem(msg, i);
2937 		rsge = sk_msg_elem_cpy(msg, i);
2938 		rsge_copy = sk_msg_elem_is_copy(msg, i);
2939 
2940 		psge->length = start - offset;
2941 		rsge.length -= psge->length;
2942 		rsge.offset += start - offset;
2943 
2944 		sk_msg_iter_var_next(i);
2945 		sg_unmark_end(psge);
2946 		sg_unmark_end(&rsge);
2947 	}
2948 
2949 	/* Slot(s) to place newly allocated data */
2950 	sk_msg_iter_next(msg, end);
2951 	new = i;
2952 	sk_msg_iter_var_next(i);
2953 
2954 	if (i == msg->sg.end) {
2955 		if (!rsge.length)
2956 			goto place_new;
2957 		sk_msg_iter_next(msg, end);
2958 		goto place_new;
2959 	}
2960 
2961 	/* Shift one or two slots as needed */
2962 	sge = sk_msg_elem_cpy(msg, new);
2963 	sg_unmark_end(&sge);
2964 	sge_copy = sk_msg_elem_is_copy(msg, new);
2965 
2966 	nsge = sk_msg_elem_cpy(msg, i);
2967 	nsge_copy = sk_msg_elem_is_copy(msg, i);
2968 	if (rsge.length) {
2969 		sk_msg_iter_var_next(i);
2970 		nnsge = sk_msg_elem_cpy(msg, i);
2971 		nnsge_copy = sk_msg_elem_is_copy(msg, i);
2972 		sk_msg_iter_next(msg, end);
2973 	}
2974 
2975 	while (i != msg->sg.end) {
2976 		msg->sg.data[i] = sge;
2977 		sk_msg_set_elem_copy(msg, i, sge_copy);
2978 		sge = nsge;
2979 		sge_copy = nsge_copy;
2980 		sk_msg_iter_var_next(i);
2981 		if (rsge.length) {
2982 			nsge = nnsge;
2983 			nsge_copy = nnsge_copy;
2984 			nnsge = sk_msg_elem_cpy(msg, i);
2985 			nnsge_copy = sk_msg_elem_is_copy(msg, i);
2986 		} else {
2987 			nsge = sk_msg_elem_cpy(msg, i);
2988 			nsge_copy = sk_msg_elem_is_copy(msg, i);
2989 		}
2990 	}
2991 
2992 place_new:
2993 	/* Place newly allocated data buffer */
2994 	sk_mem_charge(msg->sk, len);
2995 	msg->sg.size += len;
2996 	sk_msg_clear_elem_copy(msg, new);
2997 	sg_set_page(&msg->sg.data[new], page, len + copy, 0);
2998 	/* A free slot can have the end mark */
2999 	sg_unmark_end(&msg->sg.data[new]);
3000 	if (rsge.length) {
3001 		get_page(sg_page(&rsge));
3002 		sk_msg_iter_var_next(new);
3003 		msg->sg.data[new] = rsge;
3004 		sk_msg_set_elem_copy(msg, new, rsge_copy);
3005 	}
3006 	sk_msg_clear_elem_copy(msg, msg->sg.end);
3007 
3008 	sk_msg_reset_curr(msg);
3009 	sk_msg_compute_data_pointers(msg);
3010 	return 0;
3011 }
3012 
3013 static const struct bpf_func_proto bpf_msg_push_data_proto = {
3014 	.func		= bpf_msg_push_data,
3015 	.gpl_only	= false,
3016 	.ret_type	= RET_INTEGER,
3017 	.arg1_type	= ARG_PTR_TO_CTX,
3018 	.arg2_type	= ARG_ANYTHING,
3019 	.arg3_type	= ARG_ANYTHING,
3020 	.arg4_type	= ARG_ANYTHING,
3021 };
3022 
3023 static void sk_msg_shift_left(struct sk_msg *msg, int i)
3024 {
3025 	struct scatterlist *sge = sk_msg_elem(msg, i);
3026 	int prev;
3027 
3028 	put_page(sg_page(sge));
3029 	do {
3030 		prev = i;
3031 		sk_msg_iter_var_next(i);
3032 		sk_msg_sg_move(msg, prev, i);
3033 	} while (i != msg->sg.end);
3034 
3035 	sk_msg_iter_prev(msg, end);
3036 	sk_msg_clear_elem_copy(msg, msg->sg.end);
3037 }
3038 
3039 static void sk_msg_shift_right(struct sk_msg *msg, int i)
3040 {
3041 	struct scatterlist tmp, sge;
3042 	bool tmp_copy, sge_copy;
3043 
3044 	sk_msg_iter_next(msg, end);
3045 	sge = sk_msg_elem_cpy(msg, i);
3046 	sge_copy = sk_msg_elem_is_copy(msg, i);
3047 	sk_msg_iter_var_next(i);
3048 	tmp = sk_msg_elem_cpy(msg, i);
3049 	tmp_copy = sk_msg_elem_is_copy(msg, i);
3050 
3051 	while (i != msg->sg.end) {
3052 		msg->sg.data[i] = sge;
3053 		sk_msg_set_elem_copy(msg, i, sge_copy);
3054 		sk_msg_iter_var_next(i);
3055 		sge = tmp;
3056 		sge_copy = tmp_copy;
3057 		tmp = sk_msg_elem_cpy(msg, i);
3058 		tmp_copy = sk_msg_elem_is_copy(msg, i);
3059 	}
3060 	sk_msg_clear_elem_copy(msg, msg->sg.end);
3061 }
3062 
3063 BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,
3064 	   u32, len, u64, flags)
3065 {
3066 	u32 i = 0, l = 0, space, offset = 0;
3067 	u64 last = (u64)start + len;
3068 	u32 pop;
3069 
3070 	if (unlikely(flags))
3071 		return -EINVAL;
3072 
3073 	if (unlikely(len == 0))
3074 		return 0;
3075 
3076 	/* First find the starting scatterlist element */
3077 	i = msg->sg.start;
3078 	do {
3079 		offset += l;
3080 		l = sk_msg_elem(msg, i)->length;
3081 
3082 		if (start < offset + l)
3083 			break;
3084 		sk_msg_iter_var_next(i);
3085 	} while (i != msg->sg.end);
3086 
3087 	/* Bounds checks: start and pop must be inside message */
3088 	if (start >= offset + l || last > msg->sg.size)
3089 		return -EINVAL;
3090 
3091 	space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);
3092 
3093 	pop = len;
3094 	/* --------------| offset
3095 	 * -| start      |-------- len -------|
3096 	 *
3097 	 *  |----- a ----|-------- pop -------|----- b ----|
3098 	 *  |______________________________________________| length
3099 	 *
3100 	 *
3101 	 * a:   region at front of scatter element to save
3102 	 * b:   region at back of scatter element to save when length > A + pop
3103 	 * pop: region to pop from element, same as input 'pop' here will be
3104 	 *      decremented below per iteration.
3105 	 *
3106 	 * Two top-level cases to handle when start != offset, first B is non
3107 	 * zero and second B is zero corresponding to when a pop includes more
3108 	 * than one element.
3109 	 *
3110 	 * Then if B is non-zero AND there is no space allocate space and
3111 	 * compact A, B regions into page. If there is space shift ring to
3112 	 * the right free'ing the next element in ring to place B, leaving
3113 	 * A untouched except to reduce length.
3114 	 */
3115 	if (start != offset) {
3116 		struct scatterlist *nsge, *sge = sk_msg_elem(msg, i);
3117 		bool sge_copy = sk_msg_elem_is_copy(msg, i);
3118 		int a = start - offset;
3119 		int b = sge->length - pop - a;
3120 		u32 sge_idx = i;
3121 
3122 		sk_msg_iter_var_next(i);
3123 
3124 		if (b > 0) {
3125 			if (space) {
3126 				sge->length = a;
3127 				sk_msg_shift_right(msg, i);
3128 				nsge = sk_msg_elem(msg, i);
3129 				get_page(sg_page(sge));
3130 				sg_set_page(nsge,
3131 					    sg_page(sge),
3132 					    b, sge->offset + pop + a);
3133 				sg_unmark_end(nsge);
3134 				sk_msg_set_elem_copy(msg, i, sge_copy);
3135 			} else {
3136 				struct page *page, *orig;
3137 				u8 *to, *from;
3138 
3139 				page = alloc_pages(__GFP_NOWARN |
3140 						   __GFP_COMP   | GFP_ATOMIC,
3141 						   get_order(a + b));
3142 				if (unlikely(!page))
3143 					return -ENOMEM;
3144 
3145 				orig = sg_page(sge);
3146 				from = sg_virt(sge);
3147 				to = page_address(page);
3148 				memcpy(to, from, a);
3149 				memcpy(to + a, from + a + pop, b);
3150 				sg_set_page(sge, page, a + b, 0);
3151 				sk_msg_clear_elem_copy(msg, sge_idx);
3152 				put_page(orig);
3153 			}
3154 			pop = 0;
3155 		} else {
3156 			pop -= (sge->length - a);
3157 			sge->length = a;
3158 		}
3159 	}
3160 
3161 	/* From above the current layout _must_ be as follows,
3162 	 *
3163 	 * -| offset
3164 	 * -| start
3165 	 *
3166 	 *  |---- pop ---|---------------- b ------------|
3167 	 *  |____________________________________________| length
3168 	 *
3169 	 * Offset and start of the current msg elem are equal because in the
3170 	 * previous case we handled offset != start and either consumed the
3171 	 * entire element and advanced to the next element OR pop == 0.
3172 	 *
3173 	 * Two cases to handle here are first pop is less than the length
3174 	 * leaving some remainder b above. Simply adjust the element's layout
3175 	 * in this case. Or pop >= length of the element so that b = 0. In this
3176 	 * case advance to next element decrementing pop.
3177 	 */
3178 	while (pop) {
3179 		struct scatterlist *sge = sk_msg_elem(msg, i);
3180 
3181 		if (pop < sge->length) {
3182 			sge->length -= pop;
3183 			sge->offset += pop;
3184 			pop = 0;
3185 		} else {
3186 			pop -= sge->length;
3187 			sk_msg_shift_left(msg, i);
3188 		}
3189 	}
3190 
3191 	sk_mem_uncharge(msg->sk, len - pop);
3192 	msg->sg.size -= (len - pop);
3193 	sk_msg_reset_curr(msg);
3194 	sk_msg_compute_data_pointers(msg);
3195 	return 0;
3196 }
3197 
3198 static const struct bpf_func_proto bpf_msg_pop_data_proto = {
3199 	.func		= bpf_msg_pop_data,
3200 	.gpl_only	= false,
3201 	.ret_type	= RET_INTEGER,
3202 	.arg1_type	= ARG_PTR_TO_CTX,
3203 	.arg2_type	= ARG_ANYTHING,
3204 	.arg3_type	= ARG_ANYTHING,
3205 	.arg4_type	= ARG_ANYTHING,
3206 };
3207 
3208 #ifdef CONFIG_CGROUP_NET_CLASSID
3209 BPF_CALL_0(bpf_get_cgroup_classid_curr)
3210 {
3211 	return __task_get_classid(current);
3212 }
3213 
3214 const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
3215 	.func		= bpf_get_cgroup_classid_curr,
3216 	.gpl_only	= false,
3217 	.ret_type	= RET_INTEGER,
3218 };
3219 
3220 BPF_CALL_1(bpf_skb_cgroup_classid, const struct sk_buff *, skb)
3221 {
3222 	struct sock *sk = skb_to_full_sk(skb);
3223 
3224 	if (!sk || !sk_fullsock(sk))
3225 		return 0;
3226 
3227 	return sock_cgroup_classid(&sk->sk_cgrp_data);
3228 }
3229 
3230 static const struct bpf_func_proto bpf_skb_cgroup_classid_proto = {
3231 	.func		= bpf_skb_cgroup_classid,
3232 	.gpl_only	= false,
3233 	.ret_type	= RET_INTEGER,
3234 	.arg1_type	= ARG_PTR_TO_CTX,
3235 };
3236 #endif
3237 
3238 BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
3239 {
3240 	return task_get_classid(skb);
3241 }
3242 
3243 static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
3244 	.func           = bpf_get_cgroup_classid,
3245 	.gpl_only       = false,
3246 	.ret_type       = RET_INTEGER,
3247 	.arg1_type      = ARG_PTR_TO_CTX,
3248 };
3249 
3250 BPF_CALL_1(bpf_get_route_realm, const struct sk_buff *, skb)
3251 {
3252 	return dst_tclassid(skb);
3253 }
3254 
3255 static const struct bpf_func_proto bpf_get_route_realm_proto = {
3256 	.func           = bpf_get_route_realm,
3257 	.gpl_only       = false,
3258 	.ret_type       = RET_INTEGER,
3259 	.arg1_type      = ARG_PTR_TO_CTX,
3260 };
3261 
3262 BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb)
3263 {
3264 	/* If skb_clear_hash() was called due to mangling, we can
3265 	 * trigger SW recalculation here. Later access to hash
3266 	 * can then use the inline skb->hash via context directly
3267 	 * instead of calling this helper again.
3268 	 */
3269 	return skb_get_hash(skb);
3270 }
3271 
3272 static const struct bpf_func_proto bpf_get_hash_recalc_proto = {
3273 	.func		= bpf_get_hash_recalc,
3274 	.gpl_only	= false,
3275 	.ret_type	= RET_INTEGER,
3276 	.arg1_type	= ARG_PTR_TO_CTX,
3277 };
3278 
3279 BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb)
3280 {
3281 	/* After all direct packet write, this can be used once for
3282 	 * triggering a lazy recalc on next skb_get_hash() invocation.
3283 	 */
3284 	skb_clear_hash(skb);
3285 	return 0;
3286 }
3287 
3288 static const struct bpf_func_proto bpf_set_hash_invalid_proto = {
3289 	.func		= bpf_set_hash_invalid,
3290 	.gpl_only	= false,
3291 	.ret_type	= RET_INTEGER,
3292 	.arg1_type	= ARG_PTR_TO_CTX,
3293 };
3294 
3295 BPF_CALL_2(bpf_set_hash, struct sk_buff *, skb, u32, hash)
3296 {
3297 	/* Set user specified hash as L4(+), so that it gets returned
3298 	 * on skb_get_hash() call unless BPF prog later on triggers a
3299 	 * skb_clear_hash().
3300 	 */
3301 	__skb_set_sw_hash(skb, hash, true);
3302 	return 0;
3303 }
3304 
3305 static const struct bpf_func_proto bpf_set_hash_proto = {
3306 	.func		= bpf_set_hash,
3307 	.gpl_only	= false,
3308 	.ret_type	= RET_INTEGER,
3309 	.arg1_type	= ARG_PTR_TO_CTX,
3310 	.arg2_type	= ARG_ANYTHING,
3311 };
3312 
3313 BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto,
3314 	   u16, vlan_tci)
3315 {
3316 	int ret;
3317 
3318 	if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
3319 		     vlan_proto != htons(ETH_P_8021AD)))
3320 		vlan_proto = htons(ETH_P_8021Q);
3321 
3322 	bpf_push_mac_rcsum(skb);
3323 	ret = skb_vlan_push(skb, vlan_proto, vlan_tci);
3324 	bpf_pull_mac_rcsum(skb);
3325 	skb_reset_mac_len(skb);
3326 
3327 	bpf_compute_data_pointers(skb);
3328 	return ret;
3329 }
3330 
3331 static const struct bpf_func_proto bpf_skb_vlan_push_proto = {
3332 	.func           = bpf_skb_vlan_push,
3333 	.gpl_only       = false,
3334 	.ret_type       = RET_INTEGER,
3335 	.arg1_type      = ARG_PTR_TO_CTX,
3336 	.arg2_type      = ARG_ANYTHING,
3337 	.arg3_type      = ARG_ANYTHING,
3338 };
3339 
3340 BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
3341 {
3342 	int ret;
3343 
3344 	bpf_push_mac_rcsum(skb);
3345 	ret = skb_vlan_pop(skb);
3346 	bpf_pull_mac_rcsum(skb);
3347 
3348 	bpf_compute_data_pointers(skb);
3349 	return ret;
3350 }
3351 
3352 static const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
3353 	.func           = bpf_skb_vlan_pop,
3354 	.gpl_only       = false,
3355 	.ret_type       = RET_INTEGER,
3356 	.arg1_type      = ARG_PTR_TO_CTX,
3357 };
3358 
3359 static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
3360 {
3361 	/* The bytes in front of @off are moved with skb->data. */
3362 	if (unlikely(off > skb->len || !pskb_may_pull(skb, off)))
3363 		return -ENOMEM;
3364 
3365 	/* Caller already did skb_cow() with meta_len+len as headroom,
3366 	 * so no need to do it here.
3367 	 */
3368 	skb_push(skb, len);
3369 	skb_postpush_data_move(skb, len, off);
3370 	memset(skb->data + off, 0, len);
3371 
3372 	/* No skb_postpush_rcsum(skb, skb->data + off, len)
3373 	 * needed here as it does not change the skb->csum
3374 	 * result for checksum complete when summing over
3375 	 * zeroed blocks.
3376 	 */
3377 	return 0;
3378 }
3379 
3380 static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
3381 {
3382 	void *old_data;
3383 
3384 	/* skb_ensure_writable() is not needed here, as we're
3385 	 * already working on an uncloned skb. The first test
3386 	 * keeps off + len from wrapping.
3387 	 */
3388 	if (unlikely(off > skb->len || !pskb_may_pull(skb, off + len)))
3389 		return -ENOMEM;
3390 
3391 	old_data = skb->data;
3392 	__skb_pull(skb, len);
3393 	skb_postpull_rcsum(skb, old_data + off, len);
3394 	skb_postpull_data_move(skb, len, off);
3395 
3396 	return 0;
3397 }
3398 
3399 /*
3400  * bpf_skb_net_hdr_push() inserts bytes at @off, bpf_skb_net_hdr_pop() removes
3401  * bytes at @off. Both move the bytes in front of @off together with skb->data,
3402  * and leave the rest of the packet where it is. Return true if the transport
3403  * header is in the part that moves. A transport header that is equal to the
3404  * network header stays equal to it.
3405  */
3406 static bool bpf_skb_trans_hdr_moves(const struct sk_buff *skb, u32 off)
3407 {
3408 	if (skb->transport_header == skb->network_header)
3409 		return true;
3410 
3411 	return skb_transport_header_was_set(skb) &&
3412 	       (u32)skb_transport_offset(skb) < off;
3413 }
3414 
3415 static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
3416 {
3417 	bool trans_moves = bpf_skb_trans_hdr_moves(skb, off);
3418 	int ret;
3419 
3420 	/* There's no need for __skb_push()/__skb_pull() pair to
3421 	 * get to the start of the mac header as we're guaranteed
3422 	 * to always start from here under eBPF.
3423 	 */
3424 	ret = bpf_skb_generic_push(skb, off, len);
3425 	if (likely(!ret)) {
3426 		skb->mac_header -= len;
3427 		skb->network_header -= len;
3428 		if (trans_moves)
3429 			skb->transport_header -= len;
3430 	}
3431 
3432 	return ret;
3433 }
3434 
3435 static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
3436 {
3437 	bool trans_moves = bpf_skb_trans_hdr_moves(skb, off);
3438 	int ret;
3439 
3440 	/* Same here, __skb_push()/__skb_pull() pair not needed. */
3441 	ret = bpf_skb_generic_pop(skb, off, len);
3442 	if (likely(!ret)) {
3443 		skb->mac_header += len;
3444 		skb->network_header += len;
3445 		if (trans_moves) {
3446 			skb->transport_header += len;
3447 		} else if (skb_transport_header_was_set(skb) &&
3448 			   skb_transport_offset(skb) < (int)off) {
3449 			/* It was in the removed bytes or before skb->data. */
3450 			skb_set_transport_header(skb, off);
3451 		}
3452 		/* Do not leave the inner transport header in front of @off. */
3453 		if (skb->encapsulation &&
3454 		    skb_inner_transport_offset(skb) < (int)off)
3455 			skb_set_inner_transport_header(skb, off);
3456 	}
3457 
3458 	return ret;
3459 }
3460 
3461 static int bpf_skb_proto_4_to_6(struct sk_buff *skb)
3462 {
3463 	const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3464 	const u8 meta_len = skb_metadata_len(skb);
3465 	u32 off = skb_mac_header_len(skb);
3466 	int ret;
3467 
3468 	ret = skb_cow(skb, meta_len + len_diff);
3469 	if (unlikely(ret < 0))
3470 		return ret;
3471 
3472 	ret = bpf_skb_net_hdr_push(skb, off, len_diff);
3473 	if (unlikely(ret < 0))
3474 		return ret;
3475 
3476 	if (skb_is_gso(skb)) {
3477 		struct skb_shared_info *shinfo = skb_shinfo(skb);
3478 
3479 		/* SKB_GSO_TCPV4 needs to be changed into SKB_GSO_TCPV6. */
3480 		if (shinfo->gso_type & SKB_GSO_TCPV4) {
3481 			shinfo->gso_type &= ~SKB_GSO_TCPV4;
3482 			shinfo->gso_type |=  SKB_GSO_TCPV6;
3483 		}
3484 		shinfo->gso_type |=  SKB_GSO_DODGY;
3485 	}
3486 
3487 	skb->protocol = htons(ETH_P_IPV6);
3488 	skb_clear_hash(skb);
3489 
3490 	return 0;
3491 }
3492 
3493 static int bpf_skb_proto_6_to_4(struct sk_buff *skb)
3494 {
3495 	const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
3496 	u32 off = skb_mac_header_len(skb);
3497 	int ret;
3498 
3499 	ret = skb_unclone(skb, GFP_ATOMIC);
3500 	if (unlikely(ret < 0))
3501 		return ret;
3502 
3503 	ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
3504 	if (unlikely(ret < 0))
3505 		return ret;
3506 
3507 	if (skb_is_gso(skb)) {
3508 		struct skb_shared_info *shinfo = skb_shinfo(skb);
3509 
3510 		/* SKB_GSO_TCPV6 needs to be changed into SKB_GSO_TCPV4. */
3511 		if (shinfo->gso_type & SKB_GSO_TCPV6) {
3512 			shinfo->gso_type &= ~SKB_GSO_TCPV6;
3513 			shinfo->gso_type |=  SKB_GSO_TCPV4;
3514 		}
3515 		shinfo->gso_type |=  SKB_GSO_DODGY;
3516 	}
3517 
3518 	skb->protocol = htons(ETH_P_IP);
3519 	skb_clear_hash(skb);
3520 
3521 	return 0;
3522 }
3523 
3524 static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto)
3525 {
3526 	__be16 from_proto = skb->protocol;
3527 
3528 	if (from_proto == htons(ETH_P_IP) &&
3529 	      to_proto == htons(ETH_P_IPV6))
3530 		return bpf_skb_proto_4_to_6(skb);
3531 
3532 	if (from_proto == htons(ETH_P_IPV6) &&
3533 	      to_proto == htons(ETH_P_IP))
3534 		return bpf_skb_proto_6_to_4(skb);
3535 
3536 	return -ENOTSUPP;
3537 }
3538 
3539 BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto,
3540 	   u64, flags)
3541 {
3542 	int ret;
3543 
3544 	if (unlikely(flags))
3545 		return -EINVAL;
3546 
3547 	/* General idea is that this helper does the basic groundwork
3548 	 * needed for changing the protocol, and eBPF program fills the
3549 	 * rest through bpf_skb_store_bytes(), bpf_lX_csum_replace()
3550 	 * and other helpers, rather than passing a raw buffer here.
3551 	 *
3552 	 * The rationale is to keep this minimal and without a need to
3553 	 * deal with raw packet data. F.e. even if we would pass buffers
3554 	 * here, the program still needs to call the bpf_lX_csum_replace()
3555 	 * helpers anyway. Plus, this way we keep also separation of
3556 	 * concerns, since f.e. bpf_skb_store_bytes() should only take
3557 	 * care of stores.
3558 	 *
3559 	 * Currently, additional options and extension header space are
3560 	 * not supported, but flags register is reserved so we can adapt
3561 	 * that. For offloads, we mark packet as dodgy, so that headers
3562 	 * need to be verified first.
3563 	 */
3564 	ret = bpf_skb_proto_xlat(skb, proto);
3565 	bpf_compute_data_pointers(skb);
3566 	if (ret)
3567 		return ret;
3568 
3569 	if (skb_valid_dst(skb))
3570 		skb_dst_drop(skb);
3571 
3572 	return 0;
3573 }
3574 
3575 static const struct bpf_func_proto bpf_skb_change_proto_proto = {
3576 	.func		= bpf_skb_change_proto,
3577 	.gpl_only	= false,
3578 	.ret_type	= RET_INTEGER,
3579 	.arg1_type	= ARG_PTR_TO_CTX,
3580 	.arg2_type	= ARG_ANYTHING,
3581 	.arg3_type	= ARG_ANYTHING,
3582 };
3583 
3584 BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type)
3585 {
3586 	/* We only allow a restricted subset to be changed for now. */
3587 	if (unlikely(!skb_pkt_type_ok(skb->pkt_type) ||
3588 		     !skb_pkt_type_ok(pkt_type)))
3589 		return -EINVAL;
3590 
3591 	skb->pkt_type = pkt_type;
3592 	return 0;
3593 }
3594 
3595 static const struct bpf_func_proto bpf_skb_change_type_proto = {
3596 	.func		= bpf_skb_change_type,
3597 	.gpl_only	= false,
3598 	.ret_type	= RET_INTEGER,
3599 	.arg1_type	= ARG_PTR_TO_CTX,
3600 	.arg2_type	= ARG_ANYTHING,
3601 };
3602 
3603 static u32 bpf_skb_net_base_len(const struct sk_buff *skb)
3604 {
3605 	switch (skb->protocol) {
3606 	case htons(ETH_P_IP):
3607 		return sizeof(struct iphdr);
3608 	case htons(ETH_P_IPV6):
3609 		return sizeof(struct ipv6hdr);
3610 	default:
3611 		return ~0U;
3612 	}
3613 }
3614 
3615 #define BPF_F_ADJ_ROOM_ENCAP_L3_MASK	(BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | \
3616 					 BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3617 
3618 #define BPF_F_ADJ_ROOM_DECAP_L3_MASK	(BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | \
3619 					 BPF_F_ADJ_ROOM_DECAP_L3_IPV6)
3620 
3621 #define BPF_F_ADJ_ROOM_DECAP_L4_MASK	(BPF_F_ADJ_ROOM_DECAP_L4_UDP | \
3622 					 BPF_F_ADJ_ROOM_DECAP_L4_GRE)
3623 
3624 #define BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK	(BPF_F_ADJ_ROOM_DECAP_IPXIP4 | \
3625 					 BPF_F_ADJ_ROOM_DECAP_IPXIP6)
3626 
3627 #define BPF_F_ADJ_ROOM_ENCAP_MASK	(BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \
3628 					 BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \
3629 					 BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \
3630 					 BPF_F_ADJ_ROOM_ENCAP_L2_ETH | \
3631 					 BPF_F_ADJ_ROOM_ENCAP_L2( \
3632 					  BPF_ADJ_ROOM_ENCAP_L2_MASK))
3633 
3634 #define BPF_F_ADJ_ROOM_DECAP_MASK	(BPF_F_ADJ_ROOM_DECAP_L3_MASK | \
3635 					 BPF_F_ADJ_ROOM_DECAP_L4_MASK | \
3636 					 BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)
3637 
3638 #define BPF_F_ADJ_ROOM_MASK		(BPF_F_ADJ_ROOM_FIXED_GSO | \
3639 					 BPF_F_ADJ_ROOM_ENCAP_MASK | \
3640 					 BPF_F_ADJ_ROOM_DECAP_MASK | \
3641 					 BPF_F_ADJ_ROOM_NO_CSUM_RESET)
3642 
3643 static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
3644 			    u64 flags)
3645 {
3646 	u8 inner_mac_len = flags >> BPF_ADJ_ROOM_ENCAP_L2_SHIFT;
3647 	bool encap = flags & BPF_F_ADJ_ROOM_ENCAP_L3_MASK;
3648 	u16 mac_len = 0, inner_net = 0, inner_trans = 0;
3649 	const u8 meta_len = skb_metadata_len(skb);
3650 	unsigned int gso_type = SKB_GSO_DODGY;
3651 	int ret;
3652 
3653 	if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
3654 		/* udp gso_size delineates datagrams, only allow if fixed */
3655 		if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
3656 		    !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3657 			return -ENOTSUPP;
3658 	}
3659 
3660 	/*
3661 	 * The MSS is lowered by len_diff. Leave the least that the stack lets a
3662 	 * TCP GSO packet have, or gso_segs can overflow.
3663 	 */
3664 	if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) &&
3665 	    skb_shinfo(skb)->gso_size < len_diff + TCP_MIN_GSO_SIZE)
3666 		return -ENOTSUPP;
3667 
3668 	ret = skb_cow_head(skb, meta_len + len_diff);
3669 	if (unlikely(ret < 0))
3670 		return ret;
3671 
3672 	if (encap) {
3673 		if (skb->protocol != htons(ETH_P_IP) &&
3674 		    skb->protocol != htons(ETH_P_IPV6))
3675 			return -ENOTSUPP;
3676 
3677 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 &&
3678 		    flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3679 			return -EINVAL;
3680 
3681 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE &&
3682 		    flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
3683 			return -EINVAL;
3684 
3685 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH &&
3686 		    inner_mac_len < ETH_HLEN)
3687 			return -EINVAL;
3688 
3689 		if (skb->encapsulation)
3690 			return -EALREADY;
3691 
3692 		mac_len = skb->network_header - skb->mac_header;
3693 		inner_net = skb->network_header;
3694 		if (inner_mac_len > len_diff)
3695 			return -EINVAL;
3696 		inner_trans = skb->transport_header;
3697 	}
3698 
3699 	ret = bpf_skb_net_hdr_push(skb, off, len_diff);
3700 	if (unlikely(ret < 0))
3701 		return ret;
3702 
3703 	if (encap) {
3704 		skb->inner_mac_header = inner_net - inner_mac_len;
3705 		skb->inner_network_header = inner_net;
3706 		skb->inner_transport_header = inner_trans;
3707 
3708 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH)
3709 			skb_set_inner_protocol(skb, htons(ETH_P_TEB));
3710 		else
3711 			skb_set_inner_protocol(skb, skb->protocol);
3712 
3713 		skb->encapsulation = 1;
3714 		skb_set_network_header(skb, mac_len);
3715 
3716 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
3717 			gso_type |= SKB_GSO_UDP_TUNNEL;
3718 		else if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE)
3719 			gso_type |= SKB_GSO_GRE;
3720 		else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3721 			gso_type |= SKB_GSO_IPXIP6;
3722 		else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
3723 			gso_type |= SKB_GSO_IPXIP4;
3724 
3725 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE ||
3726 		    flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP) {
3727 			int nh_len = flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 ?
3728 					sizeof(struct ipv6hdr) :
3729 					sizeof(struct iphdr);
3730 
3731 			skb_set_transport_header(skb, mac_len + nh_len);
3732 		}
3733 
3734 		/* Match skb->protocol to new outer l3 protocol */
3735 		if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
3736 			skb->protocol = htons(ETH_P_IPV6);
3737 		else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
3738 			skb->protocol = htons(ETH_P_IP);
3739 
3740 		if (skb_valid_dst(skb))
3741 			skb_dst_drop(skb);
3742 	}
3743 
3744 	if (skb_is_gso(skb)) {
3745 		struct skb_shared_info *shinfo = skb_shinfo(skb);
3746 
3747 		/* Header must be checked, and gso_segs recomputed. */
3748 		shinfo->gso_type |= gso_type;
3749 		shinfo->gso_segs = 0;
3750 
3751 		/* Due to header growth, MSS needs to be downgraded.
3752 		 * There is a BUG_ON() when segmenting the frag_list with
3753 		 * head_frag true, so linearize the skb after downgrading
3754 		 * the MSS.
3755 		 */
3756 		if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO)) {
3757 			skb_decrease_gso_size(shinfo, len_diff);
3758 			if (shinfo->frag_list)
3759 				return skb_linearize(skb);
3760 		}
3761 	}
3762 
3763 	return 0;
3764 }
3765 
3766 /*
3767  * Code that went by the inner transport header while skb->encapsulation was set
3768  * goes by the transport header from now on. So take the inner one over, unless
3769  * it is the smaller one, as it is if nobody has set it. Do not leave the result
3770  * in front of @off, where the bytes were removed.
3771  */
3772 static void bpf_skb_clear_encap(struct sk_buff *skb, u32 off)
3773 {
3774 	int thoff;
3775 
3776 	if (!skb->encapsulation)
3777 		return;
3778 
3779 	thoff = max(skb->inner_transport_header, skb->transport_header) -
3780 		skb_headroom(skb);
3781 	skb_set_transport_header(skb, max_t(int, thoff, off));
3782 	skb->encapsulation = 0;
3783 }
3784 
3785 static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
3786 			      u64 flags)
3787 {
3788 	bool decap = flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK;
3789 	int ret;
3790 
3791 	if (unlikely(flags & ~(BPF_F_ADJ_ROOM_DECAP_MASK |
3792 			       BPF_F_ADJ_ROOM_FIXED_GSO |
3793 			       BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
3794 		return -EINVAL;
3795 
3796 	if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
3797 		/* udp gso_size delineates datagrams, only allow if fixed */
3798 		if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
3799 		    !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3800 			return -ENOTSUPP;
3801 	}
3802 
3803 	/*
3804 	 * The MSS is raised by len_diff. It has 16 bits, and GSO_BY_FRAGS is
3805 	 * not a size.
3806 	 */
3807 	if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) &&
3808 	    skb_shinfo(skb)->gso_size + len_diff >= GSO_BY_FRAGS)
3809 		return -ENOTSUPP;
3810 
3811 	ret = skb_unclone(skb, GFP_ATOMIC);
3812 	if (unlikely(ret < 0))
3813 		return ret;
3814 
3815 	ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
3816 	if (unlikely(ret < 0))
3817 		return ret;
3818 
3819 	if (decap) {
3820 		/* Match skb->protocol to new outer l3 protocol */
3821 		if (flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV6)
3822 			skb->protocol = htons(ETH_P_IPV6);
3823 		else if (flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV4)
3824 			skb->protocol = htons(ETH_P_IP);
3825 
3826 		if (skb_valid_dst(skb))
3827 			skb_dst_drop(skb);
3828 	}
3829 
3830 	if (skb_is_gso(skb)) {
3831 		struct skb_shared_info *shinfo = skb_shinfo(skb);
3832 
3833 		/* Due to header shrink, MSS can be upgraded. */
3834 		if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
3835 			skb_increase_gso_size(shinfo, len_diff);
3836 
3837 		/* Selective GSO flag clearing based on decap type.
3838 		 * Only clear the flags for the tunnel layer being removed.
3839 		 */
3840 		if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP) &&
3841 		    (shinfo->gso_type & (SKB_GSO_UDP_TUNNEL |
3842 					 SKB_GSO_UDP_TUNNEL_CSUM)))
3843 			shinfo->gso_type &= ~(SKB_GSO_UDP_TUNNEL |
3844 					      SKB_GSO_UDP_TUNNEL_CSUM);
3845 		if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE) &&
3846 		    (shinfo->gso_type & (SKB_GSO_GRE | SKB_GSO_GRE_CSUM)))
3847 			shinfo->gso_type &= ~(SKB_GSO_GRE |
3848 					      SKB_GSO_GRE_CSUM);
3849 		if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4) &&
3850 		    (shinfo->gso_type & SKB_GSO_IPXIP4))
3851 			shinfo->gso_type &= ~SKB_GSO_IPXIP4;
3852 		if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6) &&
3853 		    (shinfo->gso_type & SKB_GSO_IPXIP6))
3854 			shinfo->gso_type &= ~SKB_GSO_IPXIP6;
3855 
3856 		/* Clear encapsulation flag only when no tunnel GSO flags remain */
3857 		if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK |
3858 			     BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) {
3859 			if (!(shinfo->gso_type & (SKB_GSO_UDP_TUNNEL |
3860 						  SKB_GSO_UDP_TUNNEL_CSUM |
3861 						  SKB_GSO_GRE |
3862 						  SKB_GSO_GRE_CSUM |
3863 						  SKB_GSO_IPXIP4 |
3864 						  SKB_GSO_IPXIP6 |
3865 						  SKB_GSO_ESP)))
3866 				bpf_skb_clear_encap(skb, off);
3867 		}
3868 
3869 		/* Header must be checked, and gso_segs recomputed. */
3870 		shinfo->gso_type |= SKB_GSO_DODGY;
3871 		shinfo->gso_segs = 0;
3872 	} else {
3873 		/* For non-GSO packets, clear encapsulation if decap flags are set */
3874 		if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK |
3875 			     BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK))
3876 			bpf_skb_clear_encap(skb, off);
3877 	}
3878 
3879 	return 0;
3880 }
3881 
3882 #define BPF_SKB_MAX_LEN SKB_MAX_ALLOC
3883 
3884 BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3885 	   u32, mode, u64, flags)
3886 {
3887 	u32 len_diff_abs = abs(len_diff);
3888 	bool shrink = len_diff < 0;
3889 	int ret = 0;
3890 
3891 	if (unlikely(flags || mode))
3892 		return -EINVAL;
3893 	if (unlikely(len_diff_abs > 0xfffU))
3894 		return -EFAULT;
3895 
3896 	if (!shrink) {
3897 		ret = skb_cow(skb, len_diff);
3898 		if (unlikely(ret < 0))
3899 			return ret;
3900 		__skb_push(skb, len_diff_abs);
3901 		memset(skb->data, 0, len_diff_abs);
3902 	} else {
3903 		if (unlikely(!pskb_may_pull(skb, len_diff_abs)))
3904 			return -ENOMEM;
3905 		__skb_pull(skb, len_diff_abs);
3906 	}
3907 	if (tls_sw_has_ctx_rx(skb->sk)) {
3908 		struct strp_msg *rxm = strp_msg(skb);
3909 
3910 		rxm->full_len += len_diff;
3911 	}
3912 	return ret;
3913 }
3914 
3915 static const struct bpf_func_proto sk_skb_adjust_room_proto = {
3916 	.func		= sk_skb_adjust_room,
3917 	.gpl_only	= false,
3918 	.ret_type	= RET_INTEGER,
3919 	.arg1_type	= ARG_PTR_TO_CTX,
3920 	.arg2_type	= ARG_ANYTHING,
3921 	.arg3_type	= ARG_ANYTHING,
3922 	.arg4_type	= ARG_ANYTHING,
3923 };
3924 
3925 BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3926 	   u32, mode, u64, flags)
3927 {
3928 	u32 len_cur, len_diff_abs = abs(len_diff);
3929 	u32 len_min = bpf_skb_net_base_len(skb);
3930 	u32 len_max = BPF_SKB_MAX_LEN;
3931 	__be16 proto = skb->protocol;
3932 	bool shrink = len_diff < 0;
3933 	u32 off;
3934 	int ret;
3935 
3936 	if (unlikely(flags & ~BPF_F_ADJ_ROOM_MASK))
3937 		return -EINVAL;
3938 	if (unlikely(len_diff_abs > 0xfffU))
3939 		return -EFAULT;
3940 	if (unlikely(proto != htons(ETH_P_IP) &&
3941 		     proto != htons(ETH_P_IPV6)))
3942 		return -ENOTSUPP;
3943 
3944 	off = skb_mac_header_len(skb);
3945 	switch (mode) {
3946 	case BPF_ADJ_ROOM_NET:
3947 		off += bpf_skb_net_base_len(skb);
3948 		break;
3949 	case BPF_ADJ_ROOM_MAC:
3950 		break;
3951 	default:
3952 		return -ENOTSUPP;
3953 	}
3954 
3955 	if (flags & BPF_F_ADJ_ROOM_DECAP_MASK) {
3956 		u32 len_decap_min = 0;
3957 
3958 		if (!shrink)
3959 			return -EINVAL;
3960 
3961 		/* Reject mutually exclusive decap flag pairs. */
3962 		if ((flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) ==
3963 		    BPF_F_ADJ_ROOM_DECAP_L3_MASK)
3964 			return -EINVAL;
3965 
3966 		if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_MASK) ==
3967 		    BPF_F_ADJ_ROOM_DECAP_L4_MASK)
3968 			return -EINVAL;
3969 
3970 		if ((flags & BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK) ==
3971 		    BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)
3972 			return -EINVAL;
3973 
3974 		/* Reject mutually exclusive decap tunnel type flags. */
3975 		if ((flags & BPF_F_ADJ_ROOM_DECAP_L4_MASK) &&
3976 		    (flags & BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK))
3977 			return -EINVAL;
3978 
3979 		if (flags & BPF_F_ADJ_ROOM_DECAP_L4_MASK)
3980 			len_decap_min += bpf_skb_net_base_len(skb);
3981 
3982 		if (flags & BPF_F_ADJ_ROOM_DECAP_L4_UDP)
3983 			len_decap_min += sizeof(struct udphdr);
3984 
3985 		if (flags & BPF_F_ADJ_ROOM_DECAP_L4_GRE)
3986 			len_decap_min += sizeof(struct gre_base_hdr);
3987 
3988 		if (flags & BPF_F_ADJ_ROOM_DECAP_IPXIP4)
3989 			len_decap_min += sizeof(struct iphdr);
3990 
3991 		if (flags & BPF_F_ADJ_ROOM_DECAP_IPXIP6)
3992 			len_decap_min += sizeof(struct ipv6hdr);
3993 
3994 		if (len_diff_abs < len_decap_min)
3995 			return -EINVAL;
3996 
3997 		if (flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV4)
3998 			len_min = sizeof(struct iphdr);
3999 
4000 		if (flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV6)
4001 			len_min = sizeof(struct ipv6hdr);
4002 	}
4003 
4004 	len_cur = skb->len - skb_network_offset(skb);
4005 	if ((shrink && (len_diff_abs >= len_cur ||
4006 			len_cur - len_diff_abs < len_min)) ||
4007 	    (!shrink && (skb->len + len_diff_abs > len_max &&
4008 			 !skb_is_gso(skb))))
4009 		return -ENOTSUPP;
4010 
4011 	ret = shrink ? bpf_skb_net_shrink(skb, off, len_diff_abs, flags) :
4012 		       bpf_skb_net_grow(skb, off, len_diff_abs, flags);
4013 	if (!ret && !(flags & BPF_F_ADJ_ROOM_NO_CSUM_RESET))
4014 		__skb_reset_checksum_unnecessary(skb);
4015 
4016 	bpf_compute_data_pointers(skb);
4017 	return ret;
4018 }
4019 
4020 static const struct bpf_func_proto bpf_skb_adjust_room_proto = {
4021 	.func		= bpf_skb_adjust_room,
4022 	.gpl_only	= false,
4023 	.ret_type	= RET_INTEGER,
4024 	.arg1_type	= ARG_PTR_TO_CTX,
4025 	.arg2_type	= ARG_ANYTHING,
4026 	.arg3_type	= ARG_ANYTHING,
4027 	.arg4_type	= ARG_ANYTHING,
4028 };
4029 
4030 static u32 __bpf_skb_min_len(const struct sk_buff *skb)
4031 {
4032 	int offset = skb_network_offset(skb);
4033 	u32 min_len = 0;
4034 
4035 	if (offset > 0)
4036 		min_len = offset;
4037 	if (skb_transport_header_was_set(skb)) {
4038 		offset = skb_transport_offset(skb);
4039 		if (offset > 0)
4040 			min_len = offset;
4041 	}
4042 	return min_len;
4043 }
4044 
4045 static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len)
4046 {
4047 	unsigned int old_len = skb->len;
4048 	int ret;
4049 
4050 	ret = __skb_grow_rcsum(skb, new_len);
4051 	if (!ret)
4052 		memset(skb->data + old_len, 0, new_len - old_len);
4053 	return ret;
4054 }
4055 
4056 static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len)
4057 {
4058 	if (skb->ip_summed == CHECKSUM_PARTIAL &&
4059 	    new_len < skb_checksum_start_offset(skb) + skb->csum_offset +
4060 		      sizeof(__sum16))
4061 		skb->ip_summed = CHECKSUM_NONE;
4062 
4063 	return __skb_trim_rcsum(skb, new_len);
4064 }
4065 
4066 static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,
4067 					u64 flags)
4068 {
4069 	u32 max_len = BPF_SKB_MAX_LEN;
4070 	u32 min_len = __bpf_skb_min_len(skb);
4071 	int ret;
4072 
4073 	if (unlikely(flags || new_len > max_len || new_len < min_len))
4074 		return -EINVAL;
4075 	if (skb->encapsulation)
4076 		return -ENOTSUPP;
4077 
4078 	/* The basic idea of this helper is that it's performing the
4079 	 * needed work to either grow or trim an skb, and eBPF program
4080 	 * rewrites the rest via helpers like bpf_skb_store_bytes(),
4081 	 * bpf_lX_csum_replace() and others rather than passing a raw
4082 	 * buffer here. This one is a slow path helper and intended
4083 	 * for replies with control messages.
4084 	 *
4085 	 * Like in bpf_skb_change_proto(), we want to keep this rather
4086 	 * minimal and without protocol specifics so that we are able
4087 	 * to separate concerns as in bpf_skb_store_bytes() should only
4088 	 * be the one responsible for writing buffers.
4089 	 *
4090 	 * It's really expected to be a slow path operation here for
4091 	 * control message replies, so we're implicitly linearizing,
4092 	 * uncloning and drop offloads from the skb by this.
4093 	 */
4094 	ret = __bpf_try_make_writable(skb, skb->len);
4095 	if (!ret) {
4096 		if (new_len > skb->len)
4097 			ret = bpf_skb_grow_rcsum(skb, new_len);
4098 		else if (new_len < skb->len)
4099 			ret = bpf_skb_trim_rcsum(skb, new_len);
4100 		if (!ret && skb_is_gso(skb))
4101 			skb_gso_reset(skb);
4102 	}
4103 	return ret;
4104 }
4105 
4106 BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len,
4107 	   u64, flags)
4108 {
4109 	int ret = __bpf_skb_change_tail(skb, new_len, flags);
4110 
4111 	bpf_compute_data_pointers(skb);
4112 	return ret;
4113 }
4114 
4115 static const struct bpf_func_proto bpf_skb_change_tail_proto = {
4116 	.func		= bpf_skb_change_tail,
4117 	.gpl_only	= false,
4118 	.ret_type	= RET_INTEGER,
4119 	.arg1_type	= ARG_PTR_TO_CTX,
4120 	.arg2_type	= ARG_ANYTHING,
4121 	.arg3_type	= ARG_ANYTHING,
4122 };
4123 
4124 BPF_CALL_3(sk_skb_change_tail, struct sk_buff *, skb, u32, new_len,
4125 	   u64, flags)
4126 {
4127 	return __bpf_skb_change_tail(skb, new_len, flags);
4128 }
4129 
4130 static const struct bpf_func_proto sk_skb_change_tail_proto = {
4131 	.func		= sk_skb_change_tail,
4132 	.gpl_only	= false,
4133 	.ret_type	= RET_INTEGER,
4134 	.arg1_type	= ARG_PTR_TO_CTX,
4135 	.arg2_type	= ARG_ANYTHING,
4136 	.arg3_type	= ARG_ANYTHING,
4137 };
4138 
4139 static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,
4140 					u64 flags)
4141 {
4142 	const u8 meta_len = skb_metadata_len(skb);
4143 	u32 max_len = BPF_SKB_MAX_LEN;
4144 	u32 new_len = skb->len + head_room;
4145 	int ret;
4146 
4147 	if (unlikely(flags || (int)head_room < 0 ||
4148 		     (!skb_is_gso(skb) && new_len > max_len) ||
4149 		     new_len < skb->len))
4150 		return -EINVAL;
4151 
4152 	ret = skb_cow(skb, meta_len + head_room);
4153 	if (likely(!ret)) {
4154 		/* Idea for this helper is that we currently only
4155 		 * allow to expand on mac header. This means that
4156 		 * skb->protocol network header, etc, stay as is.
4157 		 * Compared to bpf_skb_change_tail(), we're more
4158 		 * flexible due to not needing to linearize or
4159 		 * reset GSO. Intention for this helper is to be
4160 		 * used by an L3 skb that needs to push mac header
4161 		 * for redirection into L2 device.
4162 		 */
4163 		__skb_push(skb, head_room);
4164 		skb_postpush_data_move(skb, head_room, 0);
4165 		memset(skb->data, 0, head_room);
4166 		skb_reset_mac_header(skb);
4167 		skb_reset_mac_len(skb);
4168 	}
4169 
4170 	return ret;
4171 }
4172 
4173 BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room,
4174 	   u64, flags)
4175 {
4176 	int ret = __bpf_skb_change_head(skb, head_room, flags);
4177 
4178 	bpf_compute_data_pointers(skb);
4179 	return ret;
4180 }
4181 
4182 static const struct bpf_func_proto bpf_skb_change_head_proto = {
4183 	.func		= bpf_skb_change_head,
4184 	.gpl_only	= false,
4185 	.ret_type	= RET_INTEGER,
4186 	.arg1_type	= ARG_PTR_TO_CTX,
4187 	.arg2_type	= ARG_ANYTHING,
4188 	.arg3_type	= ARG_ANYTHING,
4189 };
4190 
4191 BPF_CALL_3(sk_skb_change_head, struct sk_buff *, skb, u32, head_room,
4192 	   u64, flags)
4193 {
4194 	return __bpf_skb_change_head(skb, head_room, flags);
4195 }
4196 
4197 static const struct bpf_func_proto sk_skb_change_head_proto = {
4198 	.func		= sk_skb_change_head,
4199 	.gpl_only	= false,
4200 	.ret_type	= RET_INTEGER,
4201 	.arg1_type	= ARG_PTR_TO_CTX,
4202 	.arg2_type	= ARG_ANYTHING,
4203 	.arg3_type	= ARG_ANYTHING,
4204 };
4205 
4206 BPF_CALL_1(bpf_xdp_get_buff_len, struct xdp_buff*, xdp)
4207 {
4208 	return xdp_get_buff_len(xdp);
4209 }
4210 
4211 static const struct bpf_func_proto bpf_xdp_get_buff_len_proto = {
4212 	.func		= bpf_xdp_get_buff_len,
4213 	.gpl_only	= false,
4214 	.ret_type	= RET_INTEGER,
4215 	.arg1_type	= ARG_PTR_TO_CTX,
4216 };
4217 
4218 BTF_ID_LIST_SINGLE(bpf_xdp_get_buff_len_bpf_ids, struct, xdp_buff)
4219 
4220 const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto = {
4221 	.func		= bpf_xdp_get_buff_len,
4222 	.gpl_only	= false,
4223 	.arg1_type	= ARG_PTR_TO_BTF_ID,
4224 	.arg1_btf_id	= &bpf_xdp_get_buff_len_bpf_ids[0],
4225 };
4226 
4227 static unsigned long xdp_get_metalen(const struct xdp_buff *xdp)
4228 {
4229 	return xdp_data_meta_unsupported(xdp) ? 0 :
4230 	       xdp->data - xdp->data_meta;
4231 }
4232 
4233 BPF_CALL_2(bpf_xdp_adjust_head, struct xdp_buff *, xdp, int, offset)
4234 {
4235 	void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
4236 	unsigned long metalen = xdp_get_metalen(xdp);
4237 	void *data_start = xdp_frame_end + metalen;
4238 	void *data = xdp->data + offset;
4239 
4240 	if (unlikely(data < data_start ||
4241 		     data > xdp->data_end - ETH_HLEN))
4242 		return -EINVAL;
4243 
4244 	if (metalen)
4245 		memmove(xdp->data_meta + offset,
4246 			xdp->data_meta, metalen);
4247 	xdp->data_meta += offset;
4248 	xdp->data = data;
4249 
4250 	return 0;
4251 }
4252 
4253 static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
4254 	.func		= bpf_xdp_adjust_head,
4255 	.gpl_only	= false,
4256 	.ret_type	= RET_INTEGER,
4257 	.arg1_type	= ARG_PTR_TO_CTX,
4258 	.arg2_type	= ARG_ANYTHING,
4259 };
4260 
4261 void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
4262 		      void *buf, unsigned long len, bool flush)
4263 {
4264 	unsigned long ptr_len, ptr_off = 0;
4265 	skb_frag_t *next_frag, *end_frag;
4266 	struct skb_shared_info *sinfo;
4267 	void *src, *dst;
4268 	u8 *ptr_buf;
4269 
4270 	if (likely(xdp->data_end - xdp->data >= off + len)) {
4271 		src = flush ? buf : xdp->data + off;
4272 		dst = flush ? xdp->data + off : buf;
4273 		memcpy(dst, src, len);
4274 		return;
4275 	}
4276 
4277 	sinfo = xdp_get_shared_info_from_buff(xdp);
4278 	end_frag = &sinfo->frags[sinfo->nr_frags];
4279 	next_frag = &sinfo->frags[0];
4280 
4281 	ptr_len = xdp->data_end - xdp->data;
4282 	ptr_buf = xdp->data;
4283 
4284 	while (true) {
4285 		if (off < ptr_off + ptr_len) {
4286 			unsigned long copy_off = off - ptr_off;
4287 			unsigned long copy_len = min(len, ptr_len - copy_off);
4288 
4289 			src = flush ? buf : ptr_buf + copy_off;
4290 			dst = flush ? ptr_buf + copy_off : buf;
4291 			memcpy(dst, src, copy_len);
4292 
4293 			off += copy_len;
4294 			len -= copy_len;
4295 			buf += copy_len;
4296 		}
4297 
4298 		if (!len || next_frag == end_frag)
4299 			break;
4300 
4301 		ptr_off += ptr_len;
4302 		ptr_buf = skb_frag_address(next_frag);
4303 		ptr_len = skb_frag_size(next_frag);
4304 		next_frag++;
4305 	}
4306 }
4307 
4308 void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
4309 {
4310 	u32 size = xdp->data_end - xdp->data;
4311 	struct skb_shared_info *sinfo;
4312 	void *addr = xdp->data;
4313 	int i;
4314 
4315 	if (unlikely(offset > 0xffff || len > 0xffff))
4316 		return ERR_PTR(-EFAULT);
4317 
4318 	if (unlikely(offset + len > xdp_get_buff_len(xdp)))
4319 		return ERR_PTR(-EINVAL);
4320 
4321 	if (likely(offset < size)) /* linear area */
4322 		goto out;
4323 
4324 	sinfo = xdp_get_shared_info_from_buff(xdp);
4325 	offset -= size;
4326 	for (i = 0; i < sinfo->nr_frags; i++) { /* paged area */
4327 		u32 frag_size = skb_frag_size(&sinfo->frags[i]);
4328 
4329 		if  (offset < frag_size) {
4330 			addr = skb_frag_address(&sinfo->frags[i]);
4331 			size = frag_size;
4332 			break;
4333 		}
4334 		offset -= frag_size;
4335 	}
4336 out:
4337 	return offset + len <= size ? addr + offset : NULL;
4338 }
4339 
4340 BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
4341 	   void *, buf, u32, len)
4342 {
4343 	void *ptr;
4344 
4345 	ptr = bpf_xdp_pointer(xdp, offset, len);
4346 	if (IS_ERR(ptr))
4347 		return PTR_ERR(ptr);
4348 
4349 	if (!ptr)
4350 		bpf_xdp_copy_buf(xdp, offset, buf, len, false);
4351 	else
4352 		memmove(buf, ptr, len);
4353 
4354 	return 0;
4355 }
4356 
4357 static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {
4358 	.func		= bpf_xdp_load_bytes,
4359 	.gpl_only	= false,
4360 	.ret_type	= RET_INTEGER,
4361 	.arg1_type	= ARG_PTR_TO_CTX,
4362 	.arg2_type	= ARG_ANYTHING,
4363 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
4364 	.arg4_type	= ARG_MEM_SIZE,
4365 };
4366 
4367 int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
4368 {
4369 	return ____bpf_xdp_load_bytes(xdp, offset, buf, len);
4370 }
4371 
4372 BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
4373 	   void *, buf, u32, len)
4374 {
4375 	void *ptr;
4376 
4377 	ptr = bpf_xdp_pointer(xdp, offset, len);
4378 	if (IS_ERR(ptr))
4379 		return PTR_ERR(ptr);
4380 
4381 	if (!ptr)
4382 		bpf_xdp_copy_buf(xdp, offset, buf, len, true);
4383 	else
4384 		/* through a dynptr the source may be an overlapping slice */
4385 		memmove(ptr, buf, len);
4386 
4387 	return 0;
4388 }
4389 
4390 static const struct bpf_func_proto bpf_xdp_store_bytes_proto = {
4391 	.func		= bpf_xdp_store_bytes,
4392 	.gpl_only	= false,
4393 	.ret_type	= RET_INTEGER,
4394 	.arg1_type	= ARG_PTR_TO_CTX,
4395 	.arg2_type	= ARG_ANYTHING,
4396 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
4397 	.arg4_type	= ARG_MEM_SIZE,
4398 };
4399 
4400 int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
4401 {
4402 	return ____bpf_xdp_store_bytes(xdp, offset, buf, len);
4403 }
4404 
4405 static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
4406 {
4407 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
4408 	skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags - 1];
4409 	struct xdp_rxq_info *rxq = xdp->rxq;
4410 	int tailroom;
4411 
4412 	if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz)
4413 		return -EOPNOTSUPP;
4414 
4415 	tailroom = rxq->frag_size - skb_frag_size(frag) -
4416 		   skb_frag_off(frag) % rxq->frag_size;
4417 	WARN_ON_ONCE(tailroom < 0);
4418 	if (unlikely(offset > tailroom))
4419 		return -EINVAL;
4420 
4421 	memset(skb_frag_address(frag) + skb_frag_size(frag), 0, offset);
4422 	skb_frag_size_add(frag, offset);
4423 	sinfo->xdp_frags_size += offset;
4424 	if (rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
4425 		xsk_buff_get_tail(xdp)->data_end += offset;
4426 
4427 	return 0;
4428 }
4429 
4430 static struct xdp_buff *bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
4431 					       bool tail, bool release)
4432 {
4433 	struct xdp_buff *zc_frag = tail ? xsk_buff_get_tail(xdp) :
4434 					  xsk_buff_get_head(xdp);
4435 
4436 	if (release) {
4437 		xsk_buff_del_frag(zc_frag);
4438 	} else {
4439 		if (tail)
4440 			zc_frag->data_end -= shrink;
4441 		else
4442 			zc_frag->data += shrink;
4443 	}
4444 
4445 	return zc_frag;
4446 }
4447 
4448 static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
4449 				int shrink, bool tail)
4450 {
4451 	enum xdp_mem_type mem_type = xdp->rxq->mem.type;
4452 	bool release = skb_frag_size(frag) == shrink;
4453 	netmem_ref netmem = skb_frag_netmem(frag);
4454 	struct xdp_buff *zc_frag = NULL;
4455 
4456 	if (mem_type == MEM_TYPE_XSK_BUFF_POOL) {
4457 		netmem = 0;
4458 		zc_frag = bpf_xdp_shrink_data_zc(xdp, shrink, tail, release);
4459 	}
4460 
4461 	if (release) {
4462 		__xdp_return(netmem, mem_type, false, zc_frag);
4463 	} else {
4464 		if (!tail)
4465 			skb_frag_off_add(frag, shrink);
4466 		skb_frag_size_sub(frag, shrink);
4467 	}
4468 
4469 	return release;
4470 }
4471 
4472 static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset)
4473 {
4474 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
4475 	int i, n_frags_free = 0, len_free = 0;
4476 
4477 	if (unlikely(offset > (int)xdp_get_buff_len(xdp) - ETH_HLEN))
4478 		return -EINVAL;
4479 
4480 	for (i = sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) {
4481 		skb_frag_t *frag = &sinfo->frags[i];
4482 		int shrink = min_t(int, offset, skb_frag_size(frag));
4483 
4484 		len_free += shrink;
4485 		offset -= shrink;
4486 		if (bpf_xdp_shrink_data(xdp, frag, shrink, true))
4487 			n_frags_free++;
4488 	}
4489 	sinfo->nr_frags -= n_frags_free;
4490 	sinfo->xdp_frags_size -= len_free;
4491 
4492 	if (unlikely(!sinfo->nr_frags)) {
4493 		xdp_buff_clear_frags_flag(xdp);
4494 		xdp_buff_clear_frag_pfmemalloc(xdp);
4495 		xdp->data_end -= offset;
4496 	}
4497 
4498 	return 0;
4499 }
4500 
4501 BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset)
4502 {
4503 	void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */
4504 	void *data_end = xdp->data_end + offset;
4505 
4506 	if (unlikely(xdp_buff_has_frags(xdp))) { /* non-linear xdp buff */
4507 		if (offset < 0)
4508 			return bpf_xdp_frags_shrink_tail(xdp, -offset);
4509 
4510 		return bpf_xdp_frags_increase_tail(xdp, offset);
4511 	}
4512 
4513 	/* Notice that xdp_data_hard_end have reserved some tailroom */
4514 	if (unlikely(data_end > data_hard_end))
4515 		return -EINVAL;
4516 
4517 	if (unlikely(data_end < xdp->data + ETH_HLEN))
4518 		return -EINVAL;
4519 
4520 	/* Clear memory area on grow, can contain uninit kernel memory */
4521 	if (offset > 0)
4522 		memset(xdp->data_end, 0, offset);
4523 
4524 	xdp->data_end = data_end;
4525 
4526 	return 0;
4527 }
4528 
4529 static const struct bpf_func_proto bpf_xdp_adjust_tail_proto = {
4530 	.func		= bpf_xdp_adjust_tail,
4531 	.gpl_only	= false,
4532 	.ret_type	= RET_INTEGER,
4533 	.arg1_type	= ARG_PTR_TO_CTX,
4534 	.arg2_type	= ARG_ANYTHING,
4535 };
4536 
4537 BPF_CALL_2(bpf_xdp_adjust_meta, struct xdp_buff *, xdp, int, offset)
4538 {
4539 	void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
4540 	void *meta = xdp->data_meta + offset;
4541 	unsigned long metalen = xdp->data - meta;
4542 
4543 	if (xdp_data_meta_unsupported(xdp))
4544 		return -ENOTSUPP;
4545 	if (unlikely(meta < xdp_frame_end ||
4546 		     meta > xdp->data))
4547 		return -EINVAL;
4548 	if (unlikely(xdp_metalen_invalid(metalen)))
4549 		return -EACCES;
4550 
4551 	xdp->data_meta = meta;
4552 
4553 	return 0;
4554 }
4555 
4556 static const struct bpf_func_proto bpf_xdp_adjust_meta_proto = {
4557 	.func		= bpf_xdp_adjust_meta,
4558 	.gpl_only	= false,
4559 	.ret_type	= RET_INTEGER,
4560 	.arg1_type	= ARG_PTR_TO_CTX,
4561 	.arg2_type	= ARG_ANYTHING,
4562 };
4563 
4564 /**
4565  * DOC: xdp redirect
4566  *
4567  * XDP_REDIRECT works by a three-step process, implemented in the functions
4568  * below:
4569  *
4570  * 1. The bpf_redirect() and bpf_redirect_map() helpers will lookup the target
4571  *    of the redirect and store it (along with some other metadata) in a per-CPU
4572  *    struct bpf_redirect_info.
4573  *
4574  * 2. When the program returns the XDP_REDIRECT return code, the driver will
4575  *    call xdp_do_redirect() which will use the information in struct
4576  *    bpf_redirect_info to actually enqueue the frame into a map type-specific
4577  *    bulk queue structure.
4578  *
4579  * 3. Before exiting its NAPI poll loop, the driver will call
4580  *    xdp_do_flush(), which will flush all the different bulk queues,
4581  *    thus completing the redirect. Note that xdp_do_flush() must be
4582  *    called before napi_complete_done() in the driver, as the
4583  *    XDP_REDIRECT logic relies on being inside a single NAPI instance
4584  *    through to the xdp_do_flush() call for RCU protection of all
4585  *    in-kernel data structures.
4586  */
4587 /*
4588  * Pointers to the map entries will be kept around for this whole sequence of
4589  * steps, protected by RCU. However, there is no top-level rcu_read_lock() in
4590  * the core code; instead, the RCU protection relies on everything happening
4591  * inside a single NAPI poll sequence, which means it's between a pair of calls
4592  * to local_bh_disable()/local_bh_enable().
4593  *
4594  * The map entries are marked as __rcu and the map code makes sure to
4595  * dereference those pointers with rcu_dereference_check() in a way that works
4596  * for both sections that to hold an rcu_read_lock() and sections that are
4597  * called from NAPI without a separate rcu_read_lock(). The code below does not
4598  * use RCU annotations, but relies on those in the map code.
4599  */
4600 void xdp_do_flush(void)
4601 {
4602 	struct list_head *lh_map, *lh_dev, *lh_xsk;
4603 
4604 	bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
4605 	if (lh_dev)
4606 		__dev_flush(lh_dev);
4607 	if (lh_map)
4608 		__cpu_map_flush(lh_map);
4609 	if (lh_xsk)
4610 		__xsk_map_flush(lh_xsk);
4611 }
4612 EXPORT_SYMBOL_GPL(xdp_do_flush);
4613 
4614 #if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
4615 void xdp_do_check_flushed(struct napi_struct *napi)
4616 {
4617 	struct list_head *lh_map, *lh_dev, *lh_xsk;
4618 	bool missed = false;
4619 
4620 	bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
4621 	if (lh_dev) {
4622 		__dev_flush(lh_dev);
4623 		missed = true;
4624 	}
4625 	if (lh_map) {
4626 		__cpu_map_flush(lh_map);
4627 		missed = true;
4628 	}
4629 	if (lh_xsk) {
4630 		__xsk_map_flush(lh_xsk);
4631 		missed = true;
4632 	}
4633 
4634 	WARN_ONCE(missed, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
4635 		  napi->poll);
4636 }
4637 #endif
4638 
4639 DEFINE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key);
4640 EXPORT_SYMBOL_GPL(bpf_master_redirect_enabled_key);
4641 
4642 u32 xdp_master_redirect(struct xdp_buff *xdp)
4643 {
4644 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4645 	struct net_device *master, *slave;
4646 
4647 	master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev);
4648 	if (unlikely(!master || !(master->flags & IFF_UP)))
4649 		return XDP_ABORTED;
4650 	slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp);
4651 	if (slave && slave != xdp->rxq->dev) {
4652 		/* The target device is different from the receiving device, so
4653 		 * redirect it to the new device.
4654 		 * Using XDP_REDIRECT gets the correct behaviour from XDP enabled
4655 		 * drivers to unmap the packet from their rx ring.
4656 		 */
4657 		ri->tgt_index = slave->ifindex;
4658 		ri->map_id = INT_MAX;
4659 		ri->map_type = BPF_MAP_TYPE_UNSPEC;
4660 		return XDP_REDIRECT;
4661 	}
4662 	return XDP_TX;
4663 }
4664 EXPORT_SYMBOL_GPL(xdp_master_redirect);
4665 
4666 static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri,
4667 					const struct net_device *dev,
4668 					struct xdp_buff *xdp,
4669 					const struct bpf_prog *xdp_prog)
4670 {
4671 	enum bpf_map_type map_type = ri->map_type;
4672 	void *fwd = ri->tgt_value;
4673 	u32 map_id = ri->map_id;
4674 	int err;
4675 
4676 	ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
4677 	ri->map_type = BPF_MAP_TYPE_UNSPEC;
4678 
4679 	err = __xsk_map_redirect(fwd, xdp);
4680 	if (unlikely(err))
4681 		goto err;
4682 
4683 	_trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4684 	return 0;
4685 err:
4686 	_trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4687 	return err;
4688 }
4689 
4690 static __always_inline int
4691 __xdp_do_redirect_frame(struct bpf_redirect_info *ri, struct net_device *dev,
4692 			struct xdp_frame *xdpf,
4693 			const struct bpf_prog *xdp_prog)
4694 {
4695 	enum bpf_map_type map_type = ri->map_type;
4696 	void *fwd = ri->tgt_value;
4697 	u32 map_id = ri->map_id;
4698 	u32 flags = ri->flags;
4699 	struct bpf_map *map;
4700 	int err;
4701 
4702 	ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
4703 	ri->flags = 0;
4704 	ri->map_type = BPF_MAP_TYPE_UNSPEC;
4705 
4706 	if (unlikely(!xdpf)) {
4707 		err = -EOVERFLOW;
4708 		goto err;
4709 	}
4710 
4711 	switch (map_type) {
4712 	case BPF_MAP_TYPE_DEVMAP:
4713 		fallthrough;
4714 	case BPF_MAP_TYPE_DEVMAP_HASH:
4715 		if (unlikely(flags & BPF_F_BROADCAST)) {
4716 			map = READ_ONCE(ri->map);
4717 
4718 			/* The map pointer is cleared when the map is being torn
4719 			 * down by dev_map_free()
4720 			 */
4721 			if (unlikely(!map)) {
4722 				err = -ENOENT;
4723 				break;
4724 			}
4725 
4726 			WRITE_ONCE(ri->map, NULL);
4727 			err = dev_map_enqueue_multi(xdpf, dev, map,
4728 						    flags & BPF_F_EXCLUDE_INGRESS);
4729 		} else {
4730 			err = dev_map_enqueue(fwd, xdpf, dev);
4731 		}
4732 		break;
4733 	case BPF_MAP_TYPE_CPUMAP:
4734 		err = cpu_map_enqueue(fwd, xdpf, dev);
4735 		break;
4736 	case BPF_MAP_TYPE_UNSPEC:
4737 		if (map_id == INT_MAX) {
4738 			fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index);
4739 			if (unlikely(!fwd)) {
4740 				err = -EINVAL;
4741 				break;
4742 			}
4743 			err = dev_xdp_enqueue(fwd, xdpf, dev);
4744 			break;
4745 		}
4746 		fallthrough;
4747 	default:
4748 		err = -EBADRQC;
4749 	}
4750 
4751 	if (unlikely(err))
4752 		goto err;
4753 
4754 	_trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4755 	return 0;
4756 err:
4757 	_trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4758 	return err;
4759 }
4760 
4761 int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
4762 		    const struct bpf_prog *xdp_prog)
4763 {
4764 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4765 	enum bpf_map_type map_type = ri->map_type;
4766 
4767 	if (map_type == BPF_MAP_TYPE_XSKMAP)
4768 		return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog);
4769 
4770 	return __xdp_do_redirect_frame(ri, dev, xdp_convert_buff_to_frame(xdp),
4771 				       xdp_prog);
4772 }
4773 EXPORT_SYMBOL_GPL(xdp_do_redirect);
4774 
4775 int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp,
4776 			  struct xdp_frame *xdpf,
4777 			  const struct bpf_prog *xdp_prog)
4778 {
4779 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4780 	enum bpf_map_type map_type = ri->map_type;
4781 
4782 	if (map_type == BPF_MAP_TYPE_XSKMAP)
4783 		return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog);
4784 
4785 	return __xdp_do_redirect_frame(ri, dev, xdpf, xdp_prog);
4786 }
4787 EXPORT_SYMBOL_GPL(xdp_do_redirect_frame);
4788 
4789 static int xdp_do_generic_redirect_map(struct net_device *dev,
4790 				       struct sk_buff *skb,
4791 				       struct xdp_buff *xdp,
4792 				       const struct bpf_prog *xdp_prog,
4793 				       void *fwd, enum bpf_map_type map_type,
4794 				       u32 map_id, u32 flags)
4795 {
4796 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4797 	struct bpf_map *map;
4798 	int err;
4799 
4800 	switch (map_type) {
4801 	case BPF_MAP_TYPE_DEVMAP:
4802 		fallthrough;
4803 	case BPF_MAP_TYPE_DEVMAP_HASH:
4804 		if (unlikely(flags & BPF_F_BROADCAST)) {
4805 			map = READ_ONCE(ri->map);
4806 
4807 			/* The map pointer is cleared when the map is being torn
4808 			 * down by dev_map_free()
4809 			 */
4810 			if (unlikely(!map)) {
4811 				err = -ENOENT;
4812 				break;
4813 			}
4814 
4815 			WRITE_ONCE(ri->map, NULL);
4816 			err = dev_map_redirect_multi(dev, skb, xdp_prog, map,
4817 						     flags & BPF_F_EXCLUDE_INGRESS);
4818 		} else {
4819 			err = dev_map_generic_redirect(fwd, skb, xdp_prog);
4820 		}
4821 		if (unlikely(err))
4822 			goto err;
4823 		break;
4824 	case BPF_MAP_TYPE_XSKMAP:
4825 		err = xsk_generic_rcv(fwd, xdp);
4826 		if (err)
4827 			goto err;
4828 		consume_skb(skb);
4829 		break;
4830 	case BPF_MAP_TYPE_CPUMAP:
4831 		err = cpu_map_generic_redirect(fwd, skb);
4832 		if (unlikely(err))
4833 			goto err;
4834 		break;
4835 	default:
4836 		err = -EBADRQC;
4837 		goto err;
4838 	}
4839 
4840 	_trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
4841 	return 0;
4842 err:
4843 	_trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
4844 	return err;
4845 }
4846 
4847 int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
4848 			    struct xdp_buff *xdp,
4849 			    const struct bpf_prog *xdp_prog)
4850 {
4851 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4852 	enum bpf_map_type map_type = ri->map_type;
4853 	void *fwd = ri->tgt_value;
4854 	u32 map_id = ri->map_id;
4855 	u32 flags = ri->flags;
4856 	int err;
4857 
4858 	ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
4859 	ri->flags = 0;
4860 	ri->map_type = BPF_MAP_TYPE_UNSPEC;
4861 
4862 	if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) {
4863 		fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index);
4864 		if (unlikely(!fwd)) {
4865 			err = -EINVAL;
4866 			goto err;
4867 		}
4868 
4869 		err = xdp_ok_fwd_dev(fwd, skb->len);
4870 		if (unlikely(err))
4871 			goto err;
4872 
4873 		skb->dev = fwd;
4874 		_trace_xdp_redirect(dev, xdp_prog, ri->tgt_index);
4875 		generic_xdp_tx(skb, xdp_prog);
4876 		return 0;
4877 	}
4878 
4879 	return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog, fwd, map_type, map_id, flags);
4880 err:
4881 	_trace_xdp_redirect_err(dev, xdp_prog, ri->tgt_index, err);
4882 	return err;
4883 }
4884 
4885 BPF_CALL_2(bpf_xdp_redirect, u32, ifindex, u64, flags)
4886 {
4887 	struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
4888 
4889 	if (unlikely(flags))
4890 		return XDP_ABORTED;
4891 
4892 	/* NB! Map type UNSPEC and map_id == INT_MAX (never generated
4893 	 * by map_idr) is used for ifindex based XDP redirect.
4894 	 */
4895 	ri->tgt_index = ifindex;
4896 	ri->map_id = INT_MAX;
4897 	ri->map_type = BPF_MAP_TYPE_UNSPEC;
4898 
4899 	return XDP_REDIRECT;
4900 }
4901 
4902 static const struct bpf_func_proto bpf_xdp_redirect_proto = {
4903 	.func           = bpf_xdp_redirect,
4904 	.gpl_only       = false,
4905 	.ret_type       = RET_INTEGER,
4906 	.arg1_type      = ARG_ANYTHING,
4907 	.arg2_type      = ARG_ANYTHING,
4908 };
4909 
4910 BPF_CALL_3(bpf_xdp_redirect_map, struct bpf_map *, map, u64, key,
4911 	   u64, flags)
4912 {
4913 	return map->ops->map_redirect(map, key, flags);
4914 }
4915 
4916 static const struct bpf_func_proto bpf_xdp_redirect_map_proto = {
4917 	.func           = bpf_xdp_redirect_map,
4918 	.gpl_only       = false,
4919 	.ret_type       = RET_INTEGER,
4920 	.arg1_type      = ARG_CONST_MAP_PTR,
4921 	.arg2_type      = ARG_ANYTHING,
4922 	.arg3_type      = ARG_ANYTHING,
4923 };
4924 
4925 static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
4926 				  unsigned long off, unsigned long len)
4927 {
4928 	void *ptr = skb_header_pointer(skb, off, len, dst_buff);
4929 
4930 	if (unlikely(!ptr))
4931 		return len;
4932 	if (ptr != dst_buff)
4933 		memcpy(dst_buff, ptr, len);
4934 
4935 	return 0;
4936 }
4937 
4938 BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map,
4939 	   u64, flags, void *, meta, u64, meta_size)
4940 {
4941 	u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
4942 
4943 	if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
4944 		return -EINVAL;
4945 	if (unlikely(!skb || skb_size > skb->len))
4946 		return -EFAULT;
4947 
4948 	return bpf_event_output(map, flags, meta, meta_size, skb, skb_size,
4949 				bpf_skb_copy);
4950 }
4951 
4952 static const struct bpf_func_proto bpf_skb_event_output_proto = {
4953 	.func		= bpf_skb_event_output,
4954 	.gpl_only	= true,
4955 	.ret_type	= RET_INTEGER,
4956 	.arg1_type	= ARG_PTR_TO_CTX,
4957 	.arg2_type	= ARG_CONST_MAP_PTR,
4958 	.arg3_type	= ARG_ANYTHING,
4959 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
4960 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
4961 };
4962 
4963 BTF_ID_LIST_SINGLE(bpf_skb_output_btf_ids, struct, sk_buff)
4964 
4965 const struct bpf_func_proto bpf_skb_output_proto = {
4966 	.func		= bpf_skb_event_output,
4967 	.gpl_only	= true,
4968 	.ret_type	= RET_INTEGER,
4969 	.arg1_type	= ARG_PTR_TO_BTF_ID,
4970 	.arg1_btf_id	= &bpf_skb_output_btf_ids[0],
4971 	.arg2_type	= ARG_CONST_MAP_PTR,
4972 	.arg3_type	= ARG_ANYTHING,
4973 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
4974 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
4975 };
4976 
4977 static unsigned short bpf_tunnel_key_af(u64 flags)
4978 {
4979 	return flags & BPF_F_TUNINFO_IPV6 ? AF_INET6 : AF_INET;
4980 }
4981 
4982 BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key *, to,
4983 	   u32, size, u64, flags)
4984 {
4985 	const struct ip_tunnel_info *info = skb_tunnel_info(skb);
4986 	u8 compat[sizeof(struct bpf_tunnel_key)];
4987 	void *to_orig = to;
4988 	int err;
4989 
4990 	if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6 |
4991 					 BPF_F_TUNINFO_FLAGS)))) {
4992 		err = -EINVAL;
4993 		goto err_clear;
4994 	}
4995 	if (ip_tunnel_info_af(info) != bpf_tunnel_key_af(flags)) {
4996 		err = -EPROTO;
4997 		goto err_clear;
4998 	}
4999 	if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
5000 		err = -EINVAL;
5001 		switch (size) {
5002 		case offsetof(struct bpf_tunnel_key, local_ipv6[0]):
5003 		case offsetof(struct bpf_tunnel_key, tunnel_label):
5004 		case offsetof(struct bpf_tunnel_key, tunnel_ext):
5005 			goto set_compat;
5006 		case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
5007 			/* Fixup deprecated structure layouts here, so we have
5008 			 * a common path later on.
5009 			 */
5010 			if (ip_tunnel_info_af(info) != AF_INET)
5011 				goto err_clear;
5012 set_compat:
5013 			to = (struct bpf_tunnel_key *)compat;
5014 			break;
5015 		default:
5016 			goto err_clear;
5017 		}
5018 	}
5019 
5020 	to->tunnel_id = be64_to_cpu(info->key.tun_id);
5021 	to->tunnel_tos = info->key.tos;
5022 	to->tunnel_ttl = info->key.ttl;
5023 	if (flags & BPF_F_TUNINFO_FLAGS)
5024 		to->tunnel_flags = ip_tunnel_flags_to_be16(info->key.tun_flags);
5025 	else
5026 		to->tunnel_ext = 0;
5027 
5028 	if (flags & BPF_F_TUNINFO_IPV6) {
5029 		memcpy(to->remote_ipv6, &info->key.u.ipv6.src,
5030 		       sizeof(to->remote_ipv6));
5031 		memcpy(to->local_ipv6, &info->key.u.ipv6.dst,
5032 		       sizeof(to->local_ipv6));
5033 		to->tunnel_label = be32_to_cpu(info->key.label);
5034 	} else {
5035 		to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src);
5036 		memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3);
5037 		to->local_ipv4 = be32_to_cpu(info->key.u.ipv4.dst);
5038 		memset(&to->local_ipv6[1], 0, sizeof(__u32) * 3);
5039 		to->tunnel_label = 0;
5040 	}
5041 
5042 	if (unlikely(size != sizeof(struct bpf_tunnel_key)))
5043 		memcpy(to_orig, to, size);
5044 
5045 	return 0;
5046 err_clear:
5047 	memset(to_orig, 0, size);
5048 	return err;
5049 }
5050 
5051 static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = {
5052 	.func		= bpf_skb_get_tunnel_key,
5053 	.gpl_only	= false,
5054 	.ret_type	= RET_INTEGER,
5055 	.arg1_type	= ARG_PTR_TO_CTX,
5056 	.arg2_type	= ARG_PTR_TO_UNINIT_MEM,
5057 	.arg3_type	= ARG_MEM_SIZE,
5058 	.arg4_type	= ARG_ANYTHING,
5059 };
5060 
5061 BPF_CALL_3(bpf_skb_get_tunnel_opt, struct sk_buff *, skb, u8 *, to, u32, size)
5062 {
5063 	const struct ip_tunnel_info *info = skb_tunnel_info(skb);
5064 	int err;
5065 
5066 	if (unlikely(!info ||
5067 		     !ip_tunnel_is_options_present(info->key.tun_flags))) {
5068 		err = -ENOENT;
5069 		goto err_clear;
5070 	}
5071 	if (unlikely(size < info->options_len)) {
5072 		err = -ENOMEM;
5073 		goto err_clear;
5074 	}
5075 
5076 	ip_tunnel_info_opts_get(to, info);
5077 	if (size > info->options_len)
5078 		memset(to + info->options_len, 0, size - info->options_len);
5079 
5080 	return info->options_len;
5081 err_clear:
5082 	memset(to, 0, size);
5083 	return err;
5084 }
5085 
5086 static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = {
5087 	.func		= bpf_skb_get_tunnel_opt,
5088 	.gpl_only	= false,
5089 	.ret_type	= RET_INTEGER,
5090 	.arg1_type	= ARG_PTR_TO_CTX,
5091 	.arg2_type	= ARG_PTR_TO_UNINIT_MEM,
5092 	.arg3_type	= ARG_MEM_SIZE,
5093 };
5094 
5095 static struct metadata_dst __percpu *md_dst;
5096 
5097 BPF_CALL_4(bpf_skb_set_tunnel_key, struct sk_buff *, skb,
5098 	   const struct bpf_tunnel_key *, from, u32, size, u64, flags)
5099 {
5100 	struct metadata_dst *md = this_cpu_ptr(md_dst);
5101 	u8 compat[sizeof(struct bpf_tunnel_key)];
5102 	struct ip_tunnel_info *info;
5103 
5104 	if (unlikely(flags & ~(BPF_F_TUNINFO_IPV6 | BPF_F_ZERO_CSUM_TX |
5105 			       BPF_F_DONT_FRAGMENT | BPF_F_SEQ_NUMBER |
5106 			       BPF_F_NO_TUNNEL_KEY)))
5107 		return -EINVAL;
5108 	if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
5109 		switch (size) {
5110 		case offsetof(struct bpf_tunnel_key, local_ipv6[0]):
5111 		case offsetof(struct bpf_tunnel_key, tunnel_label):
5112 		case offsetof(struct bpf_tunnel_key, tunnel_ext):
5113 		case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
5114 			/* Fixup deprecated structure layouts here, so we have
5115 			 * a common path later on.
5116 			 */
5117 			memcpy(compat, from, size);
5118 			memset(compat + size, 0, sizeof(compat) - size);
5119 			from = (const struct bpf_tunnel_key *) compat;
5120 			break;
5121 		default:
5122 			return -EINVAL;
5123 		}
5124 	}
5125 	if (unlikely((!(flags & BPF_F_TUNINFO_IPV6) && from->tunnel_label) ||
5126 		     from->tunnel_ext))
5127 		return -EINVAL;
5128 
5129 	skb_dst_drop(skb);
5130 	dst_hold((struct dst_entry *) md);
5131 	skb_dst_set(skb, (struct dst_entry *) md);
5132 
5133 	info = &md->u.tun_info;
5134 	memset(info, 0, sizeof(*info));
5135 	info->mode = IP_TUNNEL_INFO_TX;
5136 
5137 	__set_bit(IP_TUNNEL_NOCACHE_BIT, info->key.tun_flags);
5138 	__assign_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, info->key.tun_flags,
5139 		     flags & BPF_F_DONT_FRAGMENT);
5140 	__assign_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags,
5141 		     !(flags & BPF_F_ZERO_CSUM_TX));
5142 	__assign_bit(IP_TUNNEL_SEQ_BIT, info->key.tun_flags,
5143 		     flags & BPF_F_SEQ_NUMBER);
5144 	__assign_bit(IP_TUNNEL_KEY_BIT, info->key.tun_flags,
5145 		     !(flags & BPF_F_NO_TUNNEL_KEY));
5146 
5147 	info->key.tun_id = cpu_to_be64(from->tunnel_id);
5148 	info->key.tos = from->tunnel_tos;
5149 	info->key.ttl = from->tunnel_ttl;
5150 
5151 	if (flags & BPF_F_TUNINFO_IPV6) {
5152 		info->mode |= IP_TUNNEL_INFO_IPV6;
5153 		memcpy(&info->key.u.ipv6.dst, from->remote_ipv6,
5154 		       sizeof(from->remote_ipv6));
5155 		memcpy(&info->key.u.ipv6.src, from->local_ipv6,
5156 		       sizeof(from->local_ipv6));
5157 		info->key.label = cpu_to_be32(from->tunnel_label) &
5158 				  IPV6_FLOWLABEL_MASK;
5159 	} else {
5160 		info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4);
5161 		info->key.u.ipv4.src = cpu_to_be32(from->local_ipv4);
5162 		info->key.flow_flags = FLOWI_FLAG_ANYSRC;
5163 	}
5164 
5165 	return 0;
5166 }
5167 
5168 static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = {
5169 	.func		= bpf_skb_set_tunnel_key,
5170 	.gpl_only	= false,
5171 	.ret_type	= RET_INTEGER,
5172 	.arg1_type	= ARG_PTR_TO_CTX,
5173 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5174 	.arg3_type	= ARG_MEM_SIZE,
5175 	.arg4_type	= ARG_ANYTHING,
5176 };
5177 
5178 BPF_CALL_3(bpf_skb_set_tunnel_opt, struct sk_buff *, skb,
5179 	   const u8 *, from, u32, size)
5180 {
5181 	struct ip_tunnel_info *info = skb_tunnel_info(skb);
5182 	const struct metadata_dst *md = this_cpu_ptr(md_dst);
5183 	IP_TUNNEL_DECLARE_FLAGS(present) = { };
5184 
5185 	if (unlikely(info != &md->u.tun_info || (size & (sizeof(u32) - 1))))
5186 		return -EINVAL;
5187 	if (unlikely(size > IP_TUNNEL_OPTS_MAX))
5188 		return -ENOMEM;
5189 
5190 	ip_tunnel_set_options_present(present);
5191 	ip_tunnel_info_opts_set(info, from, size, present);
5192 
5193 	return 0;
5194 }
5195 
5196 static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = {
5197 	.func		= bpf_skb_set_tunnel_opt,
5198 	.gpl_only	= false,
5199 	.ret_type	= RET_INTEGER,
5200 	.arg1_type	= ARG_PTR_TO_CTX,
5201 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5202 	.arg3_type	= ARG_MEM_SIZE,
5203 };
5204 
5205 static const struct bpf_func_proto *
5206 bpf_get_skb_set_tunnel_proto(enum bpf_func_id which)
5207 {
5208 	if (!md_dst) {
5209 		struct metadata_dst __percpu *tmp;
5210 
5211 		tmp = metadata_dst_alloc_percpu(IP_TUNNEL_OPTS_MAX,
5212 						METADATA_IP_TUNNEL,
5213 						GFP_KERNEL);
5214 		if (!tmp)
5215 			return NULL;
5216 		if (cmpxchg(&md_dst, NULL, tmp))
5217 			metadata_dst_free_percpu(tmp);
5218 	}
5219 
5220 	switch (which) {
5221 	case BPF_FUNC_skb_set_tunnel_key:
5222 		return &bpf_skb_set_tunnel_key_proto;
5223 	case BPF_FUNC_skb_set_tunnel_opt:
5224 		return &bpf_skb_set_tunnel_opt_proto;
5225 	default:
5226 		return NULL;
5227 	}
5228 }
5229 
5230 BPF_CALL_3(bpf_skb_under_cgroup, struct sk_buff *, skb, struct bpf_map *, map,
5231 	   u32, idx)
5232 {
5233 	struct bpf_array *array = container_of(map, struct bpf_array, map);
5234 	struct cgroup *cgrp;
5235 	struct sock *sk;
5236 
5237 	sk = skb_to_full_sk(skb);
5238 	if (!sk || !sk_fullsock(sk))
5239 		return -ENOENT;
5240 	if (unlikely(idx >= array->map.max_entries))
5241 		return -E2BIG;
5242 
5243 	cgrp = READ_ONCE(array->ptrs[idx]);
5244 	if (unlikely(!cgrp))
5245 		return -EAGAIN;
5246 
5247 	return sk_under_cgroup_hierarchy(sk, cgrp);
5248 }
5249 
5250 static const struct bpf_func_proto bpf_skb_under_cgroup_proto = {
5251 	.func		= bpf_skb_under_cgroup,
5252 	.gpl_only	= false,
5253 	.ret_type	= RET_INTEGER,
5254 	.arg1_type	= ARG_PTR_TO_CTX,
5255 	.arg2_type	= ARG_CONST_MAP_PTR,
5256 	.arg3_type	= ARG_ANYTHING,
5257 };
5258 
5259 #ifdef CONFIG_SOCK_CGROUP_DATA
5260 static inline u64 __bpf_sk_cgroup_id(struct sock *sk)
5261 {
5262 	struct cgroup *cgrp;
5263 
5264 	sk = sk_to_full_sk(sk);
5265 	if (!sk || !sk_fullsock(sk))
5266 		return 0;
5267 
5268 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
5269 	return cgroup_id(cgrp);
5270 }
5271 
5272 BPF_CALL_1(bpf_skb_cgroup_id, const struct sk_buff *, skb)
5273 {
5274 	return __bpf_sk_cgroup_id(skb->sk);
5275 }
5276 
5277 static const struct bpf_func_proto bpf_skb_cgroup_id_proto = {
5278 	.func           = bpf_skb_cgroup_id,
5279 	.gpl_only       = false,
5280 	.ret_type       = RET_INTEGER,
5281 	.arg1_type      = ARG_PTR_TO_CTX,
5282 };
5283 
5284 static inline u64 __bpf_sk_ancestor_cgroup_id(struct sock *sk,
5285 					      int ancestor_level)
5286 {
5287 	struct cgroup *ancestor;
5288 	struct cgroup *cgrp;
5289 
5290 	sk = sk_to_full_sk(sk);
5291 	if (!sk || !sk_fullsock(sk))
5292 		return 0;
5293 
5294 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
5295 	ancestor = cgroup_ancestor(cgrp, ancestor_level);
5296 	if (!ancestor)
5297 		return 0;
5298 
5299 	return cgroup_id(ancestor);
5300 }
5301 
5302 BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int,
5303 	   ancestor_level)
5304 {
5305 	return __bpf_sk_ancestor_cgroup_id(skb->sk, ancestor_level);
5306 }
5307 
5308 static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = {
5309 	.func           = bpf_skb_ancestor_cgroup_id,
5310 	.gpl_only       = false,
5311 	.ret_type       = RET_INTEGER,
5312 	.arg1_type      = ARG_PTR_TO_CTX,
5313 	.arg2_type      = ARG_ANYTHING,
5314 };
5315 
5316 BPF_CALL_1(bpf_sk_cgroup_id, struct sock *, sk)
5317 {
5318 	return __bpf_sk_cgroup_id(sk);
5319 }
5320 
5321 static const struct bpf_func_proto bpf_sk_cgroup_id_proto = {
5322 	.func           = bpf_sk_cgroup_id,
5323 	.gpl_only       = false,
5324 	.ret_type       = RET_INTEGER,
5325 	.arg1_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5326 };
5327 
5328 BPF_CALL_2(bpf_sk_ancestor_cgroup_id, struct sock *, sk, int, ancestor_level)
5329 {
5330 	return __bpf_sk_ancestor_cgroup_id(sk, ancestor_level);
5331 }
5332 
5333 static const struct bpf_func_proto bpf_sk_ancestor_cgroup_id_proto = {
5334 	.func           = bpf_sk_ancestor_cgroup_id,
5335 	.gpl_only       = false,
5336 	.ret_type       = RET_INTEGER,
5337 	.arg1_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5338 	.arg2_type      = ARG_ANYTHING,
5339 };
5340 #endif
5341 
5342 static unsigned long bpf_xdp_copy(void *dst, const void *ctx,
5343 				  unsigned long off, unsigned long len)
5344 {
5345 	struct xdp_buff *xdp = (struct xdp_buff *)ctx;
5346 
5347 	bpf_xdp_copy_buf(xdp, off, dst, len, false);
5348 	return 0;
5349 }
5350 
5351 BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,
5352 	   u64, flags, void *, meta, u64, meta_size)
5353 {
5354 	u64 xdp_size = (flags & BPF_F_CTXLEN_MASK) >> 32;
5355 
5356 	if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
5357 		return -EINVAL;
5358 
5359 	if (unlikely(!xdp || xdp_size > xdp_get_buff_len(xdp)))
5360 		return -EFAULT;
5361 
5362 	return bpf_event_output(map, flags, meta, meta_size, xdp,
5363 				xdp_size, bpf_xdp_copy);
5364 }
5365 
5366 static const struct bpf_func_proto bpf_xdp_event_output_proto = {
5367 	.func		= bpf_xdp_event_output,
5368 	.gpl_only	= true,
5369 	.ret_type	= RET_INTEGER,
5370 	.arg1_type	= ARG_PTR_TO_CTX,
5371 	.arg2_type	= ARG_CONST_MAP_PTR,
5372 	.arg3_type	= ARG_ANYTHING,
5373 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5374 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
5375 };
5376 
5377 BTF_ID_LIST_SINGLE(bpf_xdp_output_btf_ids, struct, xdp_buff)
5378 
5379 const struct bpf_func_proto bpf_xdp_output_proto = {
5380 	.func		= bpf_xdp_event_output,
5381 	.gpl_only	= true,
5382 	.ret_type	= RET_INTEGER,
5383 	.arg1_type	= ARG_PTR_TO_BTF_ID,
5384 	.arg1_btf_id	= &bpf_xdp_output_btf_ids[0],
5385 	.arg2_type	= ARG_CONST_MAP_PTR,
5386 	.arg3_type	= ARG_ANYTHING,
5387 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5388 	.arg5_type	= ARG_MEM_SIZE_OR_ZERO,
5389 };
5390 
5391 BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb)
5392 {
5393 	return skb->sk ? __sock_gen_cookie(skb->sk) : 0;
5394 }
5395 
5396 static const struct bpf_func_proto bpf_get_socket_cookie_proto = {
5397 	.func           = bpf_get_socket_cookie,
5398 	.gpl_only       = false,
5399 	.ret_type       = RET_INTEGER,
5400 	.arg1_type      = ARG_PTR_TO_CTX,
5401 };
5402 
5403 BPF_CALL_1(bpf_get_socket_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
5404 {
5405 	return __sock_gen_cookie(ctx->sk);
5406 }
5407 
5408 static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = {
5409 	.func		= bpf_get_socket_cookie_sock_addr,
5410 	.gpl_only	= false,
5411 	.ret_type	= RET_INTEGER,
5412 	.arg1_type	= ARG_PTR_TO_CTX,
5413 };
5414 
5415 BPF_CALL_1(bpf_get_socket_cookie_sock, struct sock *, ctx)
5416 {
5417 	return __sock_gen_cookie(ctx);
5418 }
5419 
5420 static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = {
5421 	.func		= bpf_get_socket_cookie_sock,
5422 	.gpl_only	= false,
5423 	.ret_type	= RET_INTEGER,
5424 	.arg1_type	= ARG_PTR_TO_CTX,
5425 };
5426 
5427 BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk)
5428 {
5429 	return sk ? sock_gen_cookie(sk) : 0;
5430 }
5431 
5432 const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = {
5433 	.func		= bpf_get_socket_ptr_cookie,
5434 	.gpl_only	= false,
5435 	.ret_type	= RET_INTEGER,
5436 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON | PTR_MAYBE_NULL,
5437 };
5438 
5439 BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
5440 {
5441 	return __sock_gen_cookie(ctx->sk);
5442 }
5443 
5444 static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = {
5445 	.func		= bpf_get_socket_cookie_sock_ops,
5446 	.gpl_only	= false,
5447 	.ret_type	= RET_INTEGER,
5448 	.arg1_type	= ARG_PTR_TO_CTX,
5449 };
5450 
5451 static u64 __bpf_get_netns_cookie(struct sock *sk)
5452 {
5453 	const struct net *net = sk ? sock_net(sk) : &init_net;
5454 
5455 	return net->net_cookie;
5456 }
5457 
5458 BPF_CALL_1(bpf_get_netns_cookie, struct sk_buff *, skb)
5459 {
5460 	return __bpf_get_netns_cookie(skb && skb->sk ? skb->sk : NULL);
5461 }
5462 
5463 static const struct bpf_func_proto bpf_get_netns_cookie_proto = {
5464 	.func           = bpf_get_netns_cookie,
5465 	.ret_type       = RET_INTEGER,
5466 	.arg1_type      = ARG_PTR_TO_CTX_OR_NULL,
5467 };
5468 
5469 BPF_CALL_1(bpf_get_netns_cookie_sock, struct sock *, ctx)
5470 {
5471 	return __bpf_get_netns_cookie(ctx);
5472 }
5473 
5474 static const struct bpf_func_proto bpf_get_netns_cookie_sock_proto = {
5475 	.func		= bpf_get_netns_cookie_sock,
5476 	.gpl_only	= false,
5477 	.ret_type	= RET_INTEGER,
5478 	.arg1_type	= ARG_PTR_TO_CTX_OR_NULL,
5479 };
5480 
5481 BPF_CALL_1(bpf_get_netns_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
5482 {
5483 	return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
5484 }
5485 
5486 static const struct bpf_func_proto bpf_get_netns_cookie_sock_addr_proto = {
5487 	.func		= bpf_get_netns_cookie_sock_addr,
5488 	.gpl_only	= false,
5489 	.ret_type	= RET_INTEGER,
5490 	.arg1_type	= ARG_PTR_TO_CTX_OR_NULL,
5491 };
5492 
5493 BPF_CALL_1(bpf_get_netns_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
5494 {
5495 	return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
5496 }
5497 
5498 static const struct bpf_func_proto bpf_get_netns_cookie_sock_ops_proto = {
5499 	.func		= bpf_get_netns_cookie_sock_ops,
5500 	.gpl_only	= false,
5501 	.ret_type	= RET_INTEGER,
5502 	.arg1_type	= ARG_PTR_TO_CTX_OR_NULL,
5503 };
5504 
5505 BPF_CALL_1(bpf_get_netns_cookie_sk_msg, struct sk_msg *, ctx)
5506 {
5507 	return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
5508 }
5509 
5510 static const struct bpf_func_proto bpf_get_netns_cookie_sk_msg_proto = {
5511 	.func		= bpf_get_netns_cookie_sk_msg,
5512 	.gpl_only	= false,
5513 	.ret_type	= RET_INTEGER,
5514 	.arg1_type	= ARG_PTR_TO_CTX_OR_NULL,
5515 };
5516 
5517 BPF_CALL_1(bpf_get_socket_uid, struct sk_buff *, skb)
5518 {
5519 	struct sock *sk = sk_to_full_sk(skb->sk);
5520 	kuid_t kuid;
5521 
5522 	if (!sk || !sk_fullsock(sk))
5523 		return overflowuid;
5524 	kuid = sock_net_uid(sock_net(sk), sk);
5525 	return from_kuid_munged(sock_net(sk)->user_ns, kuid);
5526 }
5527 
5528 static const struct bpf_func_proto bpf_get_socket_uid_proto = {
5529 	.func           = bpf_get_socket_uid,
5530 	.gpl_only       = false,
5531 	.ret_type       = RET_INTEGER,
5532 	.arg1_type      = ARG_PTR_TO_CTX,
5533 };
5534 
5535 static int sk_bpf_set_get_cb_flags(struct sock *sk, char *optval, bool getopt)
5536 {
5537 	u32 sk_bpf_cb_flags;
5538 
5539 	if (getopt) {
5540 		*(u32 *)optval = sk->sk_bpf_cb_flags;
5541 		return 0;
5542 	}
5543 
5544 	sk_bpf_cb_flags = *(u32 *)optval;
5545 
5546 	if (sk_bpf_cb_flags & ~SK_BPF_CB_MASK)
5547 		return -EINVAL;
5548 
5549 	sk->sk_bpf_cb_flags = sk_bpf_cb_flags;
5550 
5551 	return 0;
5552 }
5553 
5554 static int sol_socket_sockopt(struct sock *sk, int optname,
5555 			      char *optval, int *optlen,
5556 			      bool getopt)
5557 {
5558 	switch (optname) {
5559 	case SO_REUSEADDR:
5560 	case SO_SNDBUF:
5561 	case SO_RCVBUF:
5562 	case SO_KEEPALIVE:
5563 	case SO_PRIORITY:
5564 	case SO_REUSEPORT:
5565 	case SO_RCVLOWAT:
5566 	case SO_MARK:
5567 	case SO_MAX_PACING_RATE:
5568 	case SO_BINDTOIFINDEX:
5569 	case SO_TXREHASH:
5570 	case SK_BPF_CB_FLAGS:
5571 		if (*optlen != sizeof(int))
5572 			return -EINVAL;
5573 		break;
5574 	case SO_BINDTODEVICE:
5575 		break;
5576 	default:
5577 		return -EINVAL;
5578 	}
5579 
5580 	if (optname == SK_BPF_CB_FLAGS)
5581 		return sk_bpf_set_get_cb_flags(sk, optval, getopt);
5582 
5583 	if (getopt) {
5584 		if (optname == SO_BINDTODEVICE)
5585 			return -EINVAL;
5586 		return sk_getsockopt(sk, SOL_SOCKET, optname,
5587 				     KERNEL_SOCKPTR(optval),
5588 				     KERNEL_SOCKPTR(optlen));
5589 	}
5590 
5591 	return sk_setsockopt(sk, SOL_SOCKET, optname,
5592 			     KERNEL_SOCKPTR(optval), *optlen);
5593 }
5594 
5595 static int bpf_sol_tcp_getsockopt(struct sock *sk, int optname,
5596 				  char *optval, int optlen)
5597 {
5598 	if (optlen != sizeof(int))
5599 		return -EINVAL;
5600 
5601 	switch (optname) {
5602 	case TCP_BPF_SOCK_OPS_CB_FLAGS: {
5603 		int cb_flags = tcp_sk(sk)->bpf_sock_ops_cb_flags;
5604 
5605 		memcpy(optval, &cb_flags, optlen);
5606 		break;
5607 	}
5608 	case TCP_BPF_RTO_MIN: {
5609 		int rto_min_us = jiffies_to_usecs(inet_csk(sk)->icsk_rto_min);
5610 
5611 		memcpy(optval, &rto_min_us, optlen);
5612 		break;
5613 	}
5614 	case TCP_BPF_DELACK_MAX: {
5615 		int delack_max_us = jiffies_to_usecs(inet_csk(sk)->icsk_delack_max);
5616 
5617 		memcpy(optval, &delack_max_us, optlen);
5618 		break;
5619 	}
5620 	default:
5621 		return -EINVAL;
5622 	}
5623 
5624 	return 0;
5625 }
5626 
5627 static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname,
5628 				  char *optval, int optlen)
5629 {
5630 	struct tcp_sock *tp = tcp_sk(sk);
5631 	unsigned long timeout;
5632 	int val;
5633 
5634 	if (optlen != sizeof(int))
5635 		return -EINVAL;
5636 
5637 	val = *(int *)optval;
5638 
5639 	/* Only some options are supported */
5640 	switch (optname) {
5641 	case TCP_BPF_IW:
5642 		if (val <= 0 || tp->data_segs_out > tp->syn_data)
5643 			return -EINVAL;
5644 		tcp_snd_cwnd_set(tp, min_t(u32, val, tp->snd_cwnd_clamp));
5645 		break;
5646 	case TCP_BPF_SNDCWND_CLAMP:
5647 		if (val <= 0)
5648 			return -EINVAL;
5649 		val = min_t(u32, val, TCP_MAX_SND_CWND);
5650 		tp->snd_cwnd_clamp = val;
5651 		WRITE_ONCE(tp->snd_ssthresh, val);
5652 		break;
5653 	case TCP_BPF_DELACK_MAX:
5654 		timeout = usecs_to_jiffies(val);
5655 		if (timeout > TCP_DELACK_MAX ||
5656 		    timeout < TCP_TIMEOUT_MIN)
5657 			return -EINVAL;
5658 		inet_csk(sk)->icsk_delack_max = timeout;
5659 		break;
5660 	case TCP_BPF_RTO_MIN:
5661 		timeout = usecs_to_jiffies(val);
5662 		if (timeout > TCP_RTO_MIN ||
5663 		    timeout < TCP_TIMEOUT_MIN)
5664 			return -EINVAL;
5665 		inet_csk(sk)->icsk_rto_min = timeout;
5666 		break;
5667 	case TCP_BPF_SOCK_OPS_CB_FLAGS:
5668 		if (val & ~(BPF_SOCK_OPS_ALL_CB_FLAGS))
5669 			return -EINVAL;
5670 		tp->bpf_sock_ops_cb_flags = val;
5671 		break;
5672 	default:
5673 		return -EINVAL;
5674 	}
5675 
5676 	return 0;
5677 }
5678 
5679 static int sol_tcp_sockopt_congestion(struct sock *sk, char *optval,
5680 				      int *optlen, bool getopt)
5681 {
5682 	struct tcp_sock *tp;
5683 	int ret;
5684 
5685 	if (*optlen < 2)
5686 		return -EINVAL;
5687 
5688 	if (getopt) {
5689 		if (!inet_csk(sk)->icsk_ca_ops)
5690 			return -EINVAL;
5691 		/* BPF expects NULL-terminated tcp-cc string */
5692 		optval[--(*optlen)] = '\0';
5693 		return do_tcp_getsockopt(sk, SOL_TCP, TCP_CONGESTION,
5694 					 KERNEL_SOCKPTR(optval),
5695 					 KERNEL_SOCKPTR(optlen));
5696 	}
5697 
5698 	/* "cdg" is the only cc that alloc a ptr
5699 	 * in inet_csk_ca area.  The bpf-tcp-cc may
5700 	 * overwrite this ptr after switching to cdg.
5701 	 */
5702 	if (*optlen >= sizeof("cdg") - 1 && !strncmp("cdg", optval, *optlen))
5703 		return -ENOTSUPP;
5704 
5705 	/* It stops this looping
5706 	 *
5707 	 * .init => bpf_setsockopt(tcp_cc) => .init =>
5708 	 * bpf_setsockopt(tcp_cc)" => .init => ....
5709 	 *
5710 	 * The second bpf_setsockopt(tcp_cc) is not allowed
5711 	 * in order to break the loop when both .init
5712 	 * are the same bpf prog.
5713 	 *
5714 	 * This applies even the second bpf_setsockopt(tcp_cc)
5715 	 * does not cause a loop.  This limits only the first
5716 	 * '.init' can call bpf_setsockopt(TCP_CONGESTION) to
5717 	 * pick a fallback cc (eg. peer does not support ECN)
5718 	 * and the second '.init' cannot fallback to
5719 	 * another.
5720 	 */
5721 	tp = tcp_sk(sk);
5722 	if (tp->bpf_chg_cc_inprogress)
5723 		return -EBUSY;
5724 
5725 	tp->bpf_chg_cc_inprogress = 1;
5726 	ret = do_tcp_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
5727 				KERNEL_SOCKPTR(optval), *optlen);
5728 	tp->bpf_chg_cc_inprogress = 0;
5729 	return ret;
5730 }
5731 
5732 static int sol_tcp_sockopt(struct sock *sk, int optname,
5733 			   char *optval, int *optlen,
5734 			   bool getopt)
5735 {
5736 	if (!sk_is_tcp(sk))
5737 		return -EINVAL;
5738 
5739 	switch (optname) {
5740 	case TCP_NODELAY:
5741 	case TCP_MAXSEG:
5742 	case TCP_KEEPIDLE:
5743 	case TCP_KEEPINTVL:
5744 	case TCP_KEEPCNT:
5745 	case TCP_SYNCNT:
5746 	case TCP_WINDOW_CLAMP:
5747 	case TCP_THIN_LINEAR_TIMEOUTS:
5748 	case TCP_USER_TIMEOUT:
5749 	case TCP_NOTSENT_LOWAT:
5750 	case TCP_SAVE_SYN:
5751 	case TCP_RTO_MAX_MS:
5752 		if (*optlen != sizeof(int))
5753 			return -EINVAL;
5754 		break;
5755 	case TCP_CONGESTION:
5756 		return sol_tcp_sockopt_congestion(sk, optval, optlen, getopt);
5757 	case TCP_SAVED_SYN:
5758 		if (*optlen < 1)
5759 			return -EINVAL;
5760 		break;
5761 	default:
5762 		if (getopt)
5763 			return bpf_sol_tcp_getsockopt(sk, optname, optval, *optlen);
5764 		return bpf_sol_tcp_setsockopt(sk, optname, optval, *optlen);
5765 	}
5766 
5767 	if (getopt) {
5768 		if (optname == TCP_SAVED_SYN) {
5769 			struct tcp_sock *tp = tcp_sk(sk);
5770 
5771 			if (!tp->saved_syn ||
5772 			    *optlen > tcp_saved_syn_len(tp->saved_syn))
5773 				return -EINVAL;
5774 			memcpy(optval, tp->saved_syn->data, *optlen);
5775 			/* It cannot free tp->saved_syn here because it
5776 			 * does not know if the user space still needs it.
5777 			 */
5778 			return 0;
5779 		}
5780 
5781 		return do_tcp_getsockopt(sk, SOL_TCP, optname,
5782 					 KERNEL_SOCKPTR(optval),
5783 					 KERNEL_SOCKPTR(optlen));
5784 	}
5785 
5786 	return do_tcp_setsockopt(sk, SOL_TCP, optname,
5787 				 KERNEL_SOCKPTR(optval), *optlen);
5788 }
5789 
5790 static bool sk_allows_sol_ip_sockopt(struct sock *sk)
5791 {
5792 	switch (sk->sk_family) {
5793 	case AF_INET:
5794 		return true;
5795 	case AF_INET6:
5796 		/* Allow getting/setting sockopt for possible ipv4-mapped ipv6 socket. */
5797 		return sk->sk_type != SOCK_RAW && !ipv6_only_sock(sk);
5798 	default:
5799 		return false;
5800 	}
5801 }
5802 
5803 static int sol_ip_sockopt(struct sock *sk, int optname,
5804 			  char *optval, int *optlen,
5805 			  bool getopt)
5806 {
5807 	if (!sk_allows_sol_ip_sockopt(sk))
5808 		return -EINVAL;
5809 
5810 	switch (optname) {
5811 	case IP_TOS:
5812 	case IP_TRANSPARENT:
5813 		if (*optlen != sizeof(int))
5814 			return -EINVAL;
5815 		break;
5816 	default:
5817 		return -EINVAL;
5818 	}
5819 
5820 	if (getopt)
5821 		return do_ip_getsockopt(sk, SOL_IP, optname,
5822 					KERNEL_SOCKPTR(optval),
5823 					KERNEL_SOCKPTR(optlen));
5824 
5825 	return do_ip_setsockopt(sk, SOL_IP, optname,
5826 				KERNEL_SOCKPTR(optval), *optlen);
5827 }
5828 
5829 static int sol_ipv6_sockopt(struct sock *sk, int optname,
5830 			    char *optval, int *optlen,
5831 			    bool getopt)
5832 {
5833 	if (sk->sk_family != AF_INET6)
5834 		return -EINVAL;
5835 
5836 	switch (optname) {
5837 	case IPV6_TCLASS:
5838 	case IPV6_AUTOFLOWLABEL:
5839 	case IPV6_TRANSPARENT:
5840 		if (*optlen != sizeof(int))
5841 			return -EINVAL;
5842 		break;
5843 	default:
5844 		return -EINVAL;
5845 	}
5846 
5847 	if (getopt)
5848 		return do_ipv6_getsockopt(sk, SOL_IPV6, optname,
5849 					  KERNEL_SOCKPTR(optval),
5850 					  KERNEL_SOCKPTR(optlen));
5851 
5852 	return do_ipv6_setsockopt(sk, SOL_IPV6, optname,
5853 				  KERNEL_SOCKPTR(optval), *optlen);
5854 }
5855 
5856 static int __bpf_setsockopt(struct sock *sk, int level, int optname,
5857 			    char *optval, int optlen)
5858 {
5859 	if (!sk_fullsock(sk))
5860 		return -EINVAL;
5861 
5862 	if (level == SOL_SOCKET)
5863 		return sol_socket_sockopt(sk, optname, optval, &optlen, false);
5864 	else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
5865 		return sol_ip_sockopt(sk, optname, optval, &optlen, false);
5866 	else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
5867 		return sol_ipv6_sockopt(sk, optname, optval, &optlen, false);
5868 	else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
5869 		return sol_tcp_sockopt(sk, optname, optval, &optlen, false);
5870 
5871 	return -EINVAL;
5872 }
5873 
5874 static bool is_locked_tcp_sock_ops(struct bpf_sock_ops_kern *bpf_sock)
5875 {
5876 	return bpf_sock->op <= BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
5877 }
5878 
5879 static int _bpf_setsockopt(struct sock *sk, int level, int optname,
5880 			   char *optval, int optlen)
5881 {
5882 	if (sk_fullsock(sk))
5883 		sock_owned_by_me(sk);
5884 	return __bpf_setsockopt(sk, level, optname, optval, optlen);
5885 }
5886 
5887 static int __bpf_getsockopt(struct sock *sk, int level, int optname,
5888 			    char *optval, int optlen)
5889 {
5890 	int err, saved_optlen = optlen;
5891 
5892 	if (!sk_fullsock(sk)) {
5893 		err = -EINVAL;
5894 		goto done;
5895 	}
5896 
5897 	if (level == SOL_SOCKET)
5898 		err = sol_socket_sockopt(sk, optname, optval, &optlen, true);
5899 	else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
5900 		err = sol_tcp_sockopt(sk, optname, optval, &optlen, true);
5901 	else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
5902 		err = sol_ip_sockopt(sk, optname, optval, &optlen, true);
5903 	else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
5904 		err = sol_ipv6_sockopt(sk, optname, optval, &optlen, true);
5905 	else
5906 		err = -EINVAL;
5907 
5908 done:
5909 	if (err)
5910 		optlen = 0;
5911 	if (optlen < saved_optlen)
5912 		memset(optval + optlen, 0, saved_optlen - optlen);
5913 	return err;
5914 }
5915 
5916 static int _bpf_getsockopt(struct sock *sk, int level, int optname,
5917 			   char *optval, int optlen)
5918 {
5919 	if (sk_fullsock(sk))
5920 		sock_owned_by_me(sk);
5921 	return __bpf_getsockopt(sk, level, optname, optval, optlen);
5922 }
5923 
5924 BPF_CALL_5(bpf_sk_setsockopt, struct sock *, sk, int, level,
5925 	   int, optname, char *, optval, int, optlen)
5926 {
5927 	return _bpf_setsockopt(sk, level, optname, optval, optlen);
5928 }
5929 
5930 const struct bpf_func_proto bpf_sk_setsockopt_proto = {
5931 	.func		= bpf_sk_setsockopt,
5932 	.gpl_only	= false,
5933 	.ret_type	= RET_INTEGER,
5934 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5935 	.arg2_type	= ARG_ANYTHING,
5936 	.arg3_type	= ARG_ANYTHING,
5937 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5938 	.arg5_type	= ARG_MEM_SIZE,
5939 };
5940 
5941 BPF_CALL_5(bpf_sk_getsockopt, struct sock *, sk, int, level,
5942 	   int, optname, char *, optval, int, optlen)
5943 {
5944 	return _bpf_getsockopt(sk, level, optname, optval, optlen);
5945 }
5946 
5947 const struct bpf_func_proto bpf_sk_getsockopt_proto = {
5948 	.func		= bpf_sk_getsockopt,
5949 	.gpl_only	= false,
5950 	.ret_type	= RET_INTEGER,
5951 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5952 	.arg2_type	= ARG_ANYTHING,
5953 	.arg3_type	= ARG_ANYTHING,
5954 	.arg4_type	= ARG_PTR_TO_UNINIT_MEM,
5955 	.arg5_type	= ARG_MEM_SIZE,
5956 };
5957 
5958 BPF_CALL_5(bpf_sk_setsockopt_nodelay, struct sock *, sk, int, level,
5959 	   int, optname, char *, optval, int, optlen)
5960 {
5961 	/*
5962 	 * TCP_NODELAY triggers tcp_push_pending_frames() and re-enters
5963 	 * CA_EVENT_TX_START in bpf_tcp_cc.
5964 	 */
5965 	if (level == SOL_TCP && optname == TCP_NODELAY)
5966 		return -EOPNOTSUPP;
5967 
5968 	return _bpf_setsockopt(sk, level, optname, optval, optlen);
5969 }
5970 
5971 const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto = {
5972 	.func		= bpf_sk_setsockopt_nodelay,
5973 	.gpl_only	= false,
5974 	.ret_type	= RET_INTEGER,
5975 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5976 	.arg2_type	= ARG_ANYTHING,
5977 	.arg3_type	= ARG_ANYTHING,
5978 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5979 	.arg5_type	= ARG_MEM_SIZE,
5980 };
5981 
5982 BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level,
5983 	   int, optname, char *, optval, int, optlen)
5984 {
5985 	return __bpf_setsockopt(sk, level, optname, optval, optlen);
5986 }
5987 
5988 const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto = {
5989 	.func		= bpf_unlocked_sk_setsockopt,
5990 	.gpl_only	= false,
5991 	.ret_type	= RET_INTEGER,
5992 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
5993 	.arg2_type	= ARG_ANYTHING,
5994 	.arg3_type	= ARG_ANYTHING,
5995 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
5996 	.arg5_type	= ARG_MEM_SIZE,
5997 };
5998 
5999 BPF_CALL_5(bpf_unlocked_sk_getsockopt, struct sock *, sk, int, level,
6000 	   int, optname, char *, optval, int, optlen)
6001 {
6002 	return __bpf_getsockopt(sk, level, optname, optval, optlen);
6003 }
6004 
6005 const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto = {
6006 	.func		= bpf_unlocked_sk_getsockopt,
6007 	.gpl_only	= false,
6008 	.ret_type	= RET_INTEGER,
6009 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
6010 	.arg2_type	= ARG_ANYTHING,
6011 	.arg3_type	= ARG_ANYTHING,
6012 	.arg4_type	= ARG_PTR_TO_UNINIT_MEM,
6013 	.arg5_type	= ARG_MEM_SIZE,
6014 };
6015 
6016 BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx,
6017 	   int, level, int, optname, char *, optval, int, optlen)
6018 {
6019 	return _bpf_setsockopt(ctx->sk, level, optname, optval, optlen);
6020 }
6021 
6022 static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = {
6023 	.func		= bpf_sock_addr_setsockopt,
6024 	.gpl_only	= false,
6025 	.ret_type	= RET_INTEGER,
6026 	.arg1_type	= ARG_PTR_TO_CTX,
6027 	.arg2_type	= ARG_ANYTHING,
6028 	.arg3_type	= ARG_ANYTHING,
6029 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
6030 	.arg5_type	= ARG_MEM_SIZE,
6031 };
6032 
6033 BPF_CALL_5(bpf_sock_addr_getsockopt, struct bpf_sock_addr_kern *, ctx,
6034 	   int, level, int, optname, char *, optval, int, optlen)
6035 {
6036 	return _bpf_getsockopt(ctx->sk, level, optname, optval, optlen);
6037 }
6038 
6039 static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = {
6040 	.func		= bpf_sock_addr_getsockopt,
6041 	.gpl_only	= false,
6042 	.ret_type	= RET_INTEGER,
6043 	.arg1_type	= ARG_PTR_TO_CTX,
6044 	.arg2_type	= ARG_ANYTHING,
6045 	.arg3_type	= ARG_ANYTHING,
6046 	.arg4_type	= ARG_PTR_TO_UNINIT_MEM,
6047 	.arg5_type	= ARG_MEM_SIZE,
6048 };
6049 
6050 static int sk_bpf_set_get_bypass_prot_mem(struct sock *sk,
6051 					  char *optval, int optlen,
6052 					  bool getopt)
6053 {
6054 	int val;
6055 
6056 	if (optlen != sizeof(int))
6057 		return -EINVAL;
6058 
6059 	if (!sk_has_account(sk))
6060 		return -EOPNOTSUPP;
6061 
6062 	if (getopt) {
6063 		*(int *)optval = sk->sk_bypass_prot_mem;
6064 		return 0;
6065 	}
6066 
6067 	val = *(int *)optval;
6068 	if (val < 0 || val > 1)
6069 		return -EINVAL;
6070 
6071 	sk->sk_bypass_prot_mem = val;
6072 	return 0;
6073 }
6074 
6075 BPF_CALL_5(bpf_sock_create_setsockopt, struct sock *, sk, int, level,
6076 	   int, optname, char *, optval, int, optlen)
6077 {
6078 	if (level == SOL_SOCKET && optname == SK_BPF_BYPASS_PROT_MEM)
6079 		return sk_bpf_set_get_bypass_prot_mem(sk, optval, optlen, false);
6080 
6081 	return __bpf_setsockopt(sk, level, optname, optval, optlen);
6082 }
6083 
6084 static const struct bpf_func_proto bpf_sock_create_setsockopt_proto = {
6085 	.func		= bpf_sock_create_setsockopt,
6086 	.gpl_only	= false,
6087 	.ret_type	= RET_INTEGER,
6088 	.arg1_type	= ARG_PTR_TO_CTX,
6089 	.arg2_type	= ARG_ANYTHING,
6090 	.arg3_type	= ARG_ANYTHING,
6091 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
6092 	.arg5_type	= ARG_MEM_SIZE,
6093 };
6094 
6095 BPF_CALL_5(bpf_sock_create_getsockopt, struct sock *, sk, int, level,
6096 	   int, optname, char *, optval, int, optlen)
6097 {
6098 	if (level == SOL_SOCKET && optname == SK_BPF_BYPASS_PROT_MEM) {
6099 		int err = sk_bpf_set_get_bypass_prot_mem(sk, optval, optlen, true);
6100 
6101 		if (err)
6102 			memset(optval, 0, optlen);
6103 
6104 		return err;
6105 	}
6106 
6107 	return __bpf_getsockopt(sk, level, optname, optval, optlen);
6108 }
6109 
6110 static const struct bpf_func_proto bpf_sock_create_getsockopt_proto = {
6111 	.func		= bpf_sock_create_getsockopt,
6112 	.gpl_only	= false,
6113 	.ret_type	= RET_INTEGER,
6114 	.arg1_type	= ARG_PTR_TO_CTX,
6115 	.arg2_type	= ARG_ANYTHING,
6116 	.arg3_type	= ARG_ANYTHING,
6117 	.arg4_type	= ARG_PTR_TO_UNINIT_MEM,
6118 	.arg5_type	= ARG_MEM_SIZE,
6119 };
6120 
6121 BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
6122 	   int, level, int, optname, char *, optval, int, optlen)
6123 {
6124 	if (!is_locked_tcp_sock_ops(bpf_sock))
6125 		return -EOPNOTSUPP;
6126 
6127 	/* TCP_NODELAY triggers tcp_push_pending_frames() and re-enters these callbacks. */
6128 	if ((bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB ||
6129 	     bpf_sock->op == BPF_SOCK_OPS_WRITE_HDR_OPT_CB) &&
6130 	    level == SOL_TCP && optname == TCP_NODELAY)
6131 		return -EOPNOTSUPP;
6132 
6133 	return _bpf_setsockopt(bpf_sock->sk, level, optname, optval, optlen);
6134 }
6135 
6136 static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = {
6137 	.func		= bpf_sock_ops_setsockopt,
6138 	.gpl_only	= false,
6139 	.ret_type	= RET_INTEGER,
6140 	.arg1_type	= ARG_PTR_TO_CTX,
6141 	.arg2_type	= ARG_ANYTHING,
6142 	.arg3_type	= ARG_ANYTHING,
6143 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
6144 	.arg5_type	= ARG_MEM_SIZE,
6145 };
6146 
6147 static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock,
6148 				int optname, const u8 **start)
6149 {
6150 	struct sk_buff *syn_skb = bpf_sock->syn_skb;
6151 	const u8 *hdr_start;
6152 	int ret;
6153 
6154 	if (syn_skb) {
6155 		/* sk is a request_sock here */
6156 
6157 		if (optname == TCP_BPF_SYN) {
6158 			hdr_start = syn_skb->data;
6159 			ret = tcp_hdrlen(syn_skb);
6160 		} else if (optname == TCP_BPF_SYN_IP) {
6161 			hdr_start = skb_network_header(syn_skb);
6162 			ret = skb_network_header_len(syn_skb) +
6163 				tcp_hdrlen(syn_skb);
6164 		} else {
6165 			/* optname == TCP_BPF_SYN_MAC */
6166 			hdr_start = skb_mac_header(syn_skb);
6167 			ret = skb_mac_header_len(syn_skb) +
6168 				skb_network_header_len(syn_skb) +
6169 				tcp_hdrlen(syn_skb);
6170 		}
6171 	} else {
6172 		struct sock *sk = bpf_sock->sk;
6173 		struct saved_syn *saved_syn;
6174 
6175 		if (sk->sk_state == TCP_NEW_SYN_RECV)
6176 			/* synack retransmit. bpf_sock->syn_skb will
6177 			 * not be available.  It has to resort to
6178 			 * saved_syn (if it is saved).
6179 			 */
6180 			saved_syn = rcu_dereference(inet_reqsk(sk)->saved_syn);
6181 		else
6182 			saved_syn = tcp_sk(sk)->saved_syn;
6183 
6184 		if (!saved_syn)
6185 			return -ENOENT;
6186 
6187 		if (optname == TCP_BPF_SYN) {
6188 			hdr_start = saved_syn->data +
6189 				saved_syn->mac_hdrlen +
6190 				saved_syn->network_hdrlen;
6191 			ret = saved_syn->tcp_hdrlen;
6192 		} else if (optname == TCP_BPF_SYN_IP) {
6193 			hdr_start = saved_syn->data +
6194 				saved_syn->mac_hdrlen;
6195 			ret = saved_syn->network_hdrlen +
6196 				saved_syn->tcp_hdrlen;
6197 		} else {
6198 			/* optname == TCP_BPF_SYN_MAC */
6199 
6200 			/* TCP_SAVE_SYN may not have saved the mac hdr */
6201 			if (!saved_syn->mac_hdrlen)
6202 				return -ENOENT;
6203 
6204 			hdr_start = saved_syn->data;
6205 			ret = saved_syn->mac_hdrlen +
6206 				saved_syn->network_hdrlen +
6207 				saved_syn->tcp_hdrlen;
6208 		}
6209 	}
6210 
6211 	*start = hdr_start;
6212 	return ret;
6213 }
6214 
6215 BPF_CALL_5(bpf_sock_ops_getsockopt, struct bpf_sock_ops_kern *, bpf_sock,
6216 	   int, level, int, optname, char *, optval, int, optlen)
6217 {
6218 	if (!is_locked_tcp_sock_ops(bpf_sock))
6219 		return -EOPNOTSUPP;
6220 
6221 	if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP &&
6222 	    optname >= TCP_BPF_SYN && optname <= TCP_BPF_SYN_MAC) {
6223 		int ret, copy_len = 0;
6224 		const u8 *start;
6225 
6226 		ret = bpf_sock_ops_get_syn(bpf_sock, optname, &start);
6227 		if (ret > 0) {
6228 			copy_len = ret;
6229 			if (optlen < copy_len) {
6230 				copy_len = optlen;
6231 				ret = -ENOSPC;
6232 			}
6233 
6234 			memcpy(optval, start, copy_len);
6235 		}
6236 
6237 		/* Zero out unused buffer at the end */
6238 		memset(optval + copy_len, 0, optlen - copy_len);
6239 
6240 		return ret;
6241 	}
6242 
6243 	return _bpf_getsockopt(bpf_sock->sk, level, optname, optval, optlen);
6244 }
6245 
6246 static const struct bpf_func_proto bpf_sock_ops_getsockopt_proto = {
6247 	.func		= bpf_sock_ops_getsockopt,
6248 	.gpl_only	= false,
6249 	.ret_type	= RET_INTEGER,
6250 	.arg1_type	= ARG_PTR_TO_CTX,
6251 	.arg2_type	= ARG_ANYTHING,
6252 	.arg3_type	= ARG_ANYTHING,
6253 	.arg4_type	= ARG_PTR_TO_UNINIT_MEM,
6254 	.arg5_type	= ARG_MEM_SIZE,
6255 };
6256 
6257 BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock,
6258 	   int, argval)
6259 {
6260 	struct sock *sk = bpf_sock->sk;
6261 	int val = argval & BPF_SOCK_OPS_ALL_CB_FLAGS;
6262 
6263 	if (!is_locked_tcp_sock_ops(bpf_sock))
6264 		return -EOPNOTSUPP;
6265 
6266 	if (!IS_ENABLED(CONFIG_INET) || !sk_fullsock(sk))
6267 		return -EINVAL;
6268 
6269 	tcp_sk(sk)->bpf_sock_ops_cb_flags = val;
6270 
6271 	return argval & (~BPF_SOCK_OPS_ALL_CB_FLAGS);
6272 }
6273 
6274 static const struct bpf_func_proto bpf_sock_ops_cb_flags_set_proto = {
6275 	.func		= bpf_sock_ops_cb_flags_set,
6276 	.gpl_only	= false,
6277 	.ret_type	= RET_INTEGER,
6278 	.arg1_type	= ARG_PTR_TO_CTX,
6279 	.arg2_type	= ARG_ANYTHING,
6280 };
6281 
6282 BPF_CALL_3(bpf_bind, struct bpf_sock_addr_kern *, ctx, struct sockaddr *, addr,
6283 	   int, addr_len)
6284 {
6285 #ifdef CONFIG_INET
6286 	struct sock *sk = ctx->sk;
6287 	u32 flags = BIND_FROM_BPF;
6288 	int err;
6289 
6290 	err = -EINVAL;
6291 	if (addr_len < offsetofend(struct sockaddr, sa_family))
6292 		return err;
6293 	if (addr->sa_family == AF_INET) {
6294 		if (addr_len < sizeof(struct sockaddr_in))
6295 			return err;
6296 		if (((struct sockaddr_in *)addr)->sin_port == htons(0))
6297 			flags |= BIND_FORCE_ADDRESS_NO_PORT;
6298 		return __inet_bind(sk, (struct sockaddr_unsized *)addr, addr_len, flags);
6299 #if IS_ENABLED(CONFIG_IPV6)
6300 	} else if (addr->sa_family == AF_INET6) {
6301 		if (addr_len < SIN6_LEN_RFC2133)
6302 			return err;
6303 		if (((struct sockaddr_in6 *)addr)->sin6_port == htons(0))
6304 			flags |= BIND_FORCE_ADDRESS_NO_PORT;
6305 
6306 		return __inet6_bind(sk, (struct sockaddr_unsized *)addr,
6307 				    addr_len, flags);
6308 #endif /* CONFIG_IPV6 */
6309 	}
6310 #endif /* CONFIG_INET */
6311 
6312 	return -EAFNOSUPPORT;
6313 }
6314 
6315 static const struct bpf_func_proto bpf_bind_proto = {
6316 	.func		= bpf_bind,
6317 	.gpl_only	= false,
6318 	.ret_type	= RET_INTEGER,
6319 	.arg1_type	= ARG_PTR_TO_CTX,
6320 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
6321 	.arg3_type	= ARG_MEM_SIZE,
6322 };
6323 
6324 #ifdef CONFIG_XFRM
6325 
6326 #if (IS_BUILTIN(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
6327     (IS_MODULE(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
6328 
6329 struct metadata_dst __percpu *xfrm_bpf_md_dst;
6330 EXPORT_SYMBOL_GPL(xfrm_bpf_md_dst);
6331 
6332 #endif
6333 
6334 BPF_CALL_5(bpf_skb_get_xfrm_state, struct sk_buff *, skb, u32, index,
6335 	   struct bpf_xfrm_state *, to, u32, size, u64, flags)
6336 {
6337 	const struct sec_path *sp = skb_sec_path(skb);
6338 	const struct xfrm_state *x;
6339 
6340 	if (!sp || unlikely(index >= sp->len || flags))
6341 		goto err_clear;
6342 
6343 	x = sp->xvec[index];
6344 
6345 	if (unlikely(size != sizeof(struct bpf_xfrm_state)))
6346 		goto err_clear;
6347 
6348 	to->reqid = x->props.reqid;
6349 	to->spi = x->id.spi;
6350 	to->family = x->props.family;
6351 	to->ext = 0;
6352 
6353 	if (to->family == AF_INET6) {
6354 		memcpy(to->remote_ipv6, x->props.saddr.a6,
6355 		       sizeof(to->remote_ipv6));
6356 	} else {
6357 		to->remote_ipv4 = x->props.saddr.a4;
6358 		memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3);
6359 	}
6360 
6361 	return 0;
6362 err_clear:
6363 	memset(to, 0, size);
6364 	return -EINVAL;
6365 }
6366 
6367 static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = {
6368 	.func		= bpf_skb_get_xfrm_state,
6369 	.gpl_only	= false,
6370 	.ret_type	= RET_INTEGER,
6371 	.arg1_type	= ARG_PTR_TO_CTX,
6372 	.arg2_type	= ARG_ANYTHING,
6373 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
6374 	.arg4_type	= ARG_MEM_SIZE,
6375 	.arg5_type	= ARG_ANYTHING,
6376 };
6377 #endif
6378 
6379 #if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)
6380 static int bpf_fib_set_fwd_params(struct net_device *dev,
6381 				  struct bpf_fib_lookup *params,
6382 				  u32 flags, u32 mtu, u32 in_ifindex)
6383 {
6384 	params->h_vlan_TCI = 0;
6385 	params->h_vlan_proto = 0;
6386 
6387 #if IS_ENABLED(CONFIG_VLAN_8021Q)
6388 	if ((flags & BPF_FIB_LOOKUP_VLAN) && is_vlan_dev(dev)) {
6389 		struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
6390 
6391 		if (!is_vlan_dev(real_dev) &&
6392 		    net_eq(dev_net(real_dev), dev_net(dev))) {
6393 			params->h_vlan_proto = vlan_dev_vlan_proto(dev);
6394 			params->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
6395 			params->ifindex = real_dev->ifindex;
6396 		} else {
6397 			params->ifindex = in_ifindex;
6398 			return BPF_FIB_LKUP_RET_VLAN_FAILURE;
6399 		}
6400 	}
6401 #endif
6402 
6403 	if (mtu)
6404 		params->mtu_result = mtu; /* union with tot_len */
6405 
6406 	return 0;
6407 }
6408 
6409 static struct net_device *bpf_fib_vlan_input_dev(struct net_device *dev,
6410 						 const struct bpf_fib_lookup *params)
6411 {
6412 	__be16 proto = params->h_vlan_proto;
6413 	struct net_device *vlan_dev;
6414 	u16 vid;
6415 
6416 	if (proto != htons(ETH_P_8021Q) && proto != htons(ETH_P_8021AD))
6417 		return ERR_PTR(-EINVAL);
6418 
6419 	vid = ntohs(params->h_vlan_TCI) & VLAN_VID_MASK;
6420 	vlan_dev = __vlan_find_dev_deep_rcu(dev, proto, vid);
6421 	if (!vlan_dev || !(vlan_dev->flags & IFF_UP) ||
6422 	    !net_eq(dev_net(vlan_dev), dev_net(dev)))
6423 		return NULL;
6424 
6425 	return vlan_dev;
6426 }
6427 #endif
6428 
6429 #if IS_ENABLED(CONFIG_INET)
6430 static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
6431 			       u32 flags, bool check_mtu)
6432 {
6433 	u32 in_ifindex = params->ifindex;
6434 	struct neighbour *neigh = NULL;
6435 	struct fib_nh_common *nhc;
6436 	struct in_device *in_dev;
6437 	struct net_device *dev;
6438 	struct fib_result res;
6439 	struct flowi4 fl4 = {};
6440 	u32 mtu = 0;
6441 	int err;
6442 
6443 	dev = dev_get_by_index_rcu(net, params->ifindex);
6444 	if (unlikely(!dev))
6445 		return -ENODEV;
6446 
6447 	if (flags & BPF_FIB_LOOKUP_VLAN_INPUT) {
6448 		dev = bpf_fib_vlan_input_dev(dev, params);
6449 		if (IS_ERR(dev))
6450 			return PTR_ERR(dev);
6451 		if (!dev)
6452 			return BPF_FIB_LKUP_RET_NOT_FWDED;
6453 	}
6454 
6455 	/* verify forwarding is enabled on this interface */
6456 	in_dev = __in_dev_get_rcu(dev);
6457 	if (unlikely(!in_dev || !IN_DEV_FORWARD(in_dev)))
6458 		return BPF_FIB_LKUP_RET_FWD_DISABLED;
6459 
6460 	if (flags & BPF_FIB_LOOKUP_OUTPUT) {
6461 		fl4.flowi4_iif = 1;
6462 		fl4.flowi4_oif = params->ifindex;
6463 	} else {
6464 		/*
6465 		 * dev->ifindex, not params->ifindex: VLAN_INPUT may have
6466 		 * resolved dev to a subinterface above.
6467 		 */
6468 		fl4.flowi4_iif = dev->ifindex;
6469 		fl4.flowi4_oif = 0;
6470 	}
6471 	fl4.flowi4_dscp = inet_dsfield_to_dscp(params->tos);
6472 	fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
6473 	fl4.flowi4_flags = 0;
6474 
6475 	fl4.flowi4_proto = params->l4_protocol;
6476 	fl4.daddr = params->ipv4_dst;
6477 	fl4.saddr = params->ipv4_src;
6478 	fl4.fl4_sport = params->sport;
6479 	fl4.fl4_dport = params->dport;
6480 	fl4.flowi4_multipath_hash = 0;
6481 
6482 	if (flags & BPF_FIB_LOOKUP_DIRECT) {
6483 		u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
6484 		struct fib_table *tb;
6485 
6486 		if (flags & BPF_FIB_LOOKUP_TBID) {
6487 			tbid = params->tbid;
6488 			/* zero out for vlan output */
6489 			params->tbid = 0;
6490 		}
6491 
6492 		tb = fib_get_table(net, tbid);
6493 		if (unlikely(!tb))
6494 			return BPF_FIB_LKUP_RET_NOT_FWDED;
6495 
6496 		err = fib_table_lookup(tb, &fl4, &res, FIB_LOOKUP_NOREF);
6497 	} else {
6498 		if (flags & BPF_FIB_LOOKUP_MARK)
6499 			fl4.flowi4_mark = params->mark;
6500 		else
6501 			fl4.flowi4_mark = 0;
6502 		fl4.flowi4_secid = 0;
6503 		fl4.flowi4_tun_key.tun_id = 0;
6504 		fl4.flowi4_uid = sock_net_uid(net, NULL);
6505 
6506 		err = fib_lookup(net, &fl4, &res, FIB_LOOKUP_NOREF);
6507 	}
6508 
6509 	if (err) {
6510 		/* map fib lookup errors to RTN_ type */
6511 		if (err == -EINVAL)
6512 			return BPF_FIB_LKUP_RET_BLACKHOLE;
6513 		if (err == -EHOSTUNREACH)
6514 			return BPF_FIB_LKUP_RET_UNREACHABLE;
6515 		if (err == -EACCES)
6516 			return BPF_FIB_LKUP_RET_PROHIBIT;
6517 
6518 		return BPF_FIB_LKUP_RET_NOT_FWDED;
6519 	}
6520 
6521 	if (res.type != RTN_UNICAST)
6522 		return BPF_FIB_LKUP_RET_NOT_FWDED;
6523 
6524 	if (fib_info_num_path(res.fi) > 1)
6525 		fib_select_path(net, &res, &fl4, NULL);
6526 
6527 	if (check_mtu) {
6528 		mtu = ip_mtu_from_fib_result(&res, params->ipv4_dst);
6529 		if (params->tot_len > mtu) {
6530 			params->mtu_result = mtu; /* union with tot_len */
6531 			return BPF_FIB_LKUP_RET_FRAG_NEEDED;
6532 		}
6533 	}
6534 
6535 	nhc = res.nhc;
6536 
6537 	/* do not handle lwt encaps right now */
6538 	if (nhc->nhc_lwtstate)
6539 		return BPF_FIB_LKUP_RET_UNSUPP_LWT;
6540 
6541 	dev = nhc->nhc_dev;
6542 
6543 	params->rt_metric = res.fi->fib_priority;
6544 	params->ifindex = dev->ifindex;
6545 
6546 	if (flags & BPF_FIB_LOOKUP_SRC)
6547 		params->ipv4_src = fib_result_prefsrc(net, &res);
6548 
6549 	/* xdp and cls_bpf programs are run in RCU-bh so
6550 	 * rcu_read_lock_bh is not needed here
6551 	 */
6552 	if (likely(nhc->nhc_gw_family != AF_INET6)) {
6553 		if (nhc->nhc_gw_family)
6554 			params->ipv4_dst = nhc->nhc_gw.ipv4;
6555 	} else {
6556 		struct in6_addr *dst = (struct in6_addr *)params->ipv6_dst;
6557 
6558 		params->family = AF_INET6;
6559 		*dst = nhc->nhc_gw.ipv6;
6560 	}
6561 
6562 	if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH)
6563 		goto set_fwd_params;
6564 
6565 	if (likely(nhc->nhc_gw_family != AF_INET6))
6566 		neigh = __ipv4_neigh_lookup_noref(dev,
6567 						  (__force u32)params->ipv4_dst);
6568 	else if (IS_ENABLED(CONFIG_IPV6))
6569 		neigh = __ipv6_neigh_lookup_noref(dev, params->ipv6_dst);
6570 
6571 	if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
6572 		return BPF_FIB_LKUP_RET_NO_NEIGH;
6573 	memcpy(params->dmac, neigh->ha, ETH_ALEN);
6574 	memcpy(params->smac, dev->dev_addr, ETH_ALEN);
6575 
6576 set_fwd_params:
6577 	return bpf_fib_set_fwd_params(dev, params, flags, mtu, in_ifindex);
6578 }
6579 #endif
6580 
6581 #if IS_ENABLED(CONFIG_IPV6)
6582 static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
6583 			       u32 flags, bool check_mtu)
6584 {
6585 	struct in6_addr *src = (struct in6_addr *) params->ipv6_src;
6586 	struct in6_addr *dst = (struct in6_addr *) params->ipv6_dst;
6587 	u32 in_ifindex = params->ifindex;
6588 	struct fib6_result res = {};
6589 	struct neighbour *neigh;
6590 	struct net_device *dev;
6591 	struct inet6_dev *idev;
6592 	struct flowi6 fl6 = {};
6593 	int strict = 0;
6594 	int oif, err;
6595 	u32 mtu = 0;
6596 
6597 	/* link local addresses are never forwarded */
6598 	if (rt6_need_strict(dst) || rt6_need_strict(src))
6599 		return BPF_FIB_LKUP_RET_NOT_FWDED;
6600 
6601 	dev = dev_get_by_index_rcu(net, params->ifindex);
6602 	if (unlikely(!dev))
6603 		return -ENODEV;
6604 
6605 	if (flags & BPF_FIB_LOOKUP_VLAN_INPUT) {
6606 		dev = bpf_fib_vlan_input_dev(dev, params);
6607 		if (IS_ERR(dev))
6608 			return PTR_ERR(dev);
6609 		if (!dev)
6610 			return BPF_FIB_LKUP_RET_NOT_FWDED;
6611 	}
6612 
6613 	idev = __in6_dev_get_safely(dev);
6614 	if (unlikely(!idev || !READ_ONCE(idev->cnf.forwarding)))
6615 		return BPF_FIB_LKUP_RET_FWD_DISABLED;
6616 
6617 	if (flags & BPF_FIB_LOOKUP_OUTPUT) {
6618 		fl6.flowi6_iif = 1;
6619 		oif = fl6.flowi6_oif = params->ifindex;
6620 	} else {
6621 		/*
6622 		 * dev->ifindex, not params->ifindex: VLAN_INPUT may have
6623 		 * resolved dev to a subinterface above.
6624 		 */
6625 		oif = dev->ifindex;
6626 		fl6.flowi6_iif = oif;
6627 		fl6.flowi6_oif = 0;
6628 		strict = RT6_LOOKUP_F_HAS_SADDR;
6629 	}
6630 	fl6.flowlabel = params->flowinfo;
6631 	fl6.flowi6_scope = 0;
6632 	fl6.flowi6_flags = 0;
6633 	fl6.mp_hash = 0;
6634 
6635 	fl6.flowi6_proto = params->l4_protocol;
6636 	fl6.daddr = *dst;
6637 	fl6.saddr = *src;
6638 	fl6.fl6_sport = params->sport;
6639 	fl6.fl6_dport = params->dport;
6640 
6641 	if (flags & BPF_FIB_LOOKUP_DIRECT) {
6642 		u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
6643 		struct fib6_table *tb;
6644 
6645 		if (flags & BPF_FIB_LOOKUP_TBID) {
6646 			tbid = params->tbid;
6647 			/* zero out for vlan output */
6648 			params->tbid = 0;
6649 		}
6650 
6651 		tb = fib6_get_table(net, tbid);
6652 		if (unlikely(!tb))
6653 			return BPF_FIB_LKUP_RET_NOT_FWDED;
6654 
6655 		err = fib6_table_lookup(net, tb, oif, &fl6, &res, strict);
6656 	} else {
6657 		if (flags & BPF_FIB_LOOKUP_MARK)
6658 			fl6.flowi6_mark = params->mark;
6659 		else
6660 			fl6.flowi6_mark = 0;
6661 		fl6.flowi6_secid = 0;
6662 		fl6.flowi6_tun_key.tun_id = 0;
6663 		fl6.flowi6_uid = sock_net_uid(net, NULL);
6664 
6665 		err = fib6_lookup(net, oif, &fl6, &res, strict);
6666 	}
6667 
6668 	if (unlikely(err || IS_ERR_OR_NULL(res.f6i) ||
6669 		     res.f6i == net->ipv6.fib6_null_entry))
6670 		return BPF_FIB_LKUP_RET_NOT_FWDED;
6671 
6672 	switch (res.fib6_type) {
6673 	/* only unicast is forwarded */
6674 	case RTN_UNICAST:
6675 		break;
6676 	case RTN_BLACKHOLE:
6677 		return BPF_FIB_LKUP_RET_BLACKHOLE;
6678 	case RTN_UNREACHABLE:
6679 		return BPF_FIB_LKUP_RET_UNREACHABLE;
6680 	case RTN_PROHIBIT:
6681 		return BPF_FIB_LKUP_RET_PROHIBIT;
6682 	default:
6683 		return BPF_FIB_LKUP_RET_NOT_FWDED;
6684 	}
6685 
6686 	fib6_select_path(net, &res, &fl6, fl6.flowi6_oif,
6687 			 fl6.flowi6_oif != 0, NULL, strict);
6688 
6689 	if (check_mtu) {
6690 		mtu = ip6_mtu_from_fib6(&res, dst, src);
6691 		if (params->tot_len > mtu) {
6692 			params->mtu_result = mtu; /* union with tot_len */
6693 			return BPF_FIB_LKUP_RET_FRAG_NEEDED;
6694 		}
6695 	}
6696 
6697 	if (res.nh->fib_nh_lws)
6698 		return BPF_FIB_LKUP_RET_UNSUPP_LWT;
6699 
6700 	if (res.nh->fib_nh_gw_family)
6701 		*dst = res.nh->fib_nh_gw6;
6702 
6703 	dev = res.nh->fib_nh_dev;
6704 	params->rt_metric = res.f6i->fib6_metric;
6705 	params->ifindex = dev->ifindex;
6706 
6707 	if (flags & BPF_FIB_LOOKUP_SRC) {
6708 		if (res.f6i->fib6_prefsrc.plen) {
6709 			*src = res.f6i->fib6_prefsrc.addr;
6710 		} else {
6711 			err = ipv6_dev_get_saddr(net, dev, &fl6.daddr, 0, src);
6712 			if (err)
6713 				return BPF_FIB_LKUP_RET_NO_SRC_ADDR;
6714 		}
6715 	}
6716 
6717 	if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH)
6718 		goto set_fwd_params;
6719 
6720 	/* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is
6721 	 * not needed here.
6722 	 */
6723 	neigh = __ipv6_neigh_lookup_noref(dev, dst);
6724 	if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
6725 		return BPF_FIB_LKUP_RET_NO_NEIGH;
6726 	memcpy(params->dmac, neigh->ha, ETH_ALEN);
6727 	memcpy(params->smac, dev->dev_addr, ETH_ALEN);
6728 
6729 set_fwd_params:
6730 	return bpf_fib_set_fwd_params(dev, params, flags, mtu, in_ifindex);
6731 }
6732 #endif
6733 
6734 #define BPF_FIB_LOOKUP_MASK (BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT | \
6735 			     BPF_FIB_LOOKUP_SKIP_NEIGH | BPF_FIB_LOOKUP_TBID | \
6736 			     BPF_FIB_LOOKUP_SRC | BPF_FIB_LOOKUP_MARK | \
6737 			     BPF_FIB_LOOKUP_VLAN | BPF_FIB_LOOKUP_VLAN_INPUT)
6738 
6739 static bool bpf_fib_lookup_flags_ok(u32 flags)
6740 {
6741 	if (flags & ~BPF_FIB_LOOKUP_MASK)
6742 		return false;
6743 
6744 	if ((flags & BPF_FIB_LOOKUP_VLAN_INPUT) &&
6745 	    (flags & (BPF_FIB_LOOKUP_TBID | BPF_FIB_LOOKUP_OUTPUT)))
6746 		return false;
6747 
6748 	return true;
6749 }
6750 
6751 BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx,
6752 	   struct bpf_fib_lookup *, params, int, plen, u32, flags)
6753 {
6754 	if (plen < sizeof(*params))
6755 		return -EINVAL;
6756 
6757 	if (!bpf_fib_lookup_flags_ok(flags))
6758 		return -EINVAL;
6759 
6760 	switch (params->family) {
6761 #if IS_ENABLED(CONFIG_INET)
6762 	case AF_INET:
6763 		return bpf_ipv4_fib_lookup(dev_net(ctx->rxq->dev), params,
6764 					   flags, true);
6765 #endif
6766 #if IS_ENABLED(CONFIG_IPV6)
6767 	case AF_INET6:
6768 		return bpf_ipv6_fib_lookup(dev_net(ctx->rxq->dev), params,
6769 					   flags, true);
6770 #endif
6771 	}
6772 	return -EAFNOSUPPORT;
6773 }
6774 
6775 static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = {
6776 	.func		= bpf_xdp_fib_lookup,
6777 	.gpl_only	= true,
6778 	.ret_type	= RET_INTEGER,
6779 	.arg1_type      = ARG_PTR_TO_CTX,
6780 	.arg2_type      = ARG_PTR_TO_MEM | MEM_WRITE,
6781 	.arg3_type      = ARG_MEM_SIZE,
6782 	.arg4_type	= ARG_ANYTHING,
6783 };
6784 
6785 BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
6786 	   struct bpf_fib_lookup *, params, int, plen, u32, flags)
6787 {
6788 	struct net *net = dev_net(skb->dev);
6789 	int rc = -EAFNOSUPPORT;
6790 	bool check_mtu = false;
6791 
6792 	if (plen < sizeof(*params))
6793 		return -EINVAL;
6794 
6795 	if (!bpf_fib_lookup_flags_ok(flags))
6796 		return -EINVAL;
6797 
6798 	if (flags & BPF_FIB_LOOKUP_VLAN)
6799 		return -EINVAL;
6800 
6801 	if (params->tot_len)
6802 		check_mtu = true;
6803 
6804 	switch (params->family) {
6805 #if IS_ENABLED(CONFIG_INET)
6806 	case AF_INET:
6807 		rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
6808 		break;
6809 #endif
6810 #if IS_ENABLED(CONFIG_IPV6)
6811 	case AF_INET6:
6812 		rc = bpf_ipv6_fib_lookup(net, params, flags, check_mtu);
6813 		break;
6814 #endif
6815 	}
6816 
6817 	if (rc == BPF_FIB_LKUP_RET_SUCCESS && !check_mtu) {
6818 		struct net_device *dev;
6819 
6820 		/* When tot_len isn't provided by user, check skb
6821 		 * against MTU of FIB lookup resulting net_device
6822 		 */
6823 		dev = dev_get_by_index_rcu(net, params->ifindex);
6824 		if (unlikely(!dev))
6825 			return -ENODEV;
6826 		if (!is_skb_forwardable(dev, skb))
6827 			rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;
6828 
6829 		params->mtu_result = dev->mtu; /* union with tot_len */
6830 	}
6831 
6832 	return rc;
6833 }
6834 
6835 static const struct bpf_func_proto bpf_skb_fib_lookup_proto = {
6836 	.func		= bpf_skb_fib_lookup,
6837 	.gpl_only	= true,
6838 	.ret_type	= RET_INTEGER,
6839 	.arg1_type      = ARG_PTR_TO_CTX,
6840 	.arg2_type      = ARG_PTR_TO_MEM | MEM_WRITE,
6841 	.arg3_type      = ARG_MEM_SIZE,
6842 	.arg4_type	= ARG_ANYTHING,
6843 };
6844 
6845 static struct net_device *__dev_via_ifindex(struct net_device *dev_curr,
6846 					    u32 ifindex)
6847 {
6848 	struct net *netns = dev_net(dev_curr);
6849 
6850 	/* Non-redirect use-cases can use ifindex=0 and save ifindex lookup */
6851 	if (ifindex == 0)
6852 		return dev_curr;
6853 
6854 	return dev_get_by_index_rcu(netns, ifindex);
6855 }
6856 
6857 BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb,
6858 	   u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
6859 {
6860 	int ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
6861 	struct net_device *dev = skb->dev;
6862 	int mtu, dev_len, skb_len;
6863 
6864 	if (unlikely(flags & ~(BPF_MTU_CHK_SEGS)))
6865 		return -EINVAL;
6866 	if (unlikely(flags & BPF_MTU_CHK_SEGS && (len_diff || *mtu_len)))
6867 		return -EINVAL;
6868 
6869 	dev = __dev_via_ifindex(dev, ifindex);
6870 	if (unlikely(!dev))
6871 		return -ENODEV;
6872 
6873 	mtu = READ_ONCE(dev->mtu);
6874 	dev_len = mtu + dev->hard_header_len;
6875 
6876 	/* If set use *mtu_len as input, L3 as iph->tot_len (like fib_lookup) */
6877 	skb_len = *mtu_len ? *mtu_len + dev->hard_header_len : skb->len;
6878 
6879 	skb_len += len_diff; /* minus result pass check */
6880 	if (skb_len <= dev_len) {
6881 		ret = BPF_MTU_CHK_RET_SUCCESS;
6882 		goto out;
6883 	}
6884 	/* At this point, skb->len exceed MTU, but as it include length of all
6885 	 * segments, it can still be below MTU.  The SKB can possibly get
6886 	 * re-segmented in transmit path (see validate_xmit_skb).  Thus, user
6887 	 * must choose if segs are to be MTU checked.
6888 	 */
6889 	if (skb_is_gso(skb)) {
6890 		ret = BPF_MTU_CHK_RET_SUCCESS;
6891 		if (flags & BPF_MTU_CHK_SEGS) {
6892 			if (!skb_transport_header_was_set(skb))
6893 				return -EINVAL;
6894 			if (!skb_gso_validate_network_len(skb, mtu))
6895 				ret = BPF_MTU_CHK_RET_SEGS_TOOBIG;
6896 		}
6897 	}
6898 out:
6899 	*mtu_len = mtu;
6900 	return ret;
6901 }
6902 
6903 BPF_CALL_5(bpf_xdp_check_mtu, struct xdp_buff *, xdp,
6904 	   u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
6905 {
6906 	struct net_device *dev = xdp->rxq->dev;
6907 	int xdp_len = xdp->data_end - xdp->data;
6908 	int ret = BPF_MTU_CHK_RET_SUCCESS;
6909 	int mtu, dev_len;
6910 
6911 	/* XDP variant doesn't support multi-buffer segment check (yet) */
6912 	if (unlikely(flags))
6913 		return -EINVAL;
6914 
6915 	dev = __dev_via_ifindex(dev, ifindex);
6916 	if (unlikely(!dev))
6917 		return -ENODEV;
6918 
6919 	mtu = READ_ONCE(dev->mtu);
6920 	dev_len = mtu + dev->hard_header_len;
6921 
6922 	/* Use *mtu_len as input, L3 as iph->tot_len (like fib_lookup) */
6923 	if (*mtu_len)
6924 		xdp_len = *mtu_len + dev->hard_header_len;
6925 
6926 	xdp_len += len_diff; /* minus result pass check */
6927 	if (xdp_len > dev_len)
6928 		ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
6929 
6930 	*mtu_len = mtu;
6931 	return ret;
6932 }
6933 
6934 static const struct bpf_func_proto bpf_skb_check_mtu_proto = {
6935 	.func		= bpf_skb_check_mtu,
6936 	.gpl_only	= true,
6937 	.ret_type	= RET_INTEGER,
6938 	.arg1_type      = ARG_PTR_TO_CTX,
6939 	.arg2_type      = ARG_ANYTHING,
6940 	.arg3_type      = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_WRITE | MEM_ALIGNED,
6941 	.arg3_size	= sizeof(u32),
6942 	.arg4_type      = ARG_ANYTHING,
6943 	.arg5_type      = ARG_ANYTHING,
6944 };
6945 
6946 static const struct bpf_func_proto bpf_xdp_check_mtu_proto = {
6947 	.func		= bpf_xdp_check_mtu,
6948 	.gpl_only	= true,
6949 	.ret_type	= RET_INTEGER,
6950 	.arg1_type      = ARG_PTR_TO_CTX,
6951 	.arg2_type      = ARG_ANYTHING,
6952 	.arg3_type      = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_WRITE | MEM_ALIGNED,
6953 	.arg3_size	= sizeof(u32),
6954 	.arg4_type      = ARG_ANYTHING,
6955 	.arg5_type      = ARG_ANYTHING,
6956 };
6957 
6958 #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
6959 static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)
6960 {
6961 	int err;
6962 	struct ipv6_sr_hdr *srh = (struct ipv6_sr_hdr *)hdr;
6963 
6964 	/* With ipv6.disable=1 the netns has no seg6 data and no IPv6 tables. */
6965 	if (!ipv6_mod_enabled())
6966 		return -EAFNOSUPPORT;
6967 
6968 	if (!seg6_validate_srh(srh, len, false))
6969 		return -EINVAL;
6970 
6971 	/*
6972 	 * The encap mode names IPv6 as the inner protocol, and neither mode
6973 	 * updates skb->protocol.
6974 	 */
6975 	if (skb->protocol != htons(ETH_P_IPV6))
6976 		return -EBADMSG;
6977 
6978 	switch (type) {
6979 	case BPF_LWT_ENCAP_SEG6_INLINE:
6980 		err = seg6_do_srh_inline(skb, srh);
6981 		break;
6982 	case BPF_LWT_ENCAP_SEG6:
6983 		skb_reset_inner_headers(skb);
6984 		skb->encapsulation = 1;
6985 		err = seg6_do_srh_encap(skb, srh, IPPROTO_IPV6);
6986 		break;
6987 	default:
6988 		return -EINVAL;
6989 	}
6990 
6991 	bpf_compute_data_pointers(skb);
6992 	if (err)
6993 		return err;
6994 
6995 	skb_set_transport_header(skb, sizeof(struct ipv6hdr));
6996 
6997 	return seg6_lookup_nexthop(skb, NULL, 0);
6998 }
6999 #endif /* CONFIG_IPV6_SEG6_BPF */
7000 
7001 #if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
7002 static int bpf_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len,
7003 			     bool ingress)
7004 {
7005 	return bpf_lwt_push_ip_encap(skb, hdr, len, ingress);
7006 }
7007 #endif
7008 
7009 BPF_CALL_4(bpf_lwt_in_push_encap, struct sk_buff *, skb, u32, type, void *, hdr,
7010 	   u32, len)
7011 {
7012 	switch (type) {
7013 #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
7014 	case BPF_LWT_ENCAP_SEG6:
7015 	case BPF_LWT_ENCAP_SEG6_INLINE:
7016 		return bpf_push_seg6_encap(skb, type, hdr, len);
7017 #endif
7018 #if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
7019 	case BPF_LWT_ENCAP_IP:
7020 		return bpf_push_ip_encap(skb, hdr, len, true /* ingress */);
7021 #endif
7022 	default:
7023 		return -EINVAL;
7024 	}
7025 }
7026 
7027 BPF_CALL_4(bpf_lwt_xmit_push_encap, struct sk_buff *, skb, u32, type,
7028 	   void *, hdr, u32, len)
7029 {
7030 	switch (type) {
7031 #if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
7032 	case BPF_LWT_ENCAP_IP:
7033 		return bpf_push_ip_encap(skb, hdr, len, false /* egress */);
7034 #endif
7035 	default:
7036 		return -EINVAL;
7037 	}
7038 }
7039 
7040 static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = {
7041 	.func		= bpf_lwt_in_push_encap,
7042 	.gpl_only	= false,
7043 	.ret_type	= RET_INTEGER,
7044 	.arg1_type	= ARG_PTR_TO_CTX,
7045 	.arg2_type	= ARG_ANYTHING,
7046 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7047 	.arg4_type	= ARG_MEM_SIZE
7048 };
7049 
7050 static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = {
7051 	.func		= bpf_lwt_xmit_push_encap,
7052 	.gpl_only	= false,
7053 	.ret_type	= RET_INTEGER,
7054 	.arg1_type	= ARG_PTR_TO_CTX,
7055 	.arg2_type	= ARG_ANYTHING,
7056 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7057 	.arg4_type	= ARG_MEM_SIZE
7058 };
7059 
7060 #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
7061 BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
7062 	   const void *, from, u32, len)
7063 {
7064 	struct seg6_bpf_srh_state *srh_state =
7065 		this_cpu_ptr(&seg6_bpf_srh_states);
7066 	struct ipv6_sr_hdr *srh = srh_state->srh;
7067 	void *srh_tlvs, *srh_end, *ptr;
7068 	int srhoff;
7069 
7070 	lockdep_assert_held(&srh_state->bh_lock);
7071 	if (srh == NULL)
7072 		return -EINVAL;
7073 
7074 	srhoff = (unsigned char *)srh - skb->data;
7075 	srh_tlvs = (void *)((char *)srh + ((srh->first_segment + 1) << 4));
7076 	srh_end = (void *)((char *)srh + sizeof(*srh) + srh_state->hdrlen);
7077 
7078 	ptr = skb->data + offset;
7079 	if (ptr >= srh_tlvs && ptr + len <= srh_end)
7080 		srh_state->valid = false;
7081 	else if (ptr < (void *)&srh->flags ||
7082 		 ptr + len > (void *)&srh->segments)
7083 		return -EFAULT;
7084 
7085 	if (unlikely(bpf_try_make_writable(skb, offset + len)))
7086 		return -EFAULT;
7087 	/* The head may have moved. */
7088 	srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
7089 
7090 	memcpy(skb->data + offset, from, len);
7091 	return 0;
7092 }
7093 
7094 static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = {
7095 	.func		= bpf_lwt_seg6_store_bytes,
7096 	.gpl_only	= false,
7097 	.ret_type	= RET_INTEGER,
7098 	.arg1_type	= ARG_PTR_TO_CTX,
7099 	.arg2_type	= ARG_ANYTHING,
7100 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7101 	.arg4_type	= ARG_MEM_SIZE
7102 };
7103 
7104 static void bpf_update_srh_state(struct sk_buff *skb)
7105 {
7106 	struct seg6_bpf_srh_state *srh_state =
7107 		this_cpu_ptr(&seg6_bpf_srh_states);
7108 
7109 	/* Behind a decapsulation this is a header that nobody has looked at */
7110 	srh_state->srh = seg6_get_srh(skb, 0);
7111 	if (srh_state->srh) {
7112 		srh_state->hdrlen = srh_state->srh->hdrlen << 3;
7113 		srh_state->valid = true;
7114 	}
7115 	/* it may have been pulled into the head */
7116 	bpf_compute_data_pointers(skb);
7117 }
7118 
7119 BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb,
7120 	   u32, action, void *, param, u32, param_len)
7121 {
7122 	struct seg6_bpf_srh_state *srh_state =
7123 		this_cpu_ptr(&seg6_bpf_srh_states);
7124 	int hdroff = 0;
7125 	int err;
7126 
7127 	lockdep_assert_held(&srh_state->bh_lock);
7128 	switch (action) {
7129 	case SEG6_LOCAL_ACTION_END_X:
7130 		if (!seg6_bpf_has_valid_srh(skb))
7131 			return -EBADMSG;
7132 		if (param_len != sizeof(struct in6_addr))
7133 			return -EINVAL;
7134 		return seg6_lookup_nexthop(skb, (struct in6_addr *)param, 0);
7135 	case SEG6_LOCAL_ACTION_END_T:
7136 		if (!seg6_bpf_has_valid_srh(skb))
7137 			return -EBADMSG;
7138 		if (param_len != sizeof(int))
7139 			return -EINVAL;
7140 		return seg6_lookup_nexthop(skb, NULL, *(int *)param);
7141 	case SEG6_LOCAL_ACTION_END_DT6:
7142 		if (!seg6_bpf_has_valid_srh(skb))
7143 			return -EBADMSG;
7144 		if (param_len != sizeof(int))
7145 			return -EINVAL;
7146 
7147 		if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0)
7148 			return -EBADMSG;
7149 		/* As long as nothing has been changed, the program goes on */
7150 		if (!pskb_may_pull(skb, hdroff + sizeof(struct ipv6hdr))) {
7151 			/* but the head may have moved */
7152 			bpf_update_srh_state(skb);
7153 			return -EBADMSG;
7154 		}
7155 
7156 		__skb_pull(skb, hdroff);
7157 		skb_postpull_rcsum(skb, skb_network_header(skb), hdroff);
7158 		skb_reset_network_header(skb);
7159 		skb_reset_transport_header(skb);
7160 		skb->encapsulation = 0;
7161 
7162 		bpf_compute_data_pointers(skb);
7163 		bpf_update_srh_state(skb);
7164 		return seg6_lookup_nexthop(skb, NULL, *(int *)param);
7165 	case SEG6_LOCAL_ACTION_END_B6:
7166 		if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
7167 			return -EBADMSG;
7168 		err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE,
7169 					  param, param_len);
7170 		/* The header can be in place although err says otherwise */
7171 		bpf_update_srh_state(skb);
7172 
7173 		return err;
7174 	case SEG6_LOCAL_ACTION_END_B6_ENCAP:
7175 		if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
7176 			return -EBADMSG;
7177 		err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6,
7178 					  param, param_len);
7179 		/* The header can be in place although err says otherwise */
7180 		bpf_update_srh_state(skb);
7181 
7182 		return err;
7183 	default:
7184 		return -EINVAL;
7185 	}
7186 }
7187 
7188 static const struct bpf_func_proto bpf_lwt_seg6_action_proto = {
7189 	.func		= bpf_lwt_seg6_action,
7190 	.gpl_only	= false,
7191 	.ret_type	= RET_INTEGER,
7192 	.arg1_type	= ARG_PTR_TO_CTX,
7193 	.arg2_type	= ARG_ANYTHING,
7194 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7195 	.arg4_type	= ARG_MEM_SIZE
7196 };
7197 
7198 BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
7199 	   s32, len)
7200 {
7201 	struct seg6_bpf_srh_state *srh_state =
7202 		this_cpu_ptr(&seg6_bpf_srh_states);
7203 	bool mac_set = skb_mac_header_was_set(skb);
7204 	struct ipv6_sr_hdr *srh = srh_state->srh;
7205 	void *srh_end, *srh_tlvs, *ptr;
7206 	struct ipv6hdr *hdr;
7207 	int srhoff;
7208 	int ret;
7209 
7210 	lockdep_assert_held(&srh_state->bh_lock);
7211 	if (unlikely(srh == NULL))
7212 		return -EINVAL;
7213 
7214 	srhoff = (unsigned char *)srh - skb->data;
7215 	srh_tlvs = (void *)((unsigned char *)srh + sizeof(*srh) +
7216 			((srh->first_segment + 1) << 4));
7217 	srh_end = (void *)((unsigned char *)srh + sizeof(*srh) +
7218 			srh_state->hdrlen);
7219 	ptr = skb->data + offset;
7220 
7221 	if (unlikely(ptr < srh_tlvs || ptr > srh_end))
7222 		return -EFAULT;
7223 	if (unlikely(len < 0 && (void *)((char *)ptr - len) > srh_end))
7224 		return -EFAULT;
7225 	/* The length field of the SRH has 8 bits, in units of 8 bytes. */
7226 	if (unlikely(len > (U8_MAX << 3) - srh_state->hdrlen))
7227 		return -EFAULT;
7228 
7229 	/*
7230 	 * skb->data is the network header here. The MAC header is in front of
7231 	 * it, so bpf_skb_net_hdr_push() and bpf_skb_net_hdr_pop() do not move
7232 	 * it with the other headers.
7233 	 */
7234 	if (len > 0) {
7235 		ret = skb_cow_head(skb, len + skb->mac_len);
7236 		if (unlikely(ret < 0))
7237 			return ret;
7238 
7239 		/* The push can write over the MAC header, so move it first. */
7240 		if (mac_set)
7241 			memmove(skb->data - len - skb->mac_len,
7242 				skb_mac_header(skb), skb->mac_len);
7243 		ret = bpf_skb_net_hdr_push(skb, offset, len);
7244 	} else {
7245 		ret = bpf_skb_net_hdr_pop(skb, offset, -1 * len);
7246 		/* The pop has added -len to skb->mac_header. */
7247 		if (likely(!ret) && mac_set)
7248 			memmove(skb->data - skb->mac_len,
7249 				skb_mac_header(skb) + len, skb->mac_len);
7250 	}
7251 
7252 	bpf_compute_data_pointers(skb);
7253 	if (unlikely(ret < 0))
7254 		return ret;
7255 
7256 	if (mac_set)
7257 		skb_set_mac_header(skb, -skb->mac_len);
7258 	else
7259 		skb_unset_mac_header(skb);
7260 
7261 	hdr = (struct ipv6hdr *)skb->data;
7262 	hdr->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
7263 
7264 	/* The bytes in front of offset have moved together with skb->data. */
7265 	srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
7266 	srh_state->hdrlen += len;
7267 	srh_state->valid = false;
7268 	return 0;
7269 }
7270 
7271 static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
7272 	.func		= bpf_lwt_seg6_adjust_srh,
7273 	.gpl_only	= false,
7274 	.ret_type	= RET_INTEGER,
7275 	.arg1_type	= ARG_PTR_TO_CTX,
7276 	.arg2_type	= ARG_ANYTHING,
7277 	.arg3_type	= ARG_ANYTHING,
7278 };
7279 #endif /* CONFIG_IPV6_SEG6_BPF */
7280 
7281 #ifdef CONFIG_INET
7282 static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
7283 			      int dif, int sdif, u8 family, u8 proto)
7284 {
7285 	bool refcounted = false;
7286 	struct sock *sk = NULL;
7287 
7288 	if (family == AF_INET) {
7289 		__be32 src4 = tuple->ipv4.saddr;
7290 		__be32 dst4 = tuple->ipv4.daddr;
7291 
7292 		if (proto == IPPROTO_TCP)
7293 			sk = __inet_lookup(net, NULL, 0,
7294 					   src4, tuple->ipv4.sport,
7295 					   dst4, tuple->ipv4.dport,
7296 					   dif, sdif, &refcounted);
7297 		else
7298 			sk = __udp4_lib_lookup(net, src4, tuple->ipv4.sport,
7299 					       dst4, tuple->ipv4.dport,
7300 					       dif, sdif, NULL);
7301 #if IS_ENABLED(CONFIG_IPV6)
7302 	} else {
7303 		struct in6_addr *src6 = (struct in6_addr *)&tuple->ipv6.saddr;
7304 		struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr;
7305 
7306 		if (proto == IPPROTO_TCP)
7307 			sk = __inet6_lookup(net, NULL, 0,
7308 					    src6, tuple->ipv6.sport,
7309 					    dst6, ntohs(tuple->ipv6.dport),
7310 					    dif, sdif, &refcounted);
7311 		else if (likely(ipv6_mod_enabled()))
7312 			sk = __udp6_lib_lookup(net, src6, tuple->ipv6.sport,
7313 					       dst6, tuple->ipv6.dport,
7314 					       dif, sdif, NULL);
7315 #endif
7316 	}
7317 
7318 	if (unlikely(sk && !refcounted && !sock_flag(sk, SOCK_RCU_FREE))) {
7319 		WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
7320 		sk = NULL;
7321 	}
7322 	return sk;
7323 }
7324 
7325 /* bpf_skc_lookup performs the core lookup for different types of sockets,
7326  * taking a reference on the socket if it doesn't have the flag SOCK_RCU_FREE.
7327  */
7328 static struct sock *
7329 __bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
7330 		 struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
7331 		 u64 flags, int sdif)
7332 {
7333 	struct sock *sk = NULL;
7334 	struct net *net;
7335 	u8 family;
7336 
7337 	if (len == sizeof(tuple->ipv4))
7338 		family = AF_INET;
7339 	else if (len == sizeof(tuple->ipv6))
7340 		family = AF_INET6;
7341 	else
7342 		return NULL;
7343 
7344 	if (unlikely(flags || !((s32)netns_id < 0 || netns_id <= S32_MAX)))
7345 		goto out;
7346 
7347 	if (sdif < 0) {
7348 		if (family == AF_INET)
7349 			sdif = inet_sdif(skb);
7350 		else
7351 			sdif = inet6_sdif(skb);
7352 	}
7353 
7354 	if ((s32)netns_id < 0) {
7355 		net = caller_net;
7356 		sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);
7357 	} else {
7358 		net = get_net_ns_by_id(caller_net, netns_id);
7359 		if (unlikely(!net))
7360 			goto out;
7361 		sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);
7362 		put_net(net);
7363 	}
7364 
7365 out:
7366 	return sk;
7367 }
7368 
7369 static struct sock *
7370 bpf_sk_lookup_full_sk(struct sock *sk)
7371 {
7372 	struct sock *sk2 = sk_to_full_sk(sk);
7373 
7374 	/*
7375 	 * sk_to_full_sk() may return sk->rsk_listener, make sure the original
7376 	 * sk sock refcnt is decremented to prevent a request_sock leak.
7377 	 */
7378 	if (sk2 != sk) {
7379 		sock_gen_put(sk);
7380 		/* Ensure there is no need to bump sk2 refcnt. */
7381 		if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) {
7382 			WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
7383 			return NULL;
7384 		}
7385 		sk = sk2;
7386 	}
7387 
7388 	return sk;
7389 }
7390 
7391 static struct sock *
7392 __bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
7393 		struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
7394 		u64 flags, int sdif)
7395 {
7396 	struct sock *sk = __bpf_skc_lookup(skb, tuple, len, caller_net,
7397 					   ifindex, proto, netns_id, flags,
7398 					   sdif);
7399 	if (sk)
7400 		sk = bpf_sk_lookup_full_sk(sk);
7401 	return sk;
7402 }
7403 
7404 static struct sock *
7405 bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
7406 	       u8 proto, u64 netns_id, u64 flags)
7407 {
7408 	struct net *caller_net;
7409 	int ifindex;
7410 
7411 	if (skb->dev) {
7412 		caller_net = dev_net(skb->dev);
7413 		ifindex = skb->dev->ifindex;
7414 	} else {
7415 		caller_net = sock_net(skb->sk);
7416 		ifindex = 0;
7417 	}
7418 
7419 	return __bpf_skc_lookup(skb, tuple, len, caller_net, ifindex, proto,
7420 				netns_id, flags, -1);
7421 }
7422 
7423 static struct sock *
7424 bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
7425 	      u8 proto, u64 netns_id, u64 flags)
7426 {
7427 	struct sock *sk = bpf_skc_lookup(skb, tuple, len, proto, netns_id,
7428 					 flags);
7429 	if (sk)
7430 		sk = bpf_sk_lookup_full_sk(sk);
7431 	return sk;
7432 }
7433 
7434 BPF_CALL_5(bpf_skc_lookup_tcp, struct sk_buff *, skb,
7435 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7436 {
7437 	return (unsigned long)bpf_skc_lookup(skb, tuple, len, IPPROTO_TCP,
7438 					     netns_id, flags);
7439 }
7440 
7441 static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = {
7442 	.func		= bpf_skc_lookup_tcp,
7443 	.gpl_only	= false,
7444 	.pkt_access	= true,
7445 	.ret_type	= RET_PTR_TO_SOCK_COMMON_OR_NULL,
7446 	.arg1_type	= ARG_PTR_TO_CTX,
7447 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7448 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7449 	.arg4_type	= ARG_ANYTHING,
7450 	.arg5_type	= ARG_ANYTHING,
7451 };
7452 
7453 BPF_CALL_5(bpf_sk_lookup_tcp, struct sk_buff *, skb,
7454 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7455 {
7456 	return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_TCP,
7457 					    netns_id, flags);
7458 }
7459 
7460 static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = {
7461 	.func		= bpf_sk_lookup_tcp,
7462 	.gpl_only	= false,
7463 	.pkt_access	= true,
7464 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7465 	.arg1_type	= ARG_PTR_TO_CTX,
7466 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7467 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7468 	.arg4_type	= ARG_ANYTHING,
7469 	.arg5_type	= ARG_ANYTHING,
7470 };
7471 
7472 BPF_CALL_5(bpf_sk_lookup_udp, struct sk_buff *, skb,
7473 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7474 {
7475 	return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_UDP,
7476 					    netns_id, flags);
7477 }
7478 
7479 static const struct bpf_func_proto bpf_sk_lookup_udp_proto = {
7480 	.func		= bpf_sk_lookup_udp,
7481 	.gpl_only	= false,
7482 	.pkt_access	= true,
7483 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7484 	.arg1_type	= ARG_PTR_TO_CTX,
7485 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7486 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7487 	.arg4_type	= ARG_ANYTHING,
7488 	.arg5_type	= ARG_ANYTHING,
7489 };
7490 
7491 BPF_CALL_5(bpf_tc_skc_lookup_tcp, struct sk_buff *, skb,
7492 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7493 {
7494 	struct net_device *dev = skb->dev;
7495 	int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7496 	struct net *caller_net = dev_net(dev);
7497 
7498 	return (unsigned long)__bpf_skc_lookup(skb, tuple, len, caller_net,
7499 					       ifindex, IPPROTO_TCP, netns_id,
7500 					       flags, sdif);
7501 }
7502 
7503 static const struct bpf_func_proto bpf_tc_skc_lookup_tcp_proto = {
7504 	.func		= bpf_tc_skc_lookup_tcp,
7505 	.gpl_only	= false,
7506 	.pkt_access	= true,
7507 	.ret_type	= RET_PTR_TO_SOCK_COMMON_OR_NULL,
7508 	.arg1_type	= ARG_PTR_TO_CTX,
7509 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7510 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7511 	.arg4_type	= ARG_ANYTHING,
7512 	.arg5_type	= ARG_ANYTHING,
7513 };
7514 
7515 BPF_CALL_5(bpf_tc_sk_lookup_tcp, struct sk_buff *, skb,
7516 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7517 {
7518 	struct net_device *dev = skb->dev;
7519 	int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7520 	struct net *caller_net = dev_net(dev);
7521 
7522 	return (unsigned long)__bpf_sk_lookup(skb, tuple, len, caller_net,
7523 					      ifindex, IPPROTO_TCP, netns_id,
7524 					      flags, sdif);
7525 }
7526 
7527 static const struct bpf_func_proto bpf_tc_sk_lookup_tcp_proto = {
7528 	.func		= bpf_tc_sk_lookup_tcp,
7529 	.gpl_only	= false,
7530 	.pkt_access	= true,
7531 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7532 	.arg1_type	= ARG_PTR_TO_CTX,
7533 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7534 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7535 	.arg4_type	= ARG_ANYTHING,
7536 	.arg5_type	= ARG_ANYTHING,
7537 };
7538 
7539 BPF_CALL_5(bpf_tc_sk_lookup_udp, struct sk_buff *, skb,
7540 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7541 {
7542 	struct net_device *dev = skb->dev;
7543 	int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7544 	struct net *caller_net = dev_net(dev);
7545 
7546 	return (unsigned long)__bpf_sk_lookup(skb, tuple, len, caller_net,
7547 					      ifindex, IPPROTO_UDP, netns_id,
7548 					      flags, sdif);
7549 }
7550 
7551 static const struct bpf_func_proto bpf_tc_sk_lookup_udp_proto = {
7552 	.func		= bpf_tc_sk_lookup_udp,
7553 	.gpl_only	= false,
7554 	.pkt_access	= true,
7555 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7556 	.arg1_type	= ARG_PTR_TO_CTX,
7557 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7558 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7559 	.arg4_type	= ARG_ANYTHING,
7560 	.arg5_type	= ARG_ANYTHING,
7561 };
7562 
7563 BPF_CALL_1(bpf_sk_release, struct sock *, sk)
7564 {
7565 	if (sk && sk_is_refcounted(sk))
7566 		sock_gen_put(sk);
7567 	return 0;
7568 }
7569 
7570 static const struct bpf_func_proto bpf_sk_release_proto = {
7571 	.func		= bpf_sk_release,
7572 	.gpl_only	= false,
7573 	.ret_type	= RET_INTEGER,
7574 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON | OBJ_RELEASE,
7575 };
7576 
7577 BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx,
7578 	   struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
7579 {
7580 	struct net_device *dev = ctx->rxq->dev;
7581 	int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7582 	struct net *caller_net = dev_net(dev);
7583 
7584 	return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
7585 					      ifindex, IPPROTO_UDP, netns_id,
7586 					      flags, sdif);
7587 }
7588 
7589 static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = {
7590 	.func           = bpf_xdp_sk_lookup_udp,
7591 	.gpl_only       = false,
7592 	.pkt_access     = true,
7593 	.ret_type       = RET_PTR_TO_SOCKET_OR_NULL,
7594 	.arg1_type      = ARG_PTR_TO_CTX,
7595 	.arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
7596 	.arg3_type      = ARG_MEM_SIZE_OR_ZERO,
7597 	.arg4_type      = ARG_ANYTHING,
7598 	.arg5_type      = ARG_ANYTHING,
7599 };
7600 
7601 BPF_CALL_5(bpf_xdp_skc_lookup_tcp, struct xdp_buff *, ctx,
7602 	   struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
7603 {
7604 	struct net_device *dev = ctx->rxq->dev;
7605 	int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7606 	struct net *caller_net = dev_net(dev);
7607 
7608 	return (unsigned long)__bpf_skc_lookup(NULL, tuple, len, caller_net,
7609 					       ifindex, IPPROTO_TCP, netns_id,
7610 					       flags, sdif);
7611 }
7612 
7613 static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = {
7614 	.func           = bpf_xdp_skc_lookup_tcp,
7615 	.gpl_only       = false,
7616 	.pkt_access     = true,
7617 	.ret_type       = RET_PTR_TO_SOCK_COMMON_OR_NULL,
7618 	.arg1_type      = ARG_PTR_TO_CTX,
7619 	.arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
7620 	.arg3_type      = ARG_MEM_SIZE_OR_ZERO,
7621 	.arg4_type      = ARG_ANYTHING,
7622 	.arg5_type      = ARG_ANYTHING,
7623 };
7624 
7625 BPF_CALL_5(bpf_xdp_sk_lookup_tcp, struct xdp_buff *, ctx,
7626 	   struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
7627 {
7628 	struct net_device *dev = ctx->rxq->dev;
7629 	int ifindex = dev->ifindex, sdif = dev_sdif(dev);
7630 	struct net *caller_net = dev_net(dev);
7631 
7632 	return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
7633 					      ifindex, IPPROTO_TCP, netns_id,
7634 					      flags, sdif);
7635 }
7636 
7637 static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = {
7638 	.func           = bpf_xdp_sk_lookup_tcp,
7639 	.gpl_only       = false,
7640 	.pkt_access     = true,
7641 	.ret_type       = RET_PTR_TO_SOCKET_OR_NULL,
7642 	.arg1_type      = ARG_PTR_TO_CTX,
7643 	.arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
7644 	.arg3_type      = ARG_MEM_SIZE_OR_ZERO,
7645 	.arg4_type      = ARG_ANYTHING,
7646 	.arg5_type      = ARG_ANYTHING,
7647 };
7648 
7649 BPF_CALL_5(bpf_sock_addr_skc_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
7650 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7651 {
7652 	return (unsigned long)__bpf_skc_lookup(NULL, tuple, len,
7653 					       sock_net(ctx->sk), 0,
7654 					       IPPROTO_TCP, netns_id, flags,
7655 					       -1);
7656 }
7657 
7658 static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = {
7659 	.func		= bpf_sock_addr_skc_lookup_tcp,
7660 	.gpl_only	= false,
7661 	.ret_type	= RET_PTR_TO_SOCK_COMMON_OR_NULL,
7662 	.arg1_type	= ARG_PTR_TO_CTX,
7663 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7664 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7665 	.arg4_type	= ARG_ANYTHING,
7666 	.arg5_type	= ARG_ANYTHING,
7667 };
7668 
7669 BPF_CALL_5(bpf_sock_addr_sk_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
7670 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7671 {
7672 	return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
7673 					      sock_net(ctx->sk), 0, IPPROTO_TCP,
7674 					      netns_id, flags, -1);
7675 }
7676 
7677 static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = {
7678 	.func		= bpf_sock_addr_sk_lookup_tcp,
7679 	.gpl_only	= false,
7680 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7681 	.arg1_type	= ARG_PTR_TO_CTX,
7682 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7683 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7684 	.arg4_type	= ARG_ANYTHING,
7685 	.arg5_type	= ARG_ANYTHING,
7686 };
7687 
7688 BPF_CALL_5(bpf_sock_addr_sk_lookup_udp, struct bpf_sock_addr_kern *, ctx,
7689 	   struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
7690 {
7691 	return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
7692 					      sock_net(ctx->sk), 0, IPPROTO_UDP,
7693 					      netns_id, flags, -1);
7694 }
7695 
7696 static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = {
7697 	.func		= bpf_sock_addr_sk_lookup_udp,
7698 	.gpl_only	= false,
7699 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7700 	.arg1_type	= ARG_PTR_TO_CTX,
7701 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
7702 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
7703 	.arg4_type	= ARG_ANYTHING,
7704 	.arg5_type	= ARG_ANYTHING,
7705 };
7706 
7707 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
7708 				  struct bpf_insn_access_aux *info)
7709 {
7710 	if (off < 0 || off >= offsetofend(struct bpf_tcp_sock,
7711 					  icsk_retransmits))
7712 		return false;
7713 
7714 	if (off % size != 0)
7715 		return false;
7716 
7717 	switch (off) {
7718 	case bpf_ctx_range(struct bpf_tcp_sock, bytes_received):
7719 	case bpf_ctx_range(struct bpf_tcp_sock, bytes_acked):
7720 		return size == sizeof(__u64);
7721 	default:
7722 		return size == sizeof(__u32);
7723 	}
7724 }
7725 
7726 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
7727 				    const struct bpf_insn *si,
7728 				    struct bpf_insn *insn_buf,
7729 				    struct bpf_prog *prog, u32 *target_size)
7730 {
7731 	struct bpf_insn *insn = insn_buf;
7732 
7733 #define BPF_TCP_SOCK_GET_COMMON(FIELD)					\
7734 	do {								\
7735 		BUILD_BUG_ON(sizeof_field(struct tcp_sock, FIELD) >	\
7736 			     sizeof_field(struct bpf_tcp_sock, FIELD));	\
7737 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_sock, FIELD),\
7738 				      si->dst_reg, si->src_reg,		\
7739 				      offsetof(struct tcp_sock, FIELD)); \
7740 	} while (0)
7741 
7742 #define BPF_INET_SOCK_GET_COMMON(FIELD)					\
7743 	do {								\
7744 		BUILD_BUG_ON(sizeof_field(struct inet_connection_sock,	\
7745 					  FIELD) >			\
7746 			     sizeof_field(struct bpf_tcp_sock, FIELD));	\
7747 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			\
7748 					struct inet_connection_sock,	\
7749 					FIELD),				\
7750 				      si->dst_reg, si->src_reg,		\
7751 				      offsetof(				\
7752 					struct inet_connection_sock,	\
7753 					FIELD));			\
7754 	} while (0)
7755 
7756 	BTF_TYPE_EMIT(struct bpf_tcp_sock);
7757 
7758 	switch (si->off) {
7759 	case offsetof(struct bpf_tcp_sock, rtt_min):
7760 		BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
7761 			     sizeof(struct minmax));
7762 		BUILD_BUG_ON(sizeof(struct minmax) <
7763 			     sizeof(struct minmax_sample));
7764 
7765 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
7766 				      offsetof(struct tcp_sock, rtt_min) +
7767 				      offsetof(struct minmax_sample, v));
7768 		break;
7769 	case offsetof(struct bpf_tcp_sock, snd_cwnd):
7770 		BPF_TCP_SOCK_GET_COMMON(snd_cwnd);
7771 		break;
7772 	case offsetof(struct bpf_tcp_sock, srtt_us):
7773 		BPF_TCP_SOCK_GET_COMMON(srtt_us);
7774 		break;
7775 	case offsetof(struct bpf_tcp_sock, snd_ssthresh):
7776 		BPF_TCP_SOCK_GET_COMMON(snd_ssthresh);
7777 		break;
7778 	case offsetof(struct bpf_tcp_sock, rcv_nxt):
7779 		BPF_TCP_SOCK_GET_COMMON(rcv_nxt);
7780 		break;
7781 	case offsetof(struct bpf_tcp_sock, snd_nxt):
7782 		BPF_TCP_SOCK_GET_COMMON(snd_nxt);
7783 		break;
7784 	case offsetof(struct bpf_tcp_sock, snd_una):
7785 		BPF_TCP_SOCK_GET_COMMON(snd_una);
7786 		break;
7787 	case offsetof(struct bpf_tcp_sock, mss_cache):
7788 		BPF_TCP_SOCK_GET_COMMON(mss_cache);
7789 		break;
7790 	case offsetof(struct bpf_tcp_sock, ecn_flags):
7791 		BPF_TCP_SOCK_GET_COMMON(ecn_flags);
7792 		break;
7793 	case offsetof(struct bpf_tcp_sock, rate_delivered):
7794 		BPF_TCP_SOCK_GET_COMMON(rate_delivered);
7795 		break;
7796 	case offsetof(struct bpf_tcp_sock, rate_interval_us):
7797 		BPF_TCP_SOCK_GET_COMMON(rate_interval_us);
7798 		break;
7799 	case offsetof(struct bpf_tcp_sock, packets_out):
7800 		BPF_TCP_SOCK_GET_COMMON(packets_out);
7801 		break;
7802 	case offsetof(struct bpf_tcp_sock, retrans_out):
7803 		BPF_TCP_SOCK_GET_COMMON(retrans_out);
7804 		break;
7805 	case offsetof(struct bpf_tcp_sock, total_retrans):
7806 		BPF_TCP_SOCK_GET_COMMON(total_retrans);
7807 		break;
7808 	case offsetof(struct bpf_tcp_sock, segs_in):
7809 		BPF_TCP_SOCK_GET_COMMON(segs_in);
7810 		break;
7811 	case offsetof(struct bpf_tcp_sock, data_segs_in):
7812 		BPF_TCP_SOCK_GET_COMMON(data_segs_in);
7813 		break;
7814 	case offsetof(struct bpf_tcp_sock, segs_out):
7815 		BPF_TCP_SOCK_GET_COMMON(segs_out);
7816 		break;
7817 	case offsetof(struct bpf_tcp_sock, data_segs_out):
7818 		BPF_TCP_SOCK_GET_COMMON(data_segs_out);
7819 		break;
7820 	case offsetof(struct bpf_tcp_sock, lost_out):
7821 		BPF_TCP_SOCK_GET_COMMON(lost_out);
7822 		break;
7823 	case offsetof(struct bpf_tcp_sock, sacked_out):
7824 		BPF_TCP_SOCK_GET_COMMON(sacked_out);
7825 		break;
7826 	case offsetof(struct bpf_tcp_sock, bytes_received):
7827 		BPF_TCP_SOCK_GET_COMMON(bytes_received);
7828 		break;
7829 	case offsetof(struct bpf_tcp_sock, bytes_acked):
7830 		BPF_TCP_SOCK_GET_COMMON(bytes_acked);
7831 		break;
7832 	case offsetof(struct bpf_tcp_sock, dsack_dups):
7833 		BPF_TCP_SOCK_GET_COMMON(dsack_dups);
7834 		break;
7835 	case offsetof(struct bpf_tcp_sock, delivered):
7836 		BPF_TCP_SOCK_GET_COMMON(delivered);
7837 		break;
7838 	case offsetof(struct bpf_tcp_sock, delivered_ce):
7839 		BPF_TCP_SOCK_GET_COMMON(delivered_ce);
7840 		break;
7841 	case offsetof(struct bpf_tcp_sock, icsk_retransmits):
7842 		BPF_INET_SOCK_GET_COMMON(icsk_retransmits);
7843 		break;
7844 	}
7845 
7846 	return insn - insn_buf;
7847 }
7848 
7849 BPF_CALL_1(bpf_tcp_sock, struct sock *, sk)
7850 {
7851 	if (sk_fullsock(sk) && sk_is_tcp(sk))
7852 		return (unsigned long)sk;
7853 
7854 	return (unsigned long)NULL;
7855 }
7856 
7857 const struct bpf_func_proto bpf_tcp_sock_proto = {
7858 	.func		= bpf_tcp_sock,
7859 	.gpl_only	= false,
7860 	.ret_type	= RET_PTR_TO_TCP_SOCK_OR_NULL,
7861 	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
7862 };
7863 
7864 BPF_CALL_1(bpf_get_listener_sock, struct sock *, sk)
7865 {
7866 	sk = sk_to_full_sk(sk);
7867 
7868 	if (sk && sk->sk_state == TCP_LISTEN && sock_flag(sk, SOCK_RCU_FREE))
7869 		return (unsigned long)sk;
7870 
7871 	return (unsigned long)NULL;
7872 }
7873 
7874 static const struct bpf_func_proto bpf_get_listener_sock_proto = {
7875 	.func		= bpf_get_listener_sock,
7876 	.gpl_only	= false,
7877 	.ret_type	= RET_PTR_TO_SOCKET_OR_NULL,
7878 	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
7879 };
7880 
7881 BPF_CALL_1(bpf_skb_ecn_set_ce, struct sk_buff *, skb)
7882 {
7883 	unsigned int iphdr_len;
7884 
7885 	switch (skb_protocol(skb, true)) {
7886 	case cpu_to_be16(ETH_P_IP):
7887 		iphdr_len = sizeof(struct iphdr);
7888 		break;
7889 	case cpu_to_be16(ETH_P_IPV6):
7890 		iphdr_len = sizeof(struct ipv6hdr);
7891 		break;
7892 	default:
7893 		return 0;
7894 	}
7895 
7896 	if (skb_headlen(skb) < iphdr_len)
7897 		return 0;
7898 
7899 	if (skb_cloned(skb) && !skb_clone_writable(skb, iphdr_len))
7900 		return 0;
7901 
7902 	return INET_ECN_set_ce(skb);
7903 }
7904 
7905 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
7906 				  struct bpf_insn_access_aux *info)
7907 {
7908 	if (off < 0 || off >= offsetofend(struct bpf_xdp_sock, queue_id))
7909 		return false;
7910 
7911 	if (off % size != 0)
7912 		return false;
7913 
7914 	switch (off) {
7915 	default:
7916 		return size == sizeof(__u32);
7917 	}
7918 }
7919 
7920 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
7921 				    const struct bpf_insn *si,
7922 				    struct bpf_insn *insn_buf,
7923 				    struct bpf_prog *prog, u32 *target_size)
7924 {
7925 	struct bpf_insn *insn = insn_buf;
7926 
7927 #define BPF_XDP_SOCK_GET(FIELD)						\
7928 	do {								\
7929 		BUILD_BUG_ON(sizeof_field(struct xdp_sock, FIELD) >	\
7930 			     sizeof_field(struct bpf_xdp_sock, FIELD));	\
7931 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_sock, FIELD),\
7932 				      si->dst_reg, si->src_reg,		\
7933 				      offsetof(struct xdp_sock, FIELD)); \
7934 	} while (0)
7935 
7936 	BTF_TYPE_EMIT(struct bpf_xdp_sock);
7937 
7938 	switch (si->off) {
7939 	case offsetof(struct bpf_xdp_sock, queue_id):
7940 		BPF_XDP_SOCK_GET(queue_id);
7941 		break;
7942 	}
7943 
7944 	return insn - insn_buf;
7945 }
7946 
7947 static const struct bpf_func_proto bpf_skb_ecn_set_ce_proto = {
7948 	.func           = bpf_skb_ecn_set_ce,
7949 	.gpl_only       = false,
7950 	.ret_type       = RET_INTEGER,
7951 	.arg1_type      = ARG_PTR_TO_CTX,
7952 };
7953 
7954 BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
7955 	   struct tcphdr *, th, u32, th_len)
7956 {
7957 #ifdef CONFIG_SYN_COOKIES
7958 	int ret;
7959 
7960 	if (unlikely(!sk || th_len < sizeof(*th)))
7961 		return -EINVAL;
7962 
7963 	/* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */
7964 	if (sk->sk_state != TCP_LISTEN || sk->sk_protocol != IPPROTO_TCP)
7965 		return -EINVAL;
7966 
7967 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies))
7968 		return -EINVAL;
7969 
7970 	if (!th->ack || th->rst || th->syn)
7971 		return -ENOENT;
7972 
7973 	if (unlikely(iph_len < sizeof(struct iphdr)))
7974 		return -EINVAL;
7975 
7976 	if (tcp_synq_no_recent_overflow(sk))
7977 		return -ENOENT;
7978 
7979 	/* Both struct iphdr and struct ipv6hdr have the version field at the
7980 	 * same offset so we can cast to the shorter header (struct iphdr).
7981 	 */
7982 	switch (((struct iphdr *)iph)->version) {
7983 	case 4:
7984 		if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
7985 			return -EINVAL;
7986 
7987 		ret = __cookie_v4_check((struct iphdr *)iph, th);
7988 		break;
7989 
7990 #if IS_ENABLED(CONFIG_IPV6)
7991 	case 6:
7992 		if (unlikely(iph_len < sizeof(struct ipv6hdr)))
7993 			return -EINVAL;
7994 
7995 		if (sk->sk_family != AF_INET6)
7996 			return -EINVAL;
7997 
7998 		ret = __cookie_v6_check((struct ipv6hdr *)iph, th);
7999 		break;
8000 #endif /* CONFIG_IPV6 */
8001 
8002 	default:
8003 		return -EPROTONOSUPPORT;
8004 	}
8005 
8006 	if (ret > 0)
8007 		return 0;
8008 
8009 	return -ENOENT;
8010 #else
8011 	return -ENOTSUPP;
8012 #endif
8013 }
8014 
8015 static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = {
8016 	.func		= bpf_tcp_check_syncookie,
8017 	.gpl_only	= true,
8018 	.pkt_access	= true,
8019 	.ret_type	= RET_INTEGER,
8020 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
8021 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8022 	.arg3_type	= ARG_MEM_SIZE,
8023 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8024 	.arg5_type	= ARG_MEM_SIZE,
8025 };
8026 
8027 BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
8028 	   struct tcphdr *, th, u32, th_len)
8029 {
8030 #ifdef CONFIG_SYN_COOKIES
8031 	u32 cookie;
8032 	u16 mss;
8033 
8034 	if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4))
8035 		return -EINVAL;
8036 
8037 	if (sk->sk_state != TCP_LISTEN || sk->sk_protocol != IPPROTO_TCP)
8038 		return -EINVAL;
8039 
8040 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies))
8041 		return -ENOENT;
8042 
8043 	if (!th->syn || th->ack || th->fin || th->rst)
8044 		return -EINVAL;
8045 
8046 	if (unlikely(iph_len < sizeof(struct iphdr)))
8047 		return -EINVAL;
8048 
8049 	/* Both struct iphdr and struct ipv6hdr have the version field at the
8050 	 * same offset so we can cast to the shorter header (struct iphdr).
8051 	 */
8052 	switch (((struct iphdr *)iph)->version) {
8053 	case 4:
8054 		if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
8055 			return -EINVAL;
8056 
8057 		mss = tcp_v4_get_syncookie(sk, iph, th, &cookie);
8058 		break;
8059 
8060 #if IS_ENABLED(CONFIG_IPV6)
8061 	case 6:
8062 		if (unlikely(iph_len < sizeof(struct ipv6hdr)))
8063 			return -EINVAL;
8064 
8065 		if (sk->sk_family != AF_INET6)
8066 			return -EINVAL;
8067 
8068 		mss = tcp_v6_get_syncookie(sk, iph, th, &cookie);
8069 		break;
8070 #endif /* CONFIG_IPV6 */
8071 
8072 	default:
8073 		return -EPROTONOSUPPORT;
8074 	}
8075 	if (mss == 0)
8076 		return -ENOENT;
8077 
8078 	return cookie | ((u64)mss << 32);
8079 #else
8080 	return -EOPNOTSUPP;
8081 #endif /* CONFIG_SYN_COOKIES */
8082 }
8083 
8084 static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = {
8085 	.func		= bpf_tcp_gen_syncookie,
8086 	.gpl_only	= true, /* __cookie_v*_init_sequence() is GPL */
8087 	.pkt_access	= true,
8088 	.ret_type	= RET_INTEGER,
8089 	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
8090 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8091 	.arg3_type	= ARG_MEM_SIZE,
8092 	.arg4_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8093 	.arg5_type	= ARG_MEM_SIZE,
8094 };
8095 
8096 BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags)
8097 {
8098 	if (!sk || flags != 0)
8099 		return -EINVAL;
8100 	if (!skb_at_tc_ingress(skb))
8101 		return -EOPNOTSUPP;
8102 	if (unlikely(dev_net(skb->dev) != sock_net(sk)))
8103 		return -ENETUNREACH;
8104 	if (sk_unhashed(sk))
8105 		return -EOPNOTSUPP;
8106 	if (sk_is_refcounted(sk) &&
8107 	    unlikely(!refcount_inc_not_zero_acquire(&sk->sk_refcnt)))
8108 		return -ENOENT;
8109 
8110 	skb_orphan(skb);
8111 	skb->sk = sk;
8112 	skb->destructor = sock_pfree;
8113 
8114 	return 0;
8115 }
8116 
8117 static const struct bpf_func_proto bpf_sk_assign_proto = {
8118 	.func		= bpf_sk_assign,
8119 	.gpl_only	= false,
8120 	.ret_type	= RET_INTEGER,
8121 	.arg1_type      = ARG_PTR_TO_CTX,
8122 	.arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
8123 	.arg3_type	= ARG_ANYTHING,
8124 };
8125 
8126 static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend,
8127 				    u8 search_kind, const u8 *magic,
8128 				    u8 magic_len, bool *eol)
8129 {
8130 	u8 kind, kind_len;
8131 
8132 	*eol = false;
8133 
8134 	while (op < opend) {
8135 		kind = op[0];
8136 
8137 		if (kind == TCPOPT_EOL) {
8138 			*eol = true;
8139 			return ERR_PTR(-ENOMSG);
8140 		} else if (kind == TCPOPT_NOP) {
8141 			op++;
8142 			continue;
8143 		}
8144 
8145 		if (opend - op < 2 || opend - op < op[1] || op[1] < 2)
8146 			/* Something is wrong in the received header.
8147 			 * Follow the TCP stack's tcp_parse_options()
8148 			 * and just bail here.
8149 			 */
8150 			return ERR_PTR(-EFAULT);
8151 
8152 		kind_len = op[1];
8153 		if (search_kind == kind) {
8154 			if (!magic_len)
8155 				return op;
8156 
8157 			if (magic_len > kind_len - 2)
8158 				return ERR_PTR(-ENOMSG);
8159 
8160 			if (!memcmp(&op[2], magic, magic_len))
8161 				return op;
8162 		}
8163 
8164 		op += kind_len;
8165 	}
8166 
8167 	return ERR_PTR(-ENOMSG);
8168 }
8169 
8170 int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock,
8171 				void *search_res, u32 len, u64 flags)
8172 {
8173 	bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN;
8174 	const u8 *op, *opend, *magic, *search = search_res;
8175 	u8 search_kind, search_len, copy_len, magic_len;
8176 	int ret;
8177 
8178 	/* 2 byte is the minimal option len except TCPOPT_NOP and
8179 	 * TCPOPT_EOL which are useless for the bpf prog to learn
8180 	 * and this helper disallow loading them also.
8181 	 */
8182 	if (len < 2 || flags & ~BPF_LOAD_HDR_OPT_TCP_SYN)
8183 		return -EINVAL;
8184 
8185 	search_kind = search[0];
8186 	search_len = search[1];
8187 
8188 	if (search_len > len || search_kind == TCPOPT_NOP ||
8189 	    search_kind == TCPOPT_EOL)
8190 		return -EINVAL;
8191 
8192 	if (search_kind == TCPOPT_EXP || search_kind == 253) {
8193 		/* 16 or 32 bit magic.  +2 for kind and kind length */
8194 		if (search_len != 4 && search_len != 6)
8195 			return -EINVAL;
8196 		magic = &search[2];
8197 		magic_len = search_len - 2;
8198 	} else {
8199 		if (search_len)
8200 			return -EINVAL;
8201 		magic = NULL;
8202 		magic_len = 0;
8203 	}
8204 
8205 	if (load_syn) {
8206 		ret = bpf_sock_ops_get_syn(bpf_sock, TCP_BPF_SYN, &op);
8207 		if (ret < 0)
8208 			return ret;
8209 
8210 		opend = op + ret;
8211 		op += sizeof(struct tcphdr);
8212 	} else {
8213 		if (!bpf_sock->skb ||
8214 		    bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB)
8215 			/* This bpf_sock->op cannot call this helper */
8216 			return -EPERM;
8217 
8218 		opend = bpf_sock->skb_data_end;
8219 		op = bpf_sock->skb->data + sizeof(struct tcphdr);
8220 	}
8221 
8222 	op = bpf_search_tcp_opt(op, opend, search_kind, magic, magic_len,
8223 				&eol);
8224 	if (IS_ERR(op))
8225 		return PTR_ERR(op);
8226 
8227 	copy_len = op[1];
8228 	ret = copy_len;
8229 	if (copy_len > len) {
8230 		ret = -ENOSPC;
8231 		copy_len = len;
8232 	}
8233 
8234 	memcpy(search_res, op, copy_len);
8235 	return ret;
8236 }
8237 
8238 BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
8239 	   void *, search_res, u32, len, u64, flags)
8240 {
8241 	if (!is_locked_tcp_sock_ops(bpf_sock))
8242 		return -EOPNOTSUPP;
8243 
8244 	return __bpf_sock_ops_load_hdr_opt(bpf_sock, search_res, len, flags);
8245 }
8246 
8247 static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = {
8248 	.func		= bpf_sock_ops_load_hdr_opt,
8249 	.gpl_only	= false,
8250 	.ret_type	= RET_INTEGER,
8251 	.arg1_type	= ARG_PTR_TO_CTX,
8252 	.arg2_type	= ARG_PTR_TO_MEM | MEM_WRITE,
8253 	.arg3_type	= ARG_MEM_SIZE,
8254 	.arg4_type	= ARG_ANYTHING,
8255 };
8256 
8257 int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock,
8258 				 const void *from, u32 len, u64 flags)
8259 {
8260 	u8 new_kind, new_kind_len, magic_len = 0, *opend;
8261 	const u8 *op, *new_op, *magic = NULL;
8262 	struct sk_buff *skb;
8263 	bool eol;
8264 
8265 	if (len < 2 || flags)
8266 		return -EINVAL;
8267 
8268 	new_op = from;
8269 	new_kind = new_op[0];
8270 	new_kind_len = new_op[1];
8271 
8272 	if (new_kind_len > len || new_kind == TCPOPT_NOP ||
8273 	    new_kind == TCPOPT_EOL)
8274 		return -EINVAL;
8275 
8276 	if (new_kind_len > bpf_sock->remaining_opt_len)
8277 		return -ENOSPC;
8278 
8279 	/* 253 is another experimental kind */
8280 	if (new_kind == TCPOPT_EXP || new_kind == 253)  {
8281 		if (new_kind_len < 4)
8282 			return -EINVAL;
8283 		/* Match for the 2 byte magic also.
8284 		 * RFC 6994: the magic could be 2 or 4 bytes.
8285 		 * Hence, matching by 2 byte only is on the
8286 		 * conservative side but it is the right
8287 		 * thing to do for the 'search-for-duplication'
8288 		 * purpose.
8289 		 */
8290 		magic = &new_op[2];
8291 		magic_len = 2;
8292 	}
8293 
8294 	/* Check for duplication */
8295 	skb = bpf_sock->skb;
8296 	op = skb->data + sizeof(struct tcphdr);
8297 	opend = bpf_sock->skb_data_end;
8298 
8299 	op = bpf_search_tcp_opt(op, opend, new_kind, magic, magic_len,
8300 				&eol);
8301 	if (!IS_ERR(op))
8302 		return -EEXIST;
8303 
8304 	if (PTR_ERR(op) != -ENOMSG)
8305 		return PTR_ERR(op);
8306 
8307 	if (eol)
8308 		/* The option has been ended.  Treat it as no more
8309 		 * header option can be written.
8310 		 */
8311 		return -ENOSPC;
8312 
8313 	/* No duplication found.  Store the header option. */
8314 	memcpy(opend, from, new_kind_len);
8315 
8316 	bpf_sock->remaining_opt_len -= new_kind_len;
8317 	bpf_sock->skb_data_end += new_kind_len;
8318 
8319 	return 0;
8320 }
8321 
8322 BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
8323 	   const void *, from, u32, len, u64, flags)
8324 {
8325 	if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB)
8326 		return -EPERM;
8327 
8328 	return __bpf_sock_ops_store_hdr_opt(bpf_sock, from, len, flags);
8329 }
8330 
8331 static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = {
8332 	.func		= bpf_sock_ops_store_hdr_opt,
8333 	.gpl_only	= false,
8334 	.ret_type	= RET_INTEGER,
8335 	.arg1_type	= ARG_PTR_TO_CTX,
8336 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8337 	.arg3_type	= ARG_MEM_SIZE,
8338 	.arg4_type	= ARG_ANYTHING,
8339 };
8340 
8341 BPF_CALL_3(bpf_sock_ops_reserve_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
8342 	   u32, len, u64, flags)
8343 {
8344 	if (bpf_sock->op != BPF_SOCK_OPS_HDR_OPT_LEN_CB)
8345 		return -EPERM;
8346 
8347 	if (flags || len < 2)
8348 		return -EINVAL;
8349 
8350 	if (len > bpf_sock->remaining_opt_len)
8351 		return -ENOSPC;
8352 
8353 	bpf_sock->remaining_opt_len -= len;
8354 
8355 	return 0;
8356 }
8357 
8358 static const struct bpf_func_proto bpf_sock_ops_reserve_hdr_opt_proto = {
8359 	.func		= bpf_sock_ops_reserve_hdr_opt,
8360 	.gpl_only	= false,
8361 	.ret_type	= RET_INTEGER,
8362 	.arg1_type	= ARG_PTR_TO_CTX,
8363 	.arg2_type	= ARG_ANYTHING,
8364 	.arg3_type	= ARG_ANYTHING,
8365 };
8366 
8367 BPF_CALL_3(bpf_skb_set_tstamp, struct sk_buff *, skb,
8368 	   u64, tstamp, u32, tstamp_type)
8369 {
8370 	/* skb_clear_delivery_time() is done for inet protocol */
8371 	if (skb->protocol != htons(ETH_P_IP) &&
8372 	    skb->protocol != htons(ETH_P_IPV6))
8373 		return -EOPNOTSUPP;
8374 
8375 	switch (tstamp_type) {
8376 	case BPF_SKB_CLOCK_REALTIME:
8377 		skb->tstamp = tstamp;
8378 		skb->tstamp_type = SKB_CLOCK_REALTIME;
8379 		break;
8380 	case BPF_SKB_CLOCK_MONOTONIC:
8381 		if (!tstamp)
8382 			return -EINVAL;
8383 		skb->tstamp = tstamp;
8384 		skb->tstamp_type = SKB_CLOCK_MONOTONIC;
8385 		break;
8386 	case BPF_SKB_CLOCK_TAI:
8387 		if (!tstamp)
8388 			return -EINVAL;
8389 		skb->tstamp = tstamp;
8390 		skb->tstamp_type = SKB_CLOCK_TAI;
8391 		break;
8392 	default:
8393 		return -EINVAL;
8394 	}
8395 
8396 	return 0;
8397 }
8398 
8399 static const struct bpf_func_proto bpf_skb_set_tstamp_proto = {
8400 	.func           = bpf_skb_set_tstamp,
8401 	.gpl_only       = false,
8402 	.ret_type       = RET_INTEGER,
8403 	.arg1_type      = ARG_PTR_TO_CTX,
8404 	.arg2_type      = ARG_ANYTHING,
8405 	.arg3_type      = ARG_ANYTHING,
8406 };
8407 
8408 #ifdef CONFIG_SYN_COOKIES
8409 BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv4, struct iphdr *, iph,
8410 	   struct tcphdr *, th, u32, th_len)
8411 {
8412 	u32 cookie;
8413 	u16 mss;
8414 
8415 	if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
8416 		return -EINVAL;
8417 
8418 	mss = tcp_parse_mss_option(th, 0) ?: TCP_MSS_DEFAULT;
8419 	cookie = __cookie_v4_init_sequence(iph, th, &mss);
8420 
8421 	return cookie | ((u64)mss << 32);
8422 }
8423 
8424 static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = {
8425 	.func		= bpf_tcp_raw_gen_syncookie_ipv4,
8426 	.gpl_only	= true, /* __cookie_v4_init_sequence() is GPL */
8427 	.pkt_access	= true,
8428 	.ret_type	= RET_INTEGER,
8429 	.arg1_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY,
8430 	.arg1_size	= sizeof(struct iphdr),
8431 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8432 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
8433 };
8434 
8435 BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv6, struct ipv6hdr *, iph,
8436 	   struct tcphdr *, th, u32, th_len)
8437 {
8438 #if IS_ENABLED(CONFIG_IPV6)
8439 	const u16 mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
8440 		sizeof(struct ipv6hdr);
8441 	u32 cookie;
8442 	u16 mss;
8443 
8444 	if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
8445 		return -EINVAL;
8446 
8447 	mss = tcp_parse_mss_option(th, 0) ?: mss_clamp;
8448 	cookie = __cookie_v6_init_sequence(iph, th, &mss);
8449 
8450 	return cookie | ((u64)mss << 32);
8451 #else
8452 	return -EPROTONOSUPPORT;
8453 #endif
8454 }
8455 
8456 static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = {
8457 	.func		= bpf_tcp_raw_gen_syncookie_ipv6,
8458 	.gpl_only	= true, /* __cookie_v6_init_sequence() is GPL */
8459 	.pkt_access	= true,
8460 	.ret_type	= RET_INTEGER,
8461 	.arg1_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY,
8462 	.arg1_size	= sizeof(struct ipv6hdr),
8463 	.arg2_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
8464 	.arg3_type	= ARG_MEM_SIZE_OR_ZERO,
8465 };
8466 
8467 BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph,
8468 	   struct tcphdr *, th)
8469 {
8470 	if (__cookie_v4_check(iph, th) > 0)
8471 		return 0;
8472 
8473 	return -EACCES;
8474 }
8475 
8476 static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = {
8477 	.func		= bpf_tcp_raw_check_syncookie_ipv4,
8478 	.gpl_only	= true, /* __cookie_v4_check is GPL */
8479 	.pkt_access	= true,
8480 	.ret_type	= RET_INTEGER,
8481 	.arg1_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY,
8482 	.arg1_size	= sizeof(struct iphdr),
8483 	.arg2_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY,
8484 	.arg2_size	= sizeof(struct tcphdr),
8485 };
8486 
8487 BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv6, struct ipv6hdr *, iph,
8488 	   struct tcphdr *, th)
8489 {
8490 #if IS_ENABLED(CONFIG_IPV6)
8491 	if (__cookie_v6_check(iph, th) > 0)
8492 		return 0;
8493 
8494 	return -EACCES;
8495 #else
8496 	return -EPROTONOSUPPORT;
8497 #endif
8498 }
8499 
8500 static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = {
8501 	.func		= bpf_tcp_raw_check_syncookie_ipv6,
8502 	.gpl_only	= true, /* __cookie_v6_check is GPL */
8503 	.pkt_access	= true,
8504 	.ret_type	= RET_INTEGER,
8505 	.arg1_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY,
8506 	.arg1_size	= sizeof(struct ipv6hdr),
8507 	.arg2_type	= ARG_PTR_TO_FIXED_SIZE_MEM | MEM_RDONLY,
8508 	.arg2_size	= sizeof(struct tcphdr),
8509 };
8510 #endif /* CONFIG_SYN_COOKIES */
8511 
8512 #endif /* CONFIG_INET */
8513 
8514 bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id)
8515 {
8516 	switch (func_id) {
8517 	case BPF_FUNC_clone_redirect:
8518 	case BPF_FUNC_l3_csum_replace:
8519 	case BPF_FUNC_l4_csum_replace:
8520 	case BPF_FUNC_lwt_push_encap:
8521 	case BPF_FUNC_lwt_seg6_action:
8522 	case BPF_FUNC_lwt_seg6_adjust_srh:
8523 	case BPF_FUNC_lwt_seg6_store_bytes:
8524 	case BPF_FUNC_msg_pop_data:
8525 	case BPF_FUNC_msg_pull_data:
8526 	case BPF_FUNC_msg_push_data:
8527 	case BPF_FUNC_skb_adjust_room:
8528 	case BPF_FUNC_skb_change_head:
8529 	case BPF_FUNC_skb_change_proto:
8530 	case BPF_FUNC_skb_change_tail:
8531 	case BPF_FUNC_skb_pull_data:
8532 	case BPF_FUNC_skb_store_bytes:
8533 	case BPF_FUNC_skb_vlan_pop:
8534 	case BPF_FUNC_skb_vlan_push:
8535 	case BPF_FUNC_store_hdr_opt:
8536 	case BPF_FUNC_xdp_adjust_head:
8537 	case BPF_FUNC_xdp_adjust_meta:
8538 	case BPF_FUNC_xdp_adjust_tail:
8539 	/* tail-called program could call any of the above */
8540 	case BPF_FUNC_tail_call:
8541 		return true;
8542 	default:
8543 		return false;
8544 	}
8545 }
8546 
8547 const struct bpf_func_proto bpf_event_output_data_proto __weak;
8548 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak;
8549 
8550 static const struct bpf_func_proto *
8551 sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8552 {
8553 	const struct bpf_func_proto *func_proto;
8554 
8555 	func_proto = cgroup_common_func_proto(func_id, prog);
8556 	if (func_proto)
8557 		return func_proto;
8558 
8559 	switch (func_id) {
8560 	case BPF_FUNC_get_socket_cookie:
8561 		return &bpf_get_socket_cookie_sock_proto;
8562 	case BPF_FUNC_get_netns_cookie:
8563 		return &bpf_get_netns_cookie_sock_proto;
8564 	case BPF_FUNC_perf_event_output:
8565 		return &bpf_event_output_data_proto;
8566 	case BPF_FUNC_sk_storage_get:
8567 		return &bpf_sk_storage_get_cg_sock_proto;
8568 	case BPF_FUNC_ktime_get_coarse_ns:
8569 		return &bpf_ktime_get_coarse_ns_proto;
8570 	case BPF_FUNC_setsockopt:
8571 		switch (prog->expected_attach_type) {
8572 		case BPF_CGROUP_INET_SOCK_CREATE:
8573 			return &bpf_sock_create_setsockopt_proto;
8574 		default:
8575 			return NULL;
8576 		}
8577 	case BPF_FUNC_getsockopt:
8578 		switch (prog->expected_attach_type) {
8579 		case BPF_CGROUP_INET_SOCK_CREATE:
8580 			return &bpf_sock_create_getsockopt_proto;
8581 		default:
8582 			return NULL;
8583 		}
8584 	default:
8585 		return bpf_base_func_proto(func_id, prog);
8586 	}
8587 }
8588 
8589 static const struct bpf_func_proto *
8590 sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8591 {
8592 	const struct bpf_func_proto *func_proto;
8593 
8594 	func_proto = cgroup_common_func_proto(func_id, prog);
8595 	if (func_proto)
8596 		return func_proto;
8597 
8598 	switch (func_id) {
8599 	case BPF_FUNC_bind:
8600 		switch (prog->expected_attach_type) {
8601 		case BPF_CGROUP_INET4_CONNECT:
8602 		case BPF_CGROUP_INET6_CONNECT:
8603 			return &bpf_bind_proto;
8604 		default:
8605 			return NULL;
8606 		}
8607 	case BPF_FUNC_get_socket_cookie:
8608 		return &bpf_get_socket_cookie_sock_addr_proto;
8609 	case BPF_FUNC_get_netns_cookie:
8610 		return &bpf_get_netns_cookie_sock_addr_proto;
8611 	case BPF_FUNC_perf_event_output:
8612 		return &bpf_event_output_data_proto;
8613 #ifdef CONFIG_INET
8614 	case BPF_FUNC_sk_lookup_tcp:
8615 		return &bpf_sock_addr_sk_lookup_tcp_proto;
8616 	case BPF_FUNC_sk_lookup_udp:
8617 		return &bpf_sock_addr_sk_lookup_udp_proto;
8618 	case BPF_FUNC_sk_release:
8619 		return &bpf_sk_release_proto;
8620 	case BPF_FUNC_skc_lookup_tcp:
8621 		return &bpf_sock_addr_skc_lookup_tcp_proto;
8622 #endif /* CONFIG_INET */
8623 	case BPF_FUNC_sk_storage_get:
8624 		return &bpf_sk_storage_get_proto;
8625 	case BPF_FUNC_sk_storage_delete:
8626 		return &bpf_sk_storage_delete_proto;
8627 	case BPF_FUNC_setsockopt:
8628 		switch (prog->expected_attach_type) {
8629 		case BPF_CGROUP_INET4_BIND:
8630 		case BPF_CGROUP_INET6_BIND:
8631 		case BPF_CGROUP_INET4_CONNECT:
8632 		case BPF_CGROUP_INET6_CONNECT:
8633 		case BPF_CGROUP_UNIX_CONNECT:
8634 		case BPF_CGROUP_UDP4_RECVMSG:
8635 		case BPF_CGROUP_UDP6_RECVMSG:
8636 		case BPF_CGROUP_UNIX_RECVMSG:
8637 		case BPF_CGROUP_UDP4_SENDMSG:
8638 		case BPF_CGROUP_UDP6_SENDMSG:
8639 		case BPF_CGROUP_UNIX_SENDMSG:
8640 		case BPF_CGROUP_INET4_GETPEERNAME:
8641 		case BPF_CGROUP_INET6_GETPEERNAME:
8642 		case BPF_CGROUP_INET4_GETSOCKNAME:
8643 		case BPF_CGROUP_INET6_GETSOCKNAME:
8644 			return &bpf_sock_addr_setsockopt_proto;
8645 		default:
8646 			return NULL;
8647 		}
8648 	case BPF_FUNC_getsockopt:
8649 		switch (prog->expected_attach_type) {
8650 		case BPF_CGROUP_INET4_BIND:
8651 		case BPF_CGROUP_INET6_BIND:
8652 		case BPF_CGROUP_INET4_CONNECT:
8653 		case BPF_CGROUP_INET6_CONNECT:
8654 		case BPF_CGROUP_UNIX_CONNECT:
8655 		case BPF_CGROUP_UDP4_RECVMSG:
8656 		case BPF_CGROUP_UDP6_RECVMSG:
8657 		case BPF_CGROUP_UNIX_RECVMSG:
8658 		case BPF_CGROUP_UDP4_SENDMSG:
8659 		case BPF_CGROUP_UDP6_SENDMSG:
8660 		case BPF_CGROUP_UNIX_SENDMSG:
8661 		case BPF_CGROUP_INET4_GETPEERNAME:
8662 		case BPF_CGROUP_INET6_GETPEERNAME:
8663 		case BPF_CGROUP_INET4_GETSOCKNAME:
8664 		case BPF_CGROUP_INET6_GETSOCKNAME:
8665 			return &bpf_sock_addr_getsockopt_proto;
8666 		default:
8667 			return NULL;
8668 		}
8669 	default:
8670 		return bpf_sk_base_func_proto(func_id, prog);
8671 	}
8672 }
8673 
8674 static const struct bpf_func_proto *
8675 sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8676 {
8677 	switch (func_id) {
8678 	case BPF_FUNC_skb_load_bytes:
8679 		return &bpf_skb_load_bytes_proto;
8680 	case BPF_FUNC_skb_load_bytes_relative:
8681 		return &bpf_skb_load_bytes_relative_proto;
8682 	case BPF_FUNC_get_socket_cookie:
8683 		return &bpf_get_socket_cookie_proto;
8684 	case BPF_FUNC_get_netns_cookie:
8685 		return &bpf_get_netns_cookie_proto;
8686 	case BPF_FUNC_get_socket_uid:
8687 		return &bpf_get_socket_uid_proto;
8688 	case BPF_FUNC_perf_event_output:
8689 		return &bpf_skb_event_output_proto;
8690 	default:
8691 		return bpf_sk_base_func_proto(func_id, prog);
8692 	}
8693 }
8694 
8695 const struct bpf_func_proto bpf_sk_storage_get_proto __weak;
8696 const struct bpf_func_proto bpf_sk_storage_delete_proto __weak;
8697 
8698 static const struct bpf_func_proto *
8699 cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8700 {
8701 	const struct bpf_func_proto *func_proto;
8702 
8703 	func_proto = cgroup_common_func_proto(func_id, prog);
8704 	if (func_proto)
8705 		return func_proto;
8706 
8707 	switch (func_id) {
8708 	case BPF_FUNC_sk_fullsock:
8709 		return &bpf_sk_fullsock_proto;
8710 	case BPF_FUNC_sk_storage_get:
8711 		return &bpf_sk_storage_get_proto;
8712 	case BPF_FUNC_sk_storage_delete:
8713 		return &bpf_sk_storage_delete_proto;
8714 	case BPF_FUNC_perf_event_output:
8715 		return &bpf_skb_event_output_proto;
8716 #ifdef CONFIG_SOCK_CGROUP_DATA
8717 	case BPF_FUNC_skb_cgroup_id:
8718 		return &bpf_skb_cgroup_id_proto;
8719 	case BPF_FUNC_skb_ancestor_cgroup_id:
8720 		return &bpf_skb_ancestor_cgroup_id_proto;
8721 	case BPF_FUNC_sk_cgroup_id:
8722 		return &bpf_sk_cgroup_id_proto;
8723 	case BPF_FUNC_sk_ancestor_cgroup_id:
8724 		return &bpf_sk_ancestor_cgroup_id_proto;
8725 #endif
8726 #ifdef CONFIG_INET
8727 	case BPF_FUNC_sk_lookup_tcp:
8728 		return &bpf_sk_lookup_tcp_proto;
8729 	case BPF_FUNC_sk_lookup_udp:
8730 		return &bpf_sk_lookup_udp_proto;
8731 	case BPF_FUNC_sk_release:
8732 		return &bpf_sk_release_proto;
8733 	case BPF_FUNC_skc_lookup_tcp:
8734 		return &bpf_skc_lookup_tcp_proto;
8735 	case BPF_FUNC_tcp_sock:
8736 		return &bpf_tcp_sock_proto;
8737 	case BPF_FUNC_get_listener_sock:
8738 		return &bpf_get_listener_sock_proto;
8739 	case BPF_FUNC_skb_ecn_set_ce:
8740 		return &bpf_skb_ecn_set_ce_proto;
8741 #endif
8742 	default:
8743 		return sk_filter_func_proto(func_id, prog);
8744 	}
8745 }
8746 
8747 static const struct bpf_func_proto *
8748 tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8749 {
8750 	switch (func_id) {
8751 	case BPF_FUNC_skb_store_bytes:
8752 		return &bpf_skb_store_bytes_proto;
8753 	case BPF_FUNC_skb_load_bytes:
8754 		return &bpf_skb_load_bytes_proto;
8755 	case BPF_FUNC_skb_load_bytes_relative:
8756 		return &bpf_skb_load_bytes_relative_proto;
8757 	case BPF_FUNC_skb_pull_data:
8758 		return &bpf_skb_pull_data_proto;
8759 	case BPF_FUNC_csum_diff:
8760 		return &bpf_csum_diff_proto;
8761 	case BPF_FUNC_csum_update:
8762 		return &bpf_csum_update_proto;
8763 	case BPF_FUNC_csum_level:
8764 		return &bpf_csum_level_proto;
8765 	case BPF_FUNC_l3_csum_replace:
8766 		return &bpf_l3_csum_replace_proto;
8767 	case BPF_FUNC_l4_csum_replace:
8768 		return &bpf_l4_csum_replace_proto;
8769 	case BPF_FUNC_clone_redirect:
8770 		return &bpf_clone_redirect_proto;
8771 	case BPF_FUNC_get_cgroup_classid:
8772 		return &bpf_get_cgroup_classid_proto;
8773 	case BPF_FUNC_skb_vlan_push:
8774 		return &bpf_skb_vlan_push_proto;
8775 	case BPF_FUNC_skb_vlan_pop:
8776 		return &bpf_skb_vlan_pop_proto;
8777 	case BPF_FUNC_skb_change_proto:
8778 		return &bpf_skb_change_proto_proto;
8779 	case BPF_FUNC_skb_change_type:
8780 		return &bpf_skb_change_type_proto;
8781 	case BPF_FUNC_skb_adjust_room:
8782 		return &bpf_skb_adjust_room_proto;
8783 	case BPF_FUNC_skb_change_tail:
8784 		return &bpf_skb_change_tail_proto;
8785 	case BPF_FUNC_skb_change_head:
8786 		return &bpf_skb_change_head_proto;
8787 	case BPF_FUNC_skb_get_tunnel_key:
8788 		return &bpf_skb_get_tunnel_key_proto;
8789 	case BPF_FUNC_skb_set_tunnel_key:
8790 		return bpf_get_skb_set_tunnel_proto(func_id);
8791 	case BPF_FUNC_skb_get_tunnel_opt:
8792 		return &bpf_skb_get_tunnel_opt_proto;
8793 	case BPF_FUNC_skb_set_tunnel_opt:
8794 		return bpf_get_skb_set_tunnel_proto(func_id);
8795 	case BPF_FUNC_redirect:
8796 		return &bpf_redirect_proto;
8797 	case BPF_FUNC_redirect_neigh:
8798 		return &bpf_redirect_neigh_proto;
8799 	case BPF_FUNC_redirect_peer:
8800 		return &bpf_redirect_peer_proto;
8801 	case BPF_FUNC_get_route_realm:
8802 		return &bpf_get_route_realm_proto;
8803 	case BPF_FUNC_get_hash_recalc:
8804 		return &bpf_get_hash_recalc_proto;
8805 	case BPF_FUNC_set_hash_invalid:
8806 		return &bpf_set_hash_invalid_proto;
8807 	case BPF_FUNC_set_hash:
8808 		return &bpf_set_hash_proto;
8809 	case BPF_FUNC_perf_event_output:
8810 		return &bpf_skb_event_output_proto;
8811 	case BPF_FUNC_get_smp_processor_id:
8812 		return &bpf_get_smp_processor_id_proto;
8813 	case BPF_FUNC_skb_under_cgroup:
8814 		return &bpf_skb_under_cgroup_proto;
8815 	case BPF_FUNC_get_socket_cookie:
8816 		return &bpf_get_socket_cookie_proto;
8817 	case BPF_FUNC_get_netns_cookie:
8818 		return &bpf_get_netns_cookie_proto;
8819 	case BPF_FUNC_get_socket_uid:
8820 		return &bpf_get_socket_uid_proto;
8821 	case BPF_FUNC_fib_lookup:
8822 		return &bpf_skb_fib_lookup_proto;
8823 	case BPF_FUNC_check_mtu:
8824 		return &bpf_skb_check_mtu_proto;
8825 	case BPF_FUNC_sk_fullsock:
8826 		return &bpf_sk_fullsock_proto;
8827 	case BPF_FUNC_sk_storage_get:
8828 		return &bpf_sk_storage_get_proto;
8829 	case BPF_FUNC_sk_storage_delete:
8830 		return &bpf_sk_storage_delete_proto;
8831 #ifdef CONFIG_XFRM
8832 	case BPF_FUNC_skb_get_xfrm_state:
8833 		return &bpf_skb_get_xfrm_state_proto;
8834 #endif
8835 #ifdef CONFIG_CGROUP_NET_CLASSID
8836 	case BPF_FUNC_skb_cgroup_classid:
8837 		return &bpf_skb_cgroup_classid_proto;
8838 #endif
8839 #ifdef CONFIG_SOCK_CGROUP_DATA
8840 	case BPF_FUNC_skb_cgroup_id:
8841 		return &bpf_skb_cgroup_id_proto;
8842 	case BPF_FUNC_skb_ancestor_cgroup_id:
8843 		return &bpf_skb_ancestor_cgroup_id_proto;
8844 #endif
8845 #ifdef CONFIG_INET
8846 	case BPF_FUNC_sk_lookup_tcp:
8847 		return &bpf_tc_sk_lookup_tcp_proto;
8848 	case BPF_FUNC_sk_lookup_udp:
8849 		return &bpf_tc_sk_lookup_udp_proto;
8850 	case BPF_FUNC_sk_release:
8851 		return &bpf_sk_release_proto;
8852 	case BPF_FUNC_tcp_sock:
8853 		return &bpf_tcp_sock_proto;
8854 	case BPF_FUNC_get_listener_sock:
8855 		return &bpf_get_listener_sock_proto;
8856 	case BPF_FUNC_skc_lookup_tcp:
8857 		return &bpf_tc_skc_lookup_tcp_proto;
8858 	case BPF_FUNC_tcp_check_syncookie:
8859 		return &bpf_tcp_check_syncookie_proto;
8860 	case BPF_FUNC_skb_ecn_set_ce:
8861 		return &bpf_skb_ecn_set_ce_proto;
8862 	case BPF_FUNC_tcp_gen_syncookie:
8863 		return &bpf_tcp_gen_syncookie_proto;
8864 	case BPF_FUNC_sk_assign:
8865 		return &bpf_sk_assign_proto;
8866 	case BPF_FUNC_skb_set_tstamp:
8867 		return &bpf_skb_set_tstamp_proto;
8868 #ifdef CONFIG_SYN_COOKIES
8869 	case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
8870 		return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
8871 	case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
8872 		return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
8873 	case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
8874 		return &bpf_tcp_raw_check_syncookie_ipv4_proto;
8875 	case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
8876 		return &bpf_tcp_raw_check_syncookie_ipv6_proto;
8877 #endif
8878 #endif
8879 	default:
8880 		return bpf_sk_base_func_proto(func_id, prog);
8881 	}
8882 }
8883 
8884 static const struct bpf_func_proto *
8885 xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8886 {
8887 	switch (func_id) {
8888 	case BPF_FUNC_perf_event_output:
8889 		return &bpf_xdp_event_output_proto;
8890 	case BPF_FUNC_get_smp_processor_id:
8891 		return &bpf_get_smp_processor_id_proto;
8892 	case BPF_FUNC_csum_diff:
8893 		return &bpf_csum_diff_proto;
8894 	case BPF_FUNC_xdp_adjust_head:
8895 		return &bpf_xdp_adjust_head_proto;
8896 	case BPF_FUNC_xdp_adjust_meta:
8897 		return &bpf_xdp_adjust_meta_proto;
8898 	case BPF_FUNC_redirect:
8899 		return &bpf_xdp_redirect_proto;
8900 	case BPF_FUNC_redirect_map:
8901 		return &bpf_xdp_redirect_map_proto;
8902 	case BPF_FUNC_xdp_adjust_tail:
8903 		return &bpf_xdp_adjust_tail_proto;
8904 	case BPF_FUNC_xdp_get_buff_len:
8905 		return &bpf_xdp_get_buff_len_proto;
8906 	case BPF_FUNC_xdp_load_bytes:
8907 		return &bpf_xdp_load_bytes_proto;
8908 	case BPF_FUNC_xdp_store_bytes:
8909 		return &bpf_xdp_store_bytes_proto;
8910 	case BPF_FUNC_fib_lookup:
8911 		return &bpf_xdp_fib_lookup_proto;
8912 	case BPF_FUNC_check_mtu:
8913 		return &bpf_xdp_check_mtu_proto;
8914 #ifdef CONFIG_INET
8915 	case BPF_FUNC_sk_lookup_udp:
8916 		return &bpf_xdp_sk_lookup_udp_proto;
8917 	case BPF_FUNC_sk_lookup_tcp:
8918 		return &bpf_xdp_sk_lookup_tcp_proto;
8919 	case BPF_FUNC_sk_release:
8920 		return &bpf_sk_release_proto;
8921 	case BPF_FUNC_skc_lookup_tcp:
8922 		return &bpf_xdp_skc_lookup_tcp_proto;
8923 	case BPF_FUNC_tcp_check_syncookie:
8924 		return &bpf_tcp_check_syncookie_proto;
8925 	case BPF_FUNC_tcp_gen_syncookie:
8926 		return &bpf_tcp_gen_syncookie_proto;
8927 #ifdef CONFIG_SYN_COOKIES
8928 	case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
8929 		return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
8930 	case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
8931 		return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
8932 	case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
8933 		return &bpf_tcp_raw_check_syncookie_ipv4_proto;
8934 	case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
8935 		return &bpf_tcp_raw_check_syncookie_ipv6_proto;
8936 #endif
8937 #endif
8938 	default:
8939 		return bpf_sk_base_func_proto(func_id, prog);
8940 	}
8941 
8942 #if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)
8943 	/* The nf_conn___init type is used in the NF_CONNTRACK kfuncs. The
8944 	 * kfuncs are defined in two different modules, and we want to be able
8945 	 * to use them interchangeably with the same BTF type ID. Because modules
8946 	 * can't de-duplicate BTF IDs between each other, we need the type to be
8947 	 * referenced in the vmlinux BTF or the verifier will get confused about
8948 	 * the different types. So we add this dummy type reference which will
8949 	 * be included in vmlinux BTF, allowing both modules to refer to the
8950 	 * same type ID.
8951 	 */
8952 	BTF_TYPE_EMIT(struct nf_conn___init);
8953 #endif
8954 }
8955 
8956 const struct bpf_func_proto bpf_sock_map_update_proto __weak;
8957 const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
8958 
8959 static const struct bpf_func_proto *
8960 sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8961 {
8962 	const struct bpf_func_proto *func_proto;
8963 
8964 	func_proto = cgroup_common_func_proto(func_id, prog);
8965 	if (func_proto)
8966 		return func_proto;
8967 
8968 	switch (func_id) {
8969 	case BPF_FUNC_setsockopt:
8970 		return &bpf_sock_ops_setsockopt_proto;
8971 	case BPF_FUNC_getsockopt:
8972 		return &bpf_sock_ops_getsockopt_proto;
8973 	case BPF_FUNC_sock_ops_cb_flags_set:
8974 		return &bpf_sock_ops_cb_flags_set_proto;
8975 	case BPF_FUNC_sock_map_update:
8976 		return &bpf_sock_map_update_proto;
8977 	case BPF_FUNC_sock_hash_update:
8978 		return &bpf_sock_hash_update_proto;
8979 	case BPF_FUNC_get_socket_cookie:
8980 		return &bpf_get_socket_cookie_sock_ops_proto;
8981 	case BPF_FUNC_perf_event_output:
8982 		return &bpf_event_output_data_proto;
8983 	case BPF_FUNC_sk_storage_get:
8984 		return &bpf_sk_storage_get_proto;
8985 	case BPF_FUNC_sk_storage_delete:
8986 		return &bpf_sk_storage_delete_proto;
8987 	case BPF_FUNC_get_netns_cookie:
8988 		return &bpf_get_netns_cookie_sock_ops_proto;
8989 #ifdef CONFIG_INET
8990 	case BPF_FUNC_load_hdr_opt:
8991 		return &bpf_sock_ops_load_hdr_opt_proto;
8992 	case BPF_FUNC_store_hdr_opt:
8993 		return &bpf_sock_ops_store_hdr_opt_proto;
8994 	case BPF_FUNC_reserve_hdr_opt:
8995 		return &bpf_sock_ops_reserve_hdr_opt_proto;
8996 	case BPF_FUNC_tcp_sock:
8997 		return &bpf_tcp_sock_proto;
8998 #endif /* CONFIG_INET */
8999 	default:
9000 		return bpf_sk_base_func_proto(func_id, prog);
9001 	}
9002 }
9003 
9004 const struct bpf_func_proto bpf_msg_redirect_map_proto __weak;
9005 const struct bpf_func_proto bpf_msg_redirect_hash_proto __weak;
9006 
9007 static const struct bpf_func_proto *
9008 sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9009 {
9010 	switch (func_id) {
9011 	case BPF_FUNC_msg_redirect_map:
9012 		return &bpf_msg_redirect_map_proto;
9013 	case BPF_FUNC_msg_redirect_hash:
9014 		return &bpf_msg_redirect_hash_proto;
9015 	case BPF_FUNC_msg_apply_bytes:
9016 		return &bpf_msg_apply_bytes_proto;
9017 	case BPF_FUNC_msg_cork_bytes:
9018 		return &bpf_msg_cork_bytes_proto;
9019 	case BPF_FUNC_msg_pull_data:
9020 		return &bpf_msg_pull_data_proto;
9021 	case BPF_FUNC_msg_push_data:
9022 		return &bpf_msg_push_data_proto;
9023 	case BPF_FUNC_msg_pop_data:
9024 		return &bpf_msg_pop_data_proto;
9025 	case BPF_FUNC_perf_event_output:
9026 		return &bpf_event_output_data_proto;
9027 	case BPF_FUNC_sk_storage_get:
9028 		return &bpf_sk_storage_get_proto;
9029 	case BPF_FUNC_sk_storage_delete:
9030 		return &bpf_sk_storage_delete_proto;
9031 	case BPF_FUNC_get_netns_cookie:
9032 		return &bpf_get_netns_cookie_sk_msg_proto;
9033 	default:
9034 		return bpf_sk_base_func_proto(func_id, prog);
9035 	}
9036 }
9037 
9038 const struct bpf_func_proto bpf_sk_redirect_map_proto __weak;
9039 const struct bpf_func_proto bpf_sk_redirect_hash_proto __weak;
9040 
9041 static const struct bpf_func_proto *
9042 sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9043 {
9044 	switch (func_id) {
9045 	case BPF_FUNC_skb_store_bytes:
9046 		return &bpf_skb_store_bytes_proto;
9047 	case BPF_FUNC_skb_load_bytes:
9048 		return &bpf_skb_load_bytes_proto;
9049 	case BPF_FUNC_skb_pull_data:
9050 		return &sk_skb_pull_data_proto;
9051 	case BPF_FUNC_skb_change_tail:
9052 		return &sk_skb_change_tail_proto;
9053 	case BPF_FUNC_skb_change_head:
9054 		return &sk_skb_change_head_proto;
9055 	case BPF_FUNC_skb_adjust_room:
9056 		return &sk_skb_adjust_room_proto;
9057 	case BPF_FUNC_get_socket_cookie:
9058 		return &bpf_get_socket_cookie_proto;
9059 	case BPF_FUNC_get_socket_uid:
9060 		return &bpf_get_socket_uid_proto;
9061 	case BPF_FUNC_sk_redirect_map:
9062 		return &bpf_sk_redirect_map_proto;
9063 	case BPF_FUNC_sk_redirect_hash:
9064 		return &bpf_sk_redirect_hash_proto;
9065 	case BPF_FUNC_perf_event_output:
9066 		return &bpf_skb_event_output_proto;
9067 #ifdef CONFIG_INET
9068 	case BPF_FUNC_sk_lookup_tcp:
9069 		return &bpf_sk_lookup_tcp_proto;
9070 	case BPF_FUNC_sk_lookup_udp:
9071 		return &bpf_sk_lookup_udp_proto;
9072 	case BPF_FUNC_sk_release:
9073 		return &bpf_sk_release_proto;
9074 	case BPF_FUNC_skc_lookup_tcp:
9075 		return &bpf_skc_lookup_tcp_proto;
9076 #endif
9077 	default:
9078 		return bpf_sk_base_func_proto(func_id, prog);
9079 	}
9080 }
9081 
9082 static const struct bpf_func_proto *
9083 flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9084 {
9085 	switch (func_id) {
9086 	case BPF_FUNC_skb_load_bytes:
9087 		return &bpf_flow_dissector_load_bytes_proto;
9088 	default:
9089 		return bpf_sk_base_func_proto(func_id, prog);
9090 	}
9091 }
9092 
9093 static const struct bpf_func_proto *
9094 lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9095 {
9096 	switch (func_id) {
9097 	case BPF_FUNC_skb_load_bytes:
9098 		return &bpf_skb_load_bytes_proto;
9099 	case BPF_FUNC_skb_pull_data:
9100 		return &bpf_skb_pull_data_proto;
9101 	case BPF_FUNC_csum_diff:
9102 		return &bpf_csum_diff_proto;
9103 	case BPF_FUNC_get_cgroup_classid:
9104 		return &bpf_get_cgroup_classid_proto;
9105 	case BPF_FUNC_get_route_realm:
9106 		return &bpf_get_route_realm_proto;
9107 	case BPF_FUNC_get_hash_recalc:
9108 		return &bpf_get_hash_recalc_proto;
9109 	case BPF_FUNC_perf_event_output:
9110 		return &bpf_skb_event_output_proto;
9111 	case BPF_FUNC_get_smp_processor_id:
9112 		return &bpf_get_smp_processor_id_proto;
9113 	case BPF_FUNC_skb_under_cgroup:
9114 		return &bpf_skb_under_cgroup_proto;
9115 	default:
9116 		return bpf_sk_base_func_proto(func_id, prog);
9117 	}
9118 }
9119 
9120 static const struct bpf_func_proto *
9121 lwt_in_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9122 {
9123 	switch (func_id) {
9124 	case BPF_FUNC_lwt_push_encap:
9125 		return &bpf_lwt_in_push_encap_proto;
9126 	default:
9127 		return lwt_out_func_proto(func_id, prog);
9128 	}
9129 }
9130 
9131 static const struct bpf_func_proto *
9132 lwt_xmit_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9133 {
9134 	switch (func_id) {
9135 	case BPF_FUNC_skb_get_tunnel_key:
9136 		return &bpf_skb_get_tunnel_key_proto;
9137 	case BPF_FUNC_skb_set_tunnel_key:
9138 		return bpf_get_skb_set_tunnel_proto(func_id);
9139 	case BPF_FUNC_skb_get_tunnel_opt:
9140 		return &bpf_skb_get_tunnel_opt_proto;
9141 	case BPF_FUNC_skb_set_tunnel_opt:
9142 		return bpf_get_skb_set_tunnel_proto(func_id);
9143 	case BPF_FUNC_redirect:
9144 		return &bpf_redirect_proto;
9145 	case BPF_FUNC_clone_redirect:
9146 		return &bpf_clone_redirect_proto;
9147 	case BPF_FUNC_skb_change_tail:
9148 		return &bpf_skb_change_tail_proto;
9149 	case BPF_FUNC_skb_change_head:
9150 		return &bpf_skb_change_head_proto;
9151 	case BPF_FUNC_skb_store_bytes:
9152 		return &bpf_skb_store_bytes_proto;
9153 	case BPF_FUNC_csum_update:
9154 		return &bpf_csum_update_proto;
9155 	case BPF_FUNC_csum_level:
9156 		return &bpf_csum_level_proto;
9157 	case BPF_FUNC_l3_csum_replace:
9158 		return &bpf_l3_csum_replace_proto;
9159 	case BPF_FUNC_l4_csum_replace:
9160 		return &bpf_l4_csum_replace_proto;
9161 	case BPF_FUNC_set_hash_invalid:
9162 		return &bpf_set_hash_invalid_proto;
9163 	case BPF_FUNC_lwt_push_encap:
9164 		return &bpf_lwt_xmit_push_encap_proto;
9165 	default:
9166 		return lwt_out_func_proto(func_id, prog);
9167 	}
9168 }
9169 
9170 static const struct bpf_func_proto *
9171 lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
9172 {
9173 	switch (func_id) {
9174 	case BPF_FUNC_skb_pull_data:
9175 		return NULL;
9176 #if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
9177 	case BPF_FUNC_lwt_seg6_store_bytes:
9178 		return &bpf_lwt_seg6_store_bytes_proto;
9179 	case BPF_FUNC_lwt_seg6_action:
9180 		return &bpf_lwt_seg6_action_proto;
9181 	case BPF_FUNC_lwt_seg6_adjust_srh:
9182 		return &bpf_lwt_seg6_adjust_srh_proto;
9183 #endif
9184 	default:
9185 		return lwt_out_func_proto(func_id, prog);
9186 	}
9187 }
9188 
9189 static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type,
9190 				    const struct bpf_prog *prog,
9191 				    struct bpf_insn_access_aux *info)
9192 {
9193 	const int size_default = sizeof(__u32);
9194 
9195 	if (off < 0 || off >= sizeof(struct __sk_buff))
9196 		return false;
9197 
9198 	/* The verifier guarantees that size > 0. */
9199 	if (off % size != 0)
9200 		return false;
9201 
9202 	switch (off) {
9203 	case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
9204 		if (off + size > offsetofend(struct __sk_buff, cb[4]))
9205 			return false;
9206 		break;
9207 	case bpf_ctx_range(struct __sk_buff, data):
9208 	case bpf_ctx_range(struct __sk_buff, data_meta):
9209 	case bpf_ctx_range(struct __sk_buff, data_end):
9210 		if (info->is_ldsx || size != size_default)
9211 			return false;
9212 		break;
9213 	case bpf_ctx_range_till(struct __sk_buff, remote_ip6[0], remote_ip6[3]):
9214 	case bpf_ctx_range_till(struct __sk_buff, local_ip6[0], local_ip6[3]):
9215 	case bpf_ctx_range_till(struct __sk_buff, remote_ip4, remote_ip4):
9216 	case bpf_ctx_range_till(struct __sk_buff, local_ip4, local_ip4):
9217 		if (size != size_default)
9218 			return false;
9219 		break;
9220 	case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
9221 		return false;
9222 	case bpf_ctx_range(struct __sk_buff, hwtstamp):
9223 		if (type == BPF_WRITE || size != sizeof(__u64))
9224 			return false;
9225 		break;
9226 	case bpf_ctx_range(struct __sk_buff, tstamp):
9227 		if (size != sizeof(__u64))
9228 			return false;
9229 		break;
9230 	case bpf_ctx_range_ptr(struct __sk_buff, sk):
9231 		if (type == BPF_WRITE || size != sizeof(__u64))
9232 			return false;
9233 		info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
9234 		break;
9235 	case offsetof(struct __sk_buff, tstamp_type):
9236 		return false;
9237 	case offsetofend(struct __sk_buff, tstamp_type) ... offsetof(struct __sk_buff, hwtstamp) - 1:
9238 		/* Explicitly prohibit access to padding in __sk_buff. */
9239 		return false;
9240 	default:
9241 		/* Only narrow read access allowed for now. */
9242 		if (type == BPF_WRITE) {
9243 			if (size != size_default)
9244 				return false;
9245 		} else {
9246 			bpf_ctx_record_field_size(info, size_default);
9247 			if (!bpf_ctx_narrow_access_ok(off, size, size_default))
9248 				return false;
9249 		}
9250 	}
9251 
9252 	return true;
9253 }
9254 
9255 static bool sk_filter_is_valid_access(int off, int size,
9256 				      enum bpf_access_type type,
9257 				      const struct bpf_prog *prog,
9258 				      struct bpf_insn_access_aux *info)
9259 {
9260 	switch (off) {
9261 	case bpf_ctx_range(struct __sk_buff, tc_classid):
9262 	case bpf_ctx_range(struct __sk_buff, data):
9263 	case bpf_ctx_range(struct __sk_buff, data_meta):
9264 	case bpf_ctx_range(struct __sk_buff, data_end):
9265 	case bpf_ctx_range_till(struct __sk_buff, family, local_port):
9266 	case bpf_ctx_range(struct __sk_buff, tstamp):
9267 	case bpf_ctx_range(struct __sk_buff, wire_len):
9268 	case bpf_ctx_range(struct __sk_buff, hwtstamp):
9269 		return false;
9270 	}
9271 
9272 	if (type == BPF_WRITE) {
9273 		switch (off) {
9274 		case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
9275 			break;
9276 		default:
9277 			return false;
9278 		}
9279 	}
9280 
9281 	return bpf_skb_is_valid_access(off, size, type, prog, info);
9282 }
9283 
9284 static bool cg_skb_is_valid_access(int off, int size,
9285 				   enum bpf_access_type type,
9286 				   const struct bpf_prog *prog,
9287 				   struct bpf_insn_access_aux *info)
9288 {
9289 	switch (off) {
9290 	case bpf_ctx_range(struct __sk_buff, tc_classid):
9291 	case bpf_ctx_range(struct __sk_buff, data_meta):
9292 	case bpf_ctx_range(struct __sk_buff, wire_len):
9293 		return false;
9294 	case bpf_ctx_range(struct __sk_buff, data):
9295 	case bpf_ctx_range(struct __sk_buff, data_end):
9296 		if (!bpf_token_capable(prog->aux->token, CAP_BPF))
9297 			return false;
9298 		break;
9299 	}
9300 
9301 	if (type == BPF_WRITE) {
9302 		switch (off) {
9303 		case bpf_ctx_range(struct __sk_buff, mark):
9304 		case bpf_ctx_range(struct __sk_buff, priority):
9305 		case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
9306 			break;
9307 		case bpf_ctx_range(struct __sk_buff, tstamp):
9308 			if (!bpf_token_capable(prog->aux->token, CAP_BPF))
9309 				return false;
9310 			break;
9311 		default:
9312 			return false;
9313 		}
9314 	}
9315 
9316 	switch (off) {
9317 	case bpf_ctx_range(struct __sk_buff, data):
9318 		info->reg_type = PTR_TO_PACKET;
9319 		break;
9320 	case bpf_ctx_range(struct __sk_buff, data_end):
9321 		info->reg_type = PTR_TO_PACKET_END;
9322 		break;
9323 	}
9324 
9325 	return bpf_skb_is_valid_access(off, size, type, prog, info);
9326 }
9327 
9328 static bool lwt_is_valid_access(int off, int size,
9329 				enum bpf_access_type type,
9330 				const struct bpf_prog *prog,
9331 				struct bpf_insn_access_aux *info)
9332 {
9333 	switch (off) {
9334 	case bpf_ctx_range(struct __sk_buff, tc_classid):
9335 	case bpf_ctx_range_till(struct __sk_buff, family, local_port):
9336 	case bpf_ctx_range(struct __sk_buff, data_meta):
9337 	case bpf_ctx_range(struct __sk_buff, tstamp):
9338 	case bpf_ctx_range(struct __sk_buff, wire_len):
9339 	case bpf_ctx_range(struct __sk_buff, hwtstamp):
9340 		return false;
9341 	}
9342 
9343 	if (type == BPF_WRITE) {
9344 		switch (off) {
9345 		case bpf_ctx_range(struct __sk_buff, mark):
9346 		case bpf_ctx_range(struct __sk_buff, priority):
9347 		case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
9348 			break;
9349 		default:
9350 			return false;
9351 		}
9352 	}
9353 
9354 	switch (off) {
9355 	case bpf_ctx_range(struct __sk_buff, data):
9356 		info->reg_type = PTR_TO_PACKET;
9357 		break;
9358 	case bpf_ctx_range(struct __sk_buff, data_end):
9359 		info->reg_type = PTR_TO_PACKET_END;
9360 		break;
9361 	}
9362 
9363 	return bpf_skb_is_valid_access(off, size, type, prog, info);
9364 }
9365 
9366 /* Attach type specific accesses */
9367 static bool __sock_filter_check_attach_type(int off,
9368 					    enum bpf_access_type access_type,
9369 					    enum bpf_attach_type attach_type)
9370 {
9371 	switch (off) {
9372 	case offsetof(struct bpf_sock, bound_dev_if):
9373 	case offsetof(struct bpf_sock, mark):
9374 	case offsetof(struct bpf_sock, priority):
9375 		switch (attach_type) {
9376 		case BPF_CGROUP_INET_SOCK_CREATE:
9377 		case BPF_CGROUP_INET_SOCK_RELEASE:
9378 			goto full_access;
9379 		default:
9380 			return false;
9381 		}
9382 	case bpf_ctx_range(struct bpf_sock, src_ip4):
9383 		switch (attach_type) {
9384 		case BPF_CGROUP_INET4_POST_BIND:
9385 			goto read_only;
9386 		default:
9387 			return false;
9388 		}
9389 	case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
9390 		switch (attach_type) {
9391 		case BPF_CGROUP_INET6_POST_BIND:
9392 			goto read_only;
9393 		default:
9394 			return false;
9395 		}
9396 	case bpf_ctx_range(struct bpf_sock, src_port):
9397 		switch (attach_type) {
9398 		case BPF_CGROUP_INET4_POST_BIND:
9399 		case BPF_CGROUP_INET6_POST_BIND:
9400 			goto read_only;
9401 		default:
9402 			return false;
9403 		}
9404 	}
9405 read_only:
9406 	return access_type == BPF_READ;
9407 full_access:
9408 	return true;
9409 }
9410 
9411 bool bpf_sock_common_is_valid_access(int off, int size,
9412 				     enum bpf_access_type type,
9413 				     struct bpf_insn_access_aux *info)
9414 {
9415 	switch (off) {
9416 	case bpf_ctx_range_till(struct bpf_sock, type, priority):
9417 		return false;
9418 	default:
9419 		return bpf_sock_is_valid_access(off, size, type, info);
9420 	}
9421 }
9422 
9423 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
9424 			      struct bpf_insn_access_aux *info)
9425 {
9426 	const int size_default = sizeof(__u32);
9427 	int field_size;
9428 
9429 	if (off < 0 || off >= sizeof(struct bpf_sock))
9430 		return false;
9431 	if (off % size != 0)
9432 		return false;
9433 
9434 	switch (off) {
9435 	case offsetof(struct bpf_sock, state):
9436 	case offsetof(struct bpf_sock, family):
9437 	case offsetof(struct bpf_sock, type):
9438 	case offsetof(struct bpf_sock, protocol):
9439 	case offsetof(struct bpf_sock, src_port):
9440 	case offsetof(struct bpf_sock, rx_queue_mapping):
9441 	case bpf_ctx_range(struct bpf_sock, src_ip4):
9442 	case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
9443 	case bpf_ctx_range(struct bpf_sock, dst_ip4):
9444 	case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
9445 		bpf_ctx_record_field_size(info, size_default);
9446 		return bpf_ctx_narrow_access_ok(off, size, size_default);
9447 	case bpf_ctx_range(struct bpf_sock, dst_port):
9448 		field_size = size == size_default ?
9449 			size_default : sizeof_field(struct bpf_sock, dst_port);
9450 		bpf_ctx_record_field_size(info, field_size);
9451 		return bpf_ctx_narrow_access_ok(off, size, field_size);
9452 	case offsetofend(struct bpf_sock, dst_port) ...
9453 	     offsetof(struct bpf_sock, dst_ip4) - 1:
9454 		return false;
9455 	}
9456 
9457 	return size == size_default;
9458 }
9459 
9460 static bool sock_filter_is_valid_access(int off, int size,
9461 					enum bpf_access_type type,
9462 					const struct bpf_prog *prog,
9463 					struct bpf_insn_access_aux *info)
9464 {
9465 	if (!bpf_sock_is_valid_access(off, size, type, info))
9466 		return false;
9467 	return __sock_filter_check_attach_type(off, type,
9468 					       prog->expected_attach_type);
9469 }
9470 
9471 static int bpf_noop_prologue(struct bpf_insn *insn_buf, bool direct_write,
9472 			     const struct bpf_prog *prog)
9473 {
9474 	/* Neither direct read nor direct write requires any preliminary
9475 	 * action.
9476 	 */
9477 	return 0;
9478 }
9479 
9480 static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write,
9481 				const struct bpf_prog *prog, int drop_verdict)
9482 {
9483 	struct bpf_insn *insn = insn_buf;
9484 
9485 	if (!direct_write)
9486 		return 0;
9487 
9488 	/* if (!skb->cloned)
9489 	 *       goto start;
9490 	 *
9491 	 * (Fast-path, otherwise approximation that we might be
9492 	 *  a clone, do the rest in helper.)
9493 	 */
9494 	*insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET);
9495 	*insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK);
9496 	*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7);
9497 
9498 	/* ret = bpf_skb_pull_data(skb, 0); */
9499 	*insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
9500 	*insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2);
9501 	*insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
9502 			       BPF_FUNC_skb_pull_data);
9503 	/* if (!ret)
9504 	 *      goto restore;
9505 	 * return TC_ACT_SHOT;
9506 	 */
9507 	*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2);
9508 	*insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, drop_verdict);
9509 	*insn++ = BPF_EXIT_INSN();
9510 
9511 	/* restore: */
9512 	*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
9513 	/* start: */
9514 	*insn++ = prog->insnsi[0];
9515 
9516 	return insn - insn_buf;
9517 }
9518 
9519 static int bpf_gen_ld_abs(const struct bpf_insn *orig,
9520 			  struct bpf_insn *insn_buf)
9521 {
9522 	bool indirect = BPF_MODE(orig->code) == BPF_IND;
9523 	struct bpf_insn *insn = insn_buf;
9524 
9525 	if (!indirect) {
9526 		*insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm);
9527 	} else {
9528 		*insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg);
9529 		if (orig->imm)
9530 			*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm);
9531 	}
9532 	/* We're guaranteed here that CTX is in R6. */
9533 	*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);
9534 
9535 	switch (BPF_SIZE(orig->code)) {
9536 	case BPF_B:
9537 		*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache);
9538 		break;
9539 	case BPF_H:
9540 		*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache);
9541 		break;
9542 	case BPF_W:
9543 		*insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache);
9544 		break;
9545 	}
9546 
9547 	*insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2);
9548 	*insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
9549 	*insn++ = BPF_EXIT_INSN();
9550 
9551 	return insn - insn_buf;
9552 }
9553 
9554 static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write,
9555 			       const struct bpf_prog *prog)
9556 {
9557 	return bpf_unclone_prologue(insn_buf, direct_write, prog, TC_ACT_SHOT);
9558 }
9559 
9560 static bool tc_cls_act_is_valid_access(int off, int size,
9561 				       enum bpf_access_type type,
9562 				       const struct bpf_prog *prog,
9563 				       struct bpf_insn_access_aux *info)
9564 {
9565 	if (type == BPF_WRITE) {
9566 		switch (off) {
9567 		case bpf_ctx_range(struct __sk_buff, mark):
9568 		case bpf_ctx_range(struct __sk_buff, tc_index):
9569 		case bpf_ctx_range(struct __sk_buff, priority):
9570 		case bpf_ctx_range(struct __sk_buff, tc_classid):
9571 		case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
9572 		case bpf_ctx_range(struct __sk_buff, tstamp):
9573 		case bpf_ctx_range(struct __sk_buff, queue_mapping):
9574 			break;
9575 		default:
9576 			return false;
9577 		}
9578 	}
9579 
9580 	switch (off) {
9581 	case bpf_ctx_range(struct __sk_buff, data):
9582 		info->reg_type = PTR_TO_PACKET;
9583 		break;
9584 	case bpf_ctx_range(struct __sk_buff, data_meta):
9585 		info->reg_type = PTR_TO_PACKET_META;
9586 		break;
9587 	case bpf_ctx_range(struct __sk_buff, data_end):
9588 		info->reg_type = PTR_TO_PACKET_END;
9589 		break;
9590 	case bpf_ctx_range_till(struct __sk_buff, family, local_port):
9591 		return false;
9592 	case offsetof(struct __sk_buff, tstamp_type):
9593 		/* The convert_ctx_access() on reading and writing
9594 		 * __sk_buff->tstamp depends on whether the bpf prog
9595 		 * has used __sk_buff->tstamp_type or not.
9596 		 * Thus, we need to set prog->tstamp_type_access
9597 		 * earlier during is_valid_access() here.
9598 		 */
9599 		((struct bpf_prog *)prog)->tstamp_type_access = 1;
9600 		return size == sizeof(__u8);
9601 	}
9602 
9603 	return bpf_skb_is_valid_access(off, size, type, prog, info);
9604 }
9605 
9606 DEFINE_MUTEX(nf_conn_btf_access_lock);
9607 EXPORT_SYMBOL_GPL(nf_conn_btf_access_lock);
9608 
9609 int (*nfct_btf_struct_access)(struct bpf_verifier_log *log,
9610 			      const struct bpf_reg_state *reg,
9611 			      int off, int size);
9612 EXPORT_SYMBOL_GPL(nfct_btf_struct_access);
9613 
9614 static int tc_cls_act_btf_struct_access(struct bpf_verifier_log *log,
9615 					const struct bpf_reg_state *reg,
9616 					int off, int size)
9617 {
9618 	int ret = -EACCES;
9619 
9620 	mutex_lock(&nf_conn_btf_access_lock);
9621 	if (nfct_btf_struct_access)
9622 		ret = nfct_btf_struct_access(log, reg, off, size);
9623 	mutex_unlock(&nf_conn_btf_access_lock);
9624 
9625 	return ret;
9626 }
9627 
9628 static bool __is_valid_xdp_access(int off, int size)
9629 {
9630 	if (off < 0 || off >= sizeof(struct xdp_md))
9631 		return false;
9632 	if (off % size != 0)
9633 		return false;
9634 	if (size != sizeof(__u32))
9635 		return false;
9636 
9637 	return true;
9638 }
9639 
9640 static bool xdp_is_valid_access(int off, int size,
9641 				enum bpf_access_type type,
9642 				const struct bpf_prog *prog,
9643 				struct bpf_insn_access_aux *info)
9644 {
9645 	if (prog->expected_attach_type != BPF_XDP_DEVMAP) {
9646 		switch (off) {
9647 		case offsetof(struct xdp_md, egress_ifindex):
9648 			return false;
9649 		}
9650 	}
9651 
9652 	if (type == BPF_WRITE) {
9653 		if (bpf_prog_is_offloaded(prog->aux)) {
9654 			switch (off) {
9655 			case offsetof(struct xdp_md, rx_queue_index):
9656 				return __is_valid_xdp_access(off, size);
9657 			}
9658 		}
9659 		return false;
9660 	} else {
9661 		switch (off) {
9662 		case offsetof(struct xdp_md, data_meta):
9663 		case offsetof(struct xdp_md, data):
9664 		case offsetof(struct xdp_md, data_end):
9665 			if (info->is_ldsx)
9666 				return false;
9667 		}
9668 	}
9669 
9670 	switch (off) {
9671 	case offsetof(struct xdp_md, data):
9672 		info->reg_type = PTR_TO_PACKET;
9673 		break;
9674 	case offsetof(struct xdp_md, data_meta):
9675 		info->reg_type = PTR_TO_PACKET_META;
9676 		break;
9677 	case offsetof(struct xdp_md, data_end):
9678 		info->reg_type = PTR_TO_PACKET_END;
9679 		break;
9680 	}
9681 
9682 	return __is_valid_xdp_access(off, size);
9683 }
9684 
9685 void bpf_warn_invalid_xdp_action(const struct net_device *dev,
9686 				 const struct bpf_prog *prog, u32 act)
9687 {
9688 	const u32 act_max = XDP_REDIRECT;
9689 
9690 	pr_warn_once("%s XDP return value %u on prog %s (id %d) dev %s, expect packet loss!\n",
9691 		     act > act_max ? "Illegal" : "Driver unsupported",
9692 		     act, prog->aux->name, prog->aux->id, dev ? dev->name : "N/A");
9693 }
9694 EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
9695 
9696 static int xdp_btf_struct_access(struct bpf_verifier_log *log,
9697 				 const struct bpf_reg_state *reg,
9698 				 int off, int size)
9699 {
9700 	int ret = -EACCES;
9701 
9702 	mutex_lock(&nf_conn_btf_access_lock);
9703 	if (nfct_btf_struct_access)
9704 		ret = nfct_btf_struct_access(log, reg, off, size);
9705 	mutex_unlock(&nf_conn_btf_access_lock);
9706 
9707 	return ret;
9708 }
9709 
9710 static bool sock_addr_is_valid_access(int off, int size,
9711 				      enum bpf_access_type type,
9712 				      const struct bpf_prog *prog,
9713 				      struct bpf_insn_access_aux *info)
9714 {
9715 	const int size_default = sizeof(__u32);
9716 
9717 	if (off < 0 || off >= sizeof(struct bpf_sock_addr))
9718 		return false;
9719 	if (off % size != 0)
9720 		return false;
9721 
9722 	/* Disallow access to fields not belonging to the attach type's address
9723 	 * family.
9724 	 */
9725 	switch (off) {
9726 	case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
9727 		switch (prog->expected_attach_type) {
9728 		case BPF_CGROUP_INET4_BIND:
9729 		case BPF_CGROUP_INET4_CONNECT:
9730 		case BPF_CGROUP_INET4_GETPEERNAME:
9731 		case BPF_CGROUP_INET4_GETSOCKNAME:
9732 		case BPF_CGROUP_UDP4_SENDMSG:
9733 		case BPF_CGROUP_UDP4_RECVMSG:
9734 			break;
9735 		default:
9736 			return false;
9737 		}
9738 		break;
9739 	case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
9740 		switch (prog->expected_attach_type) {
9741 		case BPF_CGROUP_INET6_BIND:
9742 		case BPF_CGROUP_INET6_CONNECT:
9743 		case BPF_CGROUP_INET6_GETPEERNAME:
9744 		case BPF_CGROUP_INET6_GETSOCKNAME:
9745 		case BPF_CGROUP_UDP6_SENDMSG:
9746 		case BPF_CGROUP_UDP6_RECVMSG:
9747 			break;
9748 		default:
9749 			return false;
9750 		}
9751 		break;
9752 	case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
9753 		switch (prog->expected_attach_type) {
9754 		case BPF_CGROUP_UDP4_SENDMSG:
9755 			break;
9756 		default:
9757 			return false;
9758 		}
9759 		break;
9760 	case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
9761 				msg_src_ip6[3]):
9762 		switch (prog->expected_attach_type) {
9763 		case BPF_CGROUP_UDP6_SENDMSG:
9764 			break;
9765 		default:
9766 			return false;
9767 		}
9768 		break;
9769 	}
9770 
9771 	switch (off) {
9772 	case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
9773 	case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
9774 	case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
9775 	case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
9776 				msg_src_ip6[3]):
9777 	case bpf_ctx_range(struct bpf_sock_addr, user_port):
9778 		if (type == BPF_READ) {
9779 			bpf_ctx_record_field_size(info, size_default);
9780 
9781 			if (bpf_ctx_wide_access_ok(off, size,
9782 						   struct bpf_sock_addr,
9783 						   user_ip6))
9784 				return true;
9785 
9786 			if (bpf_ctx_wide_access_ok(off, size,
9787 						   struct bpf_sock_addr,
9788 						   msg_src_ip6))
9789 				return true;
9790 
9791 			if (!bpf_ctx_narrow_access_ok(off, size, size_default))
9792 				return false;
9793 		} else {
9794 			if (bpf_ctx_wide_access_ok(off, size,
9795 						   struct bpf_sock_addr,
9796 						   user_ip6))
9797 				return true;
9798 
9799 			if (bpf_ctx_wide_access_ok(off, size,
9800 						   struct bpf_sock_addr,
9801 						   msg_src_ip6))
9802 				return true;
9803 
9804 			if (size != size_default)
9805 				return false;
9806 		}
9807 		break;
9808 	case bpf_ctx_range_ptr(struct bpf_sock_addr, sk):
9809 		if (type != BPF_READ)
9810 			return false;
9811 		if (size != sizeof(__u64))
9812 			return false;
9813 		info->reg_type = PTR_TO_SOCKET;
9814 		break;
9815 	case bpf_ctx_range(struct bpf_sock_addr, user_family):
9816 	case bpf_ctx_range(struct bpf_sock_addr, family):
9817 	case bpf_ctx_range(struct bpf_sock_addr, type):
9818 	case bpf_ctx_range(struct bpf_sock_addr, protocol):
9819 		if (type != BPF_READ)
9820 			return false;
9821 		if (size != size_default)
9822 			return false;
9823 		break;
9824 	default:
9825 		return false;
9826 	}
9827 
9828 	return true;
9829 }
9830 
9831 static bool sock_ops_is_valid_access(int off, int size,
9832 				     enum bpf_access_type type,
9833 				     const struct bpf_prog *prog,
9834 				     struct bpf_insn_access_aux *info)
9835 {
9836 	const int size_default = sizeof(__u32);
9837 
9838 	if (off < 0 || off >= sizeof(struct bpf_sock_ops))
9839 		return false;
9840 
9841 	/* The verifier guarantees that size > 0. */
9842 	if (off % size != 0)
9843 		return false;
9844 
9845 	if (type == BPF_WRITE) {
9846 		switch (off) {
9847 		case offsetof(struct bpf_sock_ops, reply):
9848 		case offsetof(struct bpf_sock_ops, sk_txhash):
9849 			if (size != size_default)
9850 				return false;
9851 			break;
9852 		default:
9853 			return false;
9854 		}
9855 	} else {
9856 		switch (off) {
9857 		case bpf_ctx_range_till(struct bpf_sock_ops, bytes_received,
9858 					bytes_acked):
9859 			if (size != sizeof(__u64))
9860 				return false;
9861 			break;
9862 		case bpf_ctx_range_ptr(struct bpf_sock_ops, sk):
9863 			if (size != sizeof(__u64))
9864 				return false;
9865 			info->reg_type = PTR_TO_SOCKET_OR_NULL;
9866 			break;
9867 		case bpf_ctx_range_ptr(struct bpf_sock_ops, skb_data):
9868 			if (size != sizeof(__u64))
9869 				return false;
9870 			info->reg_type = PTR_TO_PACKET;
9871 			break;
9872 		case bpf_ctx_range_ptr(struct bpf_sock_ops, skb_data_end):
9873 			if (size != sizeof(__u64))
9874 				return false;
9875 			info->reg_type = PTR_TO_PACKET_END;
9876 			break;
9877 		case offsetof(struct bpf_sock_ops, skb_tcp_flags):
9878 			bpf_ctx_record_field_size(info, size_default);
9879 			return bpf_ctx_narrow_access_ok(off, size,
9880 							size_default);
9881 		case bpf_ctx_range(struct bpf_sock_ops, skb_hwtstamp):
9882 			if (size != sizeof(__u64))
9883 				return false;
9884 			break;
9885 		default:
9886 			if (size != size_default)
9887 				return false;
9888 			break;
9889 		}
9890 	}
9891 
9892 	return true;
9893 }
9894 
9895 static int sk_skb_prologue(struct bpf_insn *insn_buf, bool direct_write,
9896 			   const struct bpf_prog *prog)
9897 {
9898 	return bpf_unclone_prologue(insn_buf, direct_write, prog, SK_DROP);
9899 }
9900 
9901 static bool sk_skb_is_valid_access(int off, int size,
9902 				   enum bpf_access_type type,
9903 				   const struct bpf_prog *prog,
9904 				   struct bpf_insn_access_aux *info)
9905 {
9906 	switch (off) {
9907 	case bpf_ctx_range(struct __sk_buff, tc_classid):
9908 	case bpf_ctx_range(struct __sk_buff, data_meta):
9909 	case bpf_ctx_range(struct __sk_buff, tstamp):
9910 	case bpf_ctx_range(struct __sk_buff, wire_len):
9911 	case bpf_ctx_range(struct __sk_buff, hwtstamp):
9912 		return false;
9913 	}
9914 
9915 	if (type == BPF_WRITE) {
9916 		switch (off) {
9917 		case bpf_ctx_range(struct __sk_buff, tc_index):
9918 		case bpf_ctx_range(struct __sk_buff, priority):
9919 			break;
9920 		default:
9921 			return false;
9922 		}
9923 	}
9924 
9925 	switch (off) {
9926 	case bpf_ctx_range(struct __sk_buff, mark):
9927 		return false;
9928 	case bpf_ctx_range(struct __sk_buff, data):
9929 		info->reg_type = PTR_TO_PACKET;
9930 		break;
9931 	case bpf_ctx_range(struct __sk_buff, data_end):
9932 		info->reg_type = PTR_TO_PACKET_END;
9933 		break;
9934 	}
9935 
9936 	return bpf_skb_is_valid_access(off, size, type, prog, info);
9937 }
9938 
9939 static bool sk_msg_is_valid_access(int off, int size,
9940 				   enum bpf_access_type type,
9941 				   const struct bpf_prog *prog,
9942 				   struct bpf_insn_access_aux *info)
9943 {
9944 	if (type == BPF_WRITE)
9945 		return false;
9946 
9947 	if (off % size != 0)
9948 		return false;
9949 
9950 	switch (off) {
9951 	case bpf_ctx_range_ptr(struct sk_msg_md, data):
9952 		info->reg_type = PTR_TO_PACKET;
9953 		if (size != sizeof(__u64))
9954 			return false;
9955 		break;
9956 	case bpf_ctx_range_ptr(struct sk_msg_md, data_end):
9957 		info->reg_type = PTR_TO_PACKET_END;
9958 		if (size != sizeof(__u64))
9959 			return false;
9960 		break;
9961 	case bpf_ctx_range_ptr(struct sk_msg_md, sk):
9962 		if (size != sizeof(__u64))
9963 			return false;
9964 		info->reg_type = PTR_TO_SOCKET;
9965 		break;
9966 	case bpf_ctx_range(struct sk_msg_md, family):
9967 	case bpf_ctx_range(struct sk_msg_md, remote_ip4):
9968 	case bpf_ctx_range(struct sk_msg_md, local_ip4):
9969 	case bpf_ctx_range_till(struct sk_msg_md, remote_ip6[0], remote_ip6[3]):
9970 	case bpf_ctx_range_till(struct sk_msg_md, local_ip6[0], local_ip6[3]):
9971 	case bpf_ctx_range(struct sk_msg_md, remote_port):
9972 	case bpf_ctx_range(struct sk_msg_md, local_port):
9973 	case bpf_ctx_range(struct sk_msg_md, size):
9974 		if (size != sizeof(__u32))
9975 			return false;
9976 		break;
9977 	default:
9978 		return false;
9979 	}
9980 	return true;
9981 }
9982 
9983 static bool flow_dissector_is_valid_access(int off, int size,
9984 					   enum bpf_access_type type,
9985 					   const struct bpf_prog *prog,
9986 					   struct bpf_insn_access_aux *info)
9987 {
9988 	const int size_default = sizeof(__u32);
9989 
9990 	if (off < 0 || off >= sizeof(struct __sk_buff))
9991 		return false;
9992 
9993 	if (off % size != 0)
9994 		return false;
9995 
9996 	if (type == BPF_WRITE)
9997 		return false;
9998 
9999 	switch (off) {
10000 	case bpf_ctx_range(struct __sk_buff, data):
10001 		if (info->is_ldsx || size != size_default)
10002 			return false;
10003 		info->reg_type = PTR_TO_PACKET;
10004 		return true;
10005 	case bpf_ctx_range(struct __sk_buff, data_end):
10006 		if (info->is_ldsx || size != size_default)
10007 			return false;
10008 		info->reg_type = PTR_TO_PACKET_END;
10009 		return true;
10010 	case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
10011 		if (size != sizeof(__u64))
10012 			return false;
10013 		info->reg_type = PTR_TO_FLOW_KEYS;
10014 		return true;
10015 	default:
10016 		return false;
10017 	}
10018 }
10019 
10020 static u32 flow_dissector_convert_ctx_access(enum bpf_access_type type,
10021 					     const struct bpf_insn *si,
10022 					     struct bpf_insn *insn_buf,
10023 					     struct bpf_prog *prog,
10024 					     u32 *target_size)
10025 
10026 {
10027 	struct bpf_insn *insn = insn_buf;
10028 
10029 	switch (si->off) {
10030 	case offsetof(struct __sk_buff, data):
10031 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data),
10032 				      si->dst_reg, si->src_reg,
10033 				      offsetof(struct bpf_flow_dissector, data));
10034 		break;
10035 
10036 	case offsetof(struct __sk_buff, data_end):
10037 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data_end),
10038 				      si->dst_reg, si->src_reg,
10039 				      offsetof(struct bpf_flow_dissector, data_end));
10040 		break;
10041 
10042 	case offsetof(struct __sk_buff, flow_keys):
10043 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, flow_keys),
10044 				      si->dst_reg, si->src_reg,
10045 				      offsetof(struct bpf_flow_dissector, flow_keys));
10046 		break;
10047 	}
10048 
10049 	return insn - insn_buf;
10050 }
10051 
10052 static struct bpf_insn *bpf_convert_tstamp_type_read(const struct bpf_insn *si,
10053 						     struct bpf_insn *insn)
10054 {
10055 	__u8 value_reg = si->dst_reg;
10056 	__u8 skb_reg = si->src_reg;
10057 	BUILD_BUG_ON(__SKB_CLOCK_MAX != (int)BPF_SKB_CLOCK_TAI);
10058 	BUILD_BUG_ON(SKB_CLOCK_REALTIME != (int)BPF_SKB_CLOCK_REALTIME);
10059 	BUILD_BUG_ON(SKB_CLOCK_MONOTONIC != (int)BPF_SKB_CLOCK_MONOTONIC);
10060 	BUILD_BUG_ON(SKB_CLOCK_TAI != (int)BPF_SKB_CLOCK_TAI);
10061 	*insn++ = BPF_LDX_MEM(BPF_B, value_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
10062 	*insn++ = BPF_ALU32_IMM(BPF_AND, value_reg, SKB_TSTAMP_TYPE_MASK);
10063 #ifdef __BIG_ENDIAN_BITFIELD
10064 	*insn++ = BPF_ALU32_IMM(BPF_RSH, value_reg, SKB_TSTAMP_TYPE_RSHIFT);
10065 #else
10066 	BUILD_BUG_ON(!(SKB_TSTAMP_TYPE_MASK & 0x1));
10067 #endif
10068 
10069 	return insn;
10070 }
10071 
10072 static struct bpf_insn *bpf_convert_shinfo_access(__u8 dst_reg, __u8 skb_reg,
10073 						  struct bpf_insn *insn)
10074 {
10075 	/* si->dst_reg = skb_shinfo(SKB); */
10076 #ifdef NET_SKBUFF_DATA_USES_OFFSET
10077 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
10078 			      BPF_REG_AX, skb_reg,
10079 			      offsetof(struct sk_buff, end));
10080 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, head),
10081 			      dst_reg, skb_reg,
10082 			      offsetof(struct sk_buff, head));
10083 	*insn++ = BPF_ALU64_REG(BPF_ADD, dst_reg, BPF_REG_AX);
10084 #else
10085 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
10086 			      dst_reg, skb_reg,
10087 			      offsetof(struct sk_buff, end));
10088 #endif
10089 
10090 	return insn;
10091 }
10092 
10093 static struct bpf_insn *bpf_convert_tstamp_read(const struct bpf_prog *prog,
10094 						const struct bpf_insn *si,
10095 						struct bpf_insn *insn)
10096 {
10097 	__u8 value_reg = si->dst_reg;
10098 	__u8 skb_reg = si->src_reg;
10099 
10100 #ifdef CONFIG_NET_XGRESS
10101 	/* If the tstamp_type is read,
10102 	 * the bpf prog is aware the tstamp could have delivery time.
10103 	 * Thus, read skb->tstamp as is if tstamp_type_access is true.
10104 	 */
10105 	if (!prog->tstamp_type_access) {
10106 		/* AX is needed because src_reg and dst_reg could be the same */
10107 		__u8 tmp_reg = BPF_REG_AX;
10108 
10109 		*insn++ = BPF_LDX_MEM(BPF_B, tmp_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
10110 		/* check if ingress mask bits is set */
10111 		*insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, TC_AT_INGRESS_MASK, 1);
10112 		*insn++ = BPF_JMP_A(4);
10113 		*insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, SKB_TSTAMP_TYPE_MASK, 1);
10114 		*insn++ = BPF_JMP_A(2);
10115 		/* skb->tc_at_ingress && skb->tstamp_type,
10116 		 * read 0 as the (rcv) timestamp.
10117 		 */
10118 		*insn++ = BPF_MOV64_IMM(value_reg, 0);
10119 		*insn++ = BPF_JMP_A(1);
10120 	}
10121 #endif
10122 
10123 	*insn++ = BPF_LDX_MEM(BPF_DW, value_reg, skb_reg,
10124 			      offsetof(struct sk_buff, tstamp));
10125 	return insn;
10126 }
10127 
10128 static struct bpf_insn *bpf_convert_tstamp_write(const struct bpf_prog *prog,
10129 						 const struct bpf_insn *si,
10130 						 struct bpf_insn *insn)
10131 {
10132 	__u8 value_reg = si->src_reg;
10133 	__u8 skb_reg = si->dst_reg;
10134 
10135 #ifdef CONFIG_NET_XGRESS
10136 	/* If the tstamp_type is read,
10137 	 * the bpf prog is aware the tstamp could have delivery time.
10138 	 * Thus, write skb->tstamp as is if tstamp_type_access is true.
10139 	 * Otherwise, writing at ingress will have to clear the
10140 	 * skb->tstamp_type bit also.
10141 	 */
10142 	if (!prog->tstamp_type_access) {
10143 		__u8 tmp_reg = BPF_REG_AX;
10144 
10145 		*insn++ = BPF_LDX_MEM(BPF_B, tmp_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
10146 		/* Writing __sk_buff->tstamp as ingress, goto <clear> */
10147 		*insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, TC_AT_INGRESS_MASK, 1);
10148 		/* goto <store> */
10149 		*insn++ = BPF_JMP_A(2);
10150 		/* <clear>: skb->tstamp_type */
10151 		*insn++ = BPF_ALU32_IMM(BPF_AND, tmp_reg, ~SKB_TSTAMP_TYPE_MASK);
10152 		*insn++ = BPF_STX_MEM(BPF_B, skb_reg, tmp_reg, SKB_BF_MONO_TC_OFFSET);
10153 	}
10154 #endif
10155 
10156 	/* <store>: skb->tstamp = tstamp */
10157 	*insn++ = BPF_RAW_INSN(BPF_CLASS(si->code) | BPF_DW | BPF_MEM,
10158 			       skb_reg, value_reg, offsetof(struct sk_buff, tstamp), si->imm);
10159 	return insn;
10160 }
10161 
10162 #define BPF_EMIT_STORE(size, si, off)					\
10163 	BPF_RAW_INSN(BPF_CLASS((si)->code) | (size) | BPF_MEM,		\
10164 		     (si)->dst_reg, (si)->src_reg, (off), (si)->imm)
10165 
10166 static u32 bpf_convert_ctx_access(enum bpf_access_type type,
10167 				  const struct bpf_insn *si,
10168 				  struct bpf_insn *insn_buf,
10169 				  struct bpf_prog *prog, u32 *target_size)
10170 {
10171 	struct bpf_insn *insn = insn_buf;
10172 	int off;
10173 
10174 	switch (si->off) {
10175 	case offsetof(struct __sk_buff, len):
10176 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10177 				      bpf_target_off(struct sk_buff, len, 4,
10178 						     target_size));
10179 		break;
10180 
10181 	case offsetof(struct __sk_buff, protocol):
10182 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10183 				      bpf_target_off(struct sk_buff, protocol, 2,
10184 						     target_size));
10185 		break;
10186 
10187 	case offsetof(struct __sk_buff, vlan_proto):
10188 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10189 				      bpf_target_off(struct sk_buff, vlan_proto, 2,
10190 						     target_size));
10191 		break;
10192 
10193 	case offsetof(struct __sk_buff, priority):
10194 		if (type == BPF_WRITE)
10195 			*insn++ = BPF_EMIT_STORE(BPF_W, si,
10196 						 bpf_target_off(struct sk_buff, priority, 4,
10197 								target_size));
10198 		else
10199 			*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10200 					      bpf_target_off(struct sk_buff, priority, 4,
10201 							     target_size));
10202 		break;
10203 
10204 	case offsetof(struct __sk_buff, ingress_ifindex):
10205 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10206 				      bpf_target_off(struct sk_buff, skb_iif, 4,
10207 						     target_size));
10208 		break;
10209 
10210 	case offsetof(struct __sk_buff, ifindex):
10211 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
10212 				      si->dst_reg, si->src_reg,
10213 				      offsetof(struct sk_buff, dev));
10214 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10215 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10216 				      bpf_target_off(struct net_device, ifindex, 4,
10217 						     target_size));
10218 		break;
10219 
10220 	case offsetof(struct __sk_buff, hash):
10221 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10222 				      bpf_target_off(struct sk_buff, hash, 4,
10223 						     target_size));
10224 		break;
10225 
10226 	case offsetof(struct __sk_buff, mark):
10227 		if (type == BPF_WRITE)
10228 			*insn++ = BPF_EMIT_STORE(BPF_W, si,
10229 						 bpf_target_off(struct sk_buff, mark, 4,
10230 								target_size));
10231 		else
10232 			*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10233 					      bpf_target_off(struct sk_buff, mark, 4,
10234 							     target_size));
10235 		break;
10236 
10237 	case offsetof(struct __sk_buff, pkt_type):
10238 		*target_size = 1;
10239 		*insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg,
10240 				      PKT_TYPE_OFFSET);
10241 		*insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX);
10242 #ifdef __BIG_ENDIAN_BITFIELD
10243 		*insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5);
10244 #endif
10245 		break;
10246 
10247 	case offsetof(struct __sk_buff, queue_mapping):
10248 		if (type == BPF_WRITE) {
10249 			u32 offset = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
10250 
10251 			if (BPF_CLASS(si->code) == BPF_ST && si->imm >= NO_QUEUE_MAPPING) {
10252 				*insn++ = BPF_JMP_A(0); /* noop */
10253 				break;
10254 			}
10255 
10256 			if (BPF_CLASS(si->code) == BPF_STX)
10257 				*insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
10258 			*insn++ = BPF_EMIT_STORE(BPF_H, si, offset);
10259 		} else {
10260 			*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10261 					      bpf_target_off(struct sk_buff,
10262 							     queue_mapping,
10263 							     2, target_size));
10264 		}
10265 		break;
10266 
10267 	case offsetof(struct __sk_buff, vlan_present):
10268 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10269 				      bpf_target_off(struct sk_buff,
10270 						     vlan_all, 4, target_size));
10271 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
10272 		*insn++ = BPF_ALU32_IMM(BPF_MOV, si->dst_reg, 1);
10273 		break;
10274 
10275 	case offsetof(struct __sk_buff, vlan_tci):
10276 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10277 				      bpf_target_off(struct sk_buff, vlan_tci, 2,
10278 						     target_size));
10279 		break;
10280 
10281 	case offsetof(struct __sk_buff, cb[0]) ...
10282 	     offsetofend(struct __sk_buff, cb[4]) - 1:
10283 		BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, data) < 20);
10284 		BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
10285 			      offsetof(struct qdisc_skb_cb, data)) %
10286 			     sizeof(__u64));
10287 
10288 		prog->cb_access = 1;
10289 		off  = si->off;
10290 		off -= offsetof(struct __sk_buff, cb[0]);
10291 		off += offsetof(struct sk_buff, cb);
10292 		off += offsetof(struct qdisc_skb_cb, data);
10293 		if (type == BPF_WRITE)
10294 			*insn++ = BPF_EMIT_STORE(BPF_SIZE(si->code), si, off);
10295 		else
10296 			*insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
10297 					      si->src_reg, off);
10298 		break;
10299 
10300 	case offsetof(struct __sk_buff, tc_classid):
10301 		BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, tc_classid) != 2);
10302 
10303 		off  = si->off;
10304 		off -= offsetof(struct __sk_buff, tc_classid);
10305 		off += offsetof(struct sk_buff, cb);
10306 		off += offsetof(struct qdisc_skb_cb, tc_classid);
10307 		*target_size = 2;
10308 		if (type == BPF_WRITE)
10309 			*insn++ = BPF_EMIT_STORE(BPF_H, si, off);
10310 		else
10311 			*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg,
10312 					      si->src_reg, off);
10313 		break;
10314 
10315 	case offsetof(struct __sk_buff, data):
10316 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
10317 				      si->dst_reg, si->src_reg,
10318 				      offsetof(struct sk_buff, data));
10319 		break;
10320 
10321 	case offsetof(struct __sk_buff, data_meta):
10322 		off  = si->off;
10323 		off -= offsetof(struct __sk_buff, data_meta);
10324 		off += offsetof(struct sk_buff, cb);
10325 		off += offsetof(struct bpf_skb_data_end, data_meta);
10326 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
10327 				      si->src_reg, off);
10328 		break;
10329 
10330 	case offsetof(struct __sk_buff, data_end):
10331 		off  = si->off;
10332 		off -= offsetof(struct __sk_buff, data_end);
10333 		off += offsetof(struct sk_buff, cb);
10334 		off += offsetof(struct bpf_skb_data_end, data_end);
10335 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
10336 				      si->src_reg, off);
10337 		break;
10338 
10339 	case offsetof(struct __sk_buff, tc_index):
10340 #ifdef CONFIG_NET_SCHED
10341 		if (type == BPF_WRITE)
10342 			*insn++ = BPF_EMIT_STORE(BPF_H, si,
10343 						 bpf_target_off(struct sk_buff, tc_index, 2,
10344 								target_size));
10345 		else
10346 			*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
10347 					      bpf_target_off(struct sk_buff, tc_index, 2,
10348 							     target_size));
10349 #else
10350 		*target_size = 2;
10351 		if (type == BPF_WRITE)
10352 			*insn++ = BPF_MOV64_REG(si->dst_reg, si->dst_reg);
10353 		else
10354 			*insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
10355 #endif
10356 		break;
10357 
10358 	case offsetof(struct __sk_buff, napi_id):
10359 #if defined(CONFIG_NET_RX_BUSY_POLL)
10360 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10361 				      bpf_target_off(struct sk_buff, napi_id, 4,
10362 						     target_size));
10363 		*insn++ = BPF_JMP_IMM(BPF_JGE, si->dst_reg, MIN_NAPI_ID, 1);
10364 		*insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
10365 #else
10366 		*target_size = 4;
10367 		*insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
10368 #endif
10369 		break;
10370 	case offsetof(struct __sk_buff, family):
10371 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
10372 
10373 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10374 				      si->dst_reg, si->src_reg,
10375 				      offsetof(struct sk_buff, sk));
10376 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10377 				      bpf_target_off(struct sock_common,
10378 						     skc_family,
10379 						     2, target_size));
10380 		break;
10381 	case offsetof(struct __sk_buff, remote_ip4):
10382 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
10383 
10384 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10385 				      si->dst_reg, si->src_reg,
10386 				      offsetof(struct sk_buff, sk));
10387 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10388 				      bpf_target_off(struct sock_common,
10389 						     skc_daddr,
10390 						     4, target_size));
10391 		break;
10392 	case offsetof(struct __sk_buff, local_ip4):
10393 		BUILD_BUG_ON(sizeof_field(struct sock_common,
10394 					  skc_rcv_saddr) != 4);
10395 
10396 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10397 				      si->dst_reg, si->src_reg,
10398 				      offsetof(struct sk_buff, sk));
10399 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10400 				      bpf_target_off(struct sock_common,
10401 						     skc_rcv_saddr,
10402 						     4, target_size));
10403 		break;
10404 	case offsetof(struct __sk_buff, remote_ip6[0]) ...
10405 	     offsetof(struct __sk_buff, remote_ip6[3]):
10406 #if IS_ENABLED(CONFIG_IPV6)
10407 		BUILD_BUG_ON(sizeof_field(struct sock_common,
10408 					  skc_v6_daddr.s6_addr32[0]) != 4);
10409 
10410 		off = si->off;
10411 		off -= offsetof(struct __sk_buff, remote_ip6[0]);
10412 
10413 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10414 				      si->dst_reg, si->src_reg,
10415 				      offsetof(struct sk_buff, sk));
10416 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10417 				      offsetof(struct sock_common,
10418 					       skc_v6_daddr.s6_addr32[0]) +
10419 				      off);
10420 #else
10421 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10422 #endif
10423 		break;
10424 	case offsetof(struct __sk_buff, local_ip6[0]) ...
10425 	     offsetof(struct __sk_buff, local_ip6[3]):
10426 #if IS_ENABLED(CONFIG_IPV6)
10427 		BUILD_BUG_ON(sizeof_field(struct sock_common,
10428 					  skc_v6_rcv_saddr.s6_addr32[0]) != 4);
10429 
10430 		off = si->off;
10431 		off -= offsetof(struct __sk_buff, local_ip6[0]);
10432 
10433 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10434 				      si->dst_reg, si->src_reg,
10435 				      offsetof(struct sk_buff, sk));
10436 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10437 				      offsetof(struct sock_common,
10438 					       skc_v6_rcv_saddr.s6_addr32[0]) +
10439 				      off);
10440 #else
10441 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10442 #endif
10443 		break;
10444 
10445 	case offsetof(struct __sk_buff, remote_port):
10446 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
10447 
10448 		/*
10449 		 * The converted value is 4 bytes wide, not 2: on little
10450 		 * endian the 16-bit port is shifted into the high half.
10451 		 * Don't use bpf_target_off() here, it would report the
10452 		 * member size and the narrow-load masking in the fixup
10453 		 * pass would be skipped.
10454 		 */
10455 		*target_size = 4;
10456 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10457 				      si->dst_reg, si->src_reg,
10458 				      offsetof(struct sk_buff, sk));
10459 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10460 				      offsetof(struct sock_common, skc_dport));
10461 #ifndef __BIG_ENDIAN_BITFIELD
10462 		*insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
10463 #endif
10464 		break;
10465 
10466 	case offsetof(struct __sk_buff, local_port):
10467 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
10468 
10469 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10470 				      si->dst_reg, si->src_reg,
10471 				      offsetof(struct sk_buff, sk));
10472 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
10473 				      bpf_target_off(struct sock_common,
10474 						     skc_num, 2, target_size));
10475 		break;
10476 
10477 	case offsetof(struct __sk_buff, tstamp):
10478 		BUILD_BUG_ON(sizeof_field(struct sk_buff, tstamp) != 8);
10479 
10480 		if (type == BPF_WRITE)
10481 			insn = bpf_convert_tstamp_write(prog, si, insn);
10482 		else
10483 			insn = bpf_convert_tstamp_read(prog, si, insn);
10484 		break;
10485 
10486 	case offsetof(struct __sk_buff, tstamp_type):
10487 		insn = bpf_convert_tstamp_type_read(si, insn);
10488 		break;
10489 
10490 	case offsetof(struct __sk_buff, gso_segs):
10491 		insn = bpf_convert_shinfo_access(si->dst_reg, si->src_reg, insn);
10492 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_segs),
10493 				      si->dst_reg, si->dst_reg,
10494 				      bpf_target_off(struct skb_shared_info,
10495 						     gso_segs, 2,
10496 						     target_size));
10497 		break;
10498 	case offsetof(struct __sk_buff, gso_size):
10499 		insn = bpf_convert_shinfo_access(si->dst_reg, si->src_reg, insn);
10500 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_size),
10501 				      si->dst_reg, si->dst_reg,
10502 				      bpf_target_off(struct skb_shared_info,
10503 						     gso_size, 2,
10504 						     target_size));
10505 		break;
10506 	case offsetof(struct __sk_buff, wire_len):
10507 		BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, pkt_len) != 4);
10508 
10509 		off = si->off;
10510 		off -= offsetof(struct __sk_buff, wire_len);
10511 		off += offsetof(struct sk_buff, cb);
10512 		off += offsetof(struct qdisc_skb_cb, pkt_len);
10513 		*target_size = 4;
10514 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off);
10515 		break;
10516 
10517 	case offsetof(struct __sk_buff, sk):
10518 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
10519 				      si->dst_reg, si->src_reg,
10520 				      offsetof(struct sk_buff, sk));
10521 		break;
10522 	case offsetof(struct __sk_buff, hwtstamp):
10523 		BUILD_BUG_ON(sizeof_field(struct skb_shared_hwtstamps, hwtstamp) != 8);
10524 		BUILD_BUG_ON(offsetof(struct skb_shared_hwtstamps, hwtstamp) != 0);
10525 
10526 		insn = bpf_convert_shinfo_access(si->dst_reg, si->src_reg, insn);
10527 		*insn++ = BPF_LDX_MEM(BPF_DW,
10528 				      si->dst_reg, si->dst_reg,
10529 				      bpf_target_off(struct skb_shared_info,
10530 						     hwtstamps, 8,
10531 						     target_size));
10532 		break;
10533 	}
10534 
10535 	return insn - insn_buf;
10536 }
10537 
10538 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
10539 				const struct bpf_insn *si,
10540 				struct bpf_insn *insn_buf,
10541 				struct bpf_prog *prog, u32 *target_size)
10542 {
10543 	struct bpf_insn *insn = insn_buf;
10544 	int off;
10545 
10546 	switch (si->off) {
10547 	case offsetof(struct bpf_sock, bound_dev_if):
10548 		BUILD_BUG_ON(sizeof_field(struct sock, sk_bound_dev_if) != 4);
10549 
10550 		if (type == BPF_WRITE)
10551 			*insn++ = BPF_EMIT_STORE(BPF_W, si,
10552 						 offsetof(struct sock, sk_bound_dev_if));
10553 		else
10554 			*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10555 				      offsetof(struct sock, sk_bound_dev_if));
10556 		break;
10557 
10558 	case offsetof(struct bpf_sock, mark):
10559 		BUILD_BUG_ON(sizeof_field(struct sock, sk_mark) != 4);
10560 
10561 		if (type == BPF_WRITE)
10562 			*insn++ = BPF_EMIT_STORE(BPF_W, si,
10563 						 offsetof(struct sock, sk_mark));
10564 		else
10565 			*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10566 				      offsetof(struct sock, sk_mark));
10567 		break;
10568 
10569 	case offsetof(struct bpf_sock, priority):
10570 		BUILD_BUG_ON(sizeof_field(struct sock, sk_priority) != 4);
10571 
10572 		if (type == BPF_WRITE)
10573 			*insn++ = BPF_EMIT_STORE(BPF_W, si,
10574 						 offsetof(struct sock, sk_priority));
10575 		else
10576 			*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
10577 				      offsetof(struct sock, sk_priority));
10578 		break;
10579 
10580 	case offsetof(struct bpf_sock, family):
10581 		*insn++ = BPF_LDX_MEM(
10582 			BPF_FIELD_SIZEOF(struct sock_common, skc_family),
10583 			si->dst_reg, si->src_reg,
10584 			bpf_target_off(struct sock_common,
10585 				       skc_family,
10586 				       sizeof_field(struct sock_common,
10587 						    skc_family),
10588 				       target_size));
10589 		break;
10590 
10591 	case offsetof(struct bpf_sock, type):
10592 		*insn++ = BPF_LDX_MEM(
10593 			BPF_FIELD_SIZEOF(struct sock, sk_type),
10594 			si->dst_reg, si->src_reg,
10595 			bpf_target_off(struct sock, sk_type,
10596 				       sizeof_field(struct sock, sk_type),
10597 				       target_size));
10598 		break;
10599 
10600 	case offsetof(struct bpf_sock, protocol):
10601 		*insn++ = BPF_LDX_MEM(
10602 			BPF_FIELD_SIZEOF(struct sock, sk_protocol),
10603 			si->dst_reg, si->src_reg,
10604 			bpf_target_off(struct sock, sk_protocol,
10605 				       sizeof_field(struct sock, sk_protocol),
10606 				       target_size));
10607 		break;
10608 
10609 	case offsetof(struct bpf_sock, src_ip4):
10610 		*insn++ = BPF_LDX_MEM(
10611 			BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10612 			bpf_target_off(struct sock_common, skc_rcv_saddr,
10613 				       sizeof_field(struct sock_common,
10614 						    skc_rcv_saddr),
10615 				       target_size));
10616 		break;
10617 
10618 	case offsetof(struct bpf_sock, dst_ip4):
10619 		*insn++ = BPF_LDX_MEM(
10620 			BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10621 			bpf_target_off(struct sock_common, skc_daddr,
10622 				       sizeof_field(struct sock_common,
10623 						    skc_daddr),
10624 				       target_size));
10625 		break;
10626 
10627 	case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
10628 #if IS_ENABLED(CONFIG_IPV6)
10629 		off = si->off;
10630 		off -= offsetof(struct bpf_sock, src_ip6[0]);
10631 		*insn++ = BPF_LDX_MEM(
10632 			BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10633 			bpf_target_off(
10634 				struct sock_common,
10635 				skc_v6_rcv_saddr.s6_addr32[0],
10636 				sizeof_field(struct sock_common,
10637 					     skc_v6_rcv_saddr.s6_addr32[0]),
10638 				target_size) + off);
10639 #else
10640 		(void)off;
10641 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10642 #endif
10643 		break;
10644 
10645 	case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
10646 #if IS_ENABLED(CONFIG_IPV6)
10647 		off = si->off;
10648 		off -= offsetof(struct bpf_sock, dst_ip6[0]);
10649 		*insn++ = BPF_LDX_MEM(
10650 			BPF_SIZE(si->code), si->dst_reg, si->src_reg,
10651 			bpf_target_off(struct sock_common,
10652 				       skc_v6_daddr.s6_addr32[0],
10653 				       sizeof_field(struct sock_common,
10654 						    skc_v6_daddr.s6_addr32[0]),
10655 				       target_size) + off);
10656 #else
10657 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
10658 		*target_size = 4;
10659 #endif
10660 		break;
10661 
10662 	case offsetof(struct bpf_sock, src_port):
10663 		*insn++ = BPF_LDX_MEM(
10664 			BPF_FIELD_SIZEOF(struct sock_common, skc_num),
10665 			si->dst_reg, si->src_reg,
10666 			bpf_target_off(struct sock_common, skc_num,
10667 				       sizeof_field(struct sock_common,
10668 						    skc_num),
10669 				       target_size));
10670 		break;
10671 
10672 	case offsetof(struct bpf_sock, dst_port):
10673 		*insn++ = BPF_LDX_MEM(
10674 			BPF_FIELD_SIZEOF(struct sock_common, skc_dport),
10675 			si->dst_reg, si->src_reg,
10676 			bpf_target_off(struct sock_common, skc_dport,
10677 				       sizeof_field(struct sock_common,
10678 						    skc_dport),
10679 				       target_size));
10680 		break;
10681 
10682 	case offsetof(struct bpf_sock, state):
10683 		*insn++ = BPF_LDX_MEM(
10684 			BPF_FIELD_SIZEOF(struct sock_common, skc_state),
10685 			si->dst_reg, si->src_reg,
10686 			bpf_target_off(struct sock_common, skc_state,
10687 				       sizeof_field(struct sock_common,
10688 						    skc_state),
10689 				       target_size));
10690 		break;
10691 	case offsetof(struct bpf_sock, rx_queue_mapping):
10692 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
10693 		*insn++ = BPF_LDX_MEM(
10694 			BPF_FIELD_SIZEOF(struct sock, sk_rx_queue_mapping),
10695 			si->dst_reg, si->src_reg,
10696 			bpf_target_off(struct sock, sk_rx_queue_mapping,
10697 				       sizeof_field(struct sock,
10698 						    sk_rx_queue_mapping),
10699 				       target_size));
10700 		*insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING,
10701 				      1);
10702 		*insn++ = BPF_MOV32_IMM(si->dst_reg, -1);
10703 #else
10704 		*insn++ = BPF_MOV32_IMM(si->dst_reg, -1);
10705 #endif
10706 		*target_size = sizeof_field(struct bpf_sock, rx_queue_mapping);
10707 
10708 		break;
10709 	}
10710 
10711 	return insn - insn_buf;
10712 }
10713 
10714 static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type,
10715 					 const struct bpf_insn *si,
10716 					 struct bpf_insn *insn_buf,
10717 					 struct bpf_prog *prog, u32 *target_size)
10718 {
10719 	struct bpf_insn *insn = insn_buf;
10720 
10721 	switch (si->off) {
10722 	case offsetof(struct __sk_buff, ifindex):
10723 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
10724 				      si->dst_reg, si->src_reg,
10725 				      offsetof(struct sk_buff, dev));
10726 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10727 				      bpf_target_off(struct net_device, ifindex, 4,
10728 						     target_size));
10729 		break;
10730 	default:
10731 		return bpf_convert_ctx_access(type, si, insn_buf, prog,
10732 					      target_size);
10733 	}
10734 
10735 	return insn - insn_buf;
10736 }
10737 
10738 static u32 xdp_convert_ctx_access(enum bpf_access_type type,
10739 				  const struct bpf_insn *si,
10740 				  struct bpf_insn *insn_buf,
10741 				  struct bpf_prog *prog, u32 *target_size)
10742 {
10743 	struct bpf_insn *insn = insn_buf;
10744 
10745 	switch (si->off) {
10746 	case offsetof(struct xdp_md, data):
10747 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data),
10748 				      si->dst_reg, si->src_reg,
10749 				      offsetof(struct xdp_buff, data));
10750 		break;
10751 	case offsetof(struct xdp_md, data_meta):
10752 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_meta),
10753 				      si->dst_reg, si->src_reg,
10754 				      offsetof(struct xdp_buff, data_meta));
10755 		break;
10756 	case offsetof(struct xdp_md, data_end):
10757 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_end),
10758 				      si->dst_reg, si->src_reg,
10759 				      offsetof(struct xdp_buff, data_end));
10760 		break;
10761 	case offsetof(struct xdp_md, ingress_ifindex):
10762 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
10763 				      si->dst_reg, si->src_reg,
10764 				      offsetof(struct xdp_buff, rxq));
10765 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_rxq_info, dev),
10766 				      si->dst_reg, si->dst_reg,
10767 				      offsetof(struct xdp_rxq_info, dev));
10768 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10769 				      offsetof(struct net_device, ifindex));
10770 		break;
10771 	case offsetof(struct xdp_md, rx_queue_index):
10772 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
10773 				      si->dst_reg, si->src_reg,
10774 				      offsetof(struct xdp_buff, rxq));
10775 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10776 				      offsetof(struct xdp_rxq_info,
10777 					       queue_index));
10778 		break;
10779 	case offsetof(struct xdp_md, egress_ifindex):
10780 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, txq),
10781 				      si->dst_reg, si->src_reg,
10782 				      offsetof(struct xdp_buff, txq));
10783 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_txq_info, dev),
10784 				      si->dst_reg, si->dst_reg,
10785 				      offsetof(struct xdp_txq_info, dev));
10786 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
10787 				      offsetof(struct net_device, ifindex));
10788 		break;
10789 	}
10790 
10791 	return insn - insn_buf;
10792 }
10793 
10794 /* SOCK_ADDR_LOAD_NESTED_FIELD() loads Nested Field S.F.NF where S is type of
10795  * context Structure, F is Field in context structure that contains a pointer
10796  * to Nested Structure of type NS that has the field NF.
10797  *
10798  * SIZE encodes the load size (BPF_B, BPF_H, etc). It's up to caller to make
10799  * sure that SIZE is not greater than actual size of S.F.NF.
10800  *
10801  * If offset OFF is provided, the load happens from that offset relative to
10802  * offset of NF.
10803  */
10804 #define SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF)	       \
10805 	do {								       \
10806 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), si->dst_reg,     \
10807 				      si->src_reg, offsetof(S, F));	       \
10808 		*insn++ = BPF_LDX_MEM(					       \
10809 			SIZE, si->dst_reg, si->dst_reg,			       \
10810 			bpf_target_off(NS, NF, sizeof_field(NS, NF),	       \
10811 				       target_size)			       \
10812 				+ OFF);					       \
10813 	} while (0)
10814 
10815 #define SOCK_ADDR_LOAD_NESTED_FIELD(S, NS, F, NF)			       \
10816 	SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF,		       \
10817 					     BPF_FIELD_SIZEOF(NS, NF), 0)
10818 
10819 /* SOCK_ADDR_STORE_NESTED_FIELD_OFF() has semantic similar to
10820  * SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF() but for store operation.
10821  *
10822  * In addition it uses Temporary Field TF (member of struct S) as the 3rd
10823  * "register" since two registers available in convert_ctx_access are not
10824  * enough: we can't override neither SRC, since it contains value to store, nor
10825  * DST since it contains pointer to context that may be used by later
10826  * instructions. But we need a temporary place to save pointer to nested
10827  * structure whose field we want to store to.
10828  */
10829 #define SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, OFF, TF)	       \
10830 	do {								       \
10831 		int tmp_reg = BPF_REG_9;				       \
10832 		if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg)	       \
10833 			--tmp_reg;					       \
10834 		if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg)	       \
10835 			--tmp_reg;					       \
10836 		*insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, tmp_reg,	       \
10837 				      offsetof(S, TF));			       \
10838 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), tmp_reg,	       \
10839 				      si->dst_reg, offsetof(S, F));	       \
10840 		*insn++ = BPF_RAW_INSN(SIZE | BPF_MEM | BPF_CLASS(si->code),   \
10841 				       tmp_reg, si->src_reg,		       \
10842 			bpf_target_off(NS, NF, sizeof_field(NS, NF),	       \
10843 				       target_size)			       \
10844 				       + OFF,				       \
10845 				       si->imm);			       \
10846 		*insn++ = BPF_LDX_MEM(BPF_DW, tmp_reg, si->dst_reg,	       \
10847 				      offsetof(S, TF));			       \
10848 	} while (0)
10849 
10850 #define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF, \
10851 						      TF)		       \
10852 	do {								       \
10853 		if (type == BPF_WRITE) {				       \
10854 			SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE,   \
10855 							 OFF, TF);	       \
10856 		} else {						       \
10857 			SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(		       \
10858 				S, NS, F, NF, SIZE, OFF);  \
10859 		}							       \
10860 	} while (0)
10861 
10862 static u32 sock_addr_convert_ctx_access(enum bpf_access_type type,
10863 					const struct bpf_insn *si,
10864 					struct bpf_insn *insn_buf,
10865 					struct bpf_prog *prog, u32 *target_size)
10866 {
10867 	int off, port_size = sizeof_field(struct sockaddr_in6, sin6_port);
10868 	struct bpf_insn *insn = insn_buf;
10869 
10870 	switch (si->off) {
10871 	case offsetof(struct bpf_sock_addr, user_family):
10872 		SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10873 					    struct sockaddr, uaddr, sa_family);
10874 		break;
10875 
10876 	case offsetof(struct bpf_sock_addr, user_ip4):
10877 		SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10878 			struct bpf_sock_addr_kern, struct sockaddr_in, uaddr,
10879 			sin_addr, BPF_SIZE(si->code), 0, tmp_reg);
10880 		break;
10881 
10882 	case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
10883 		off = si->off;
10884 		off -= offsetof(struct bpf_sock_addr, user_ip6[0]);
10885 		SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10886 			struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
10887 			sin6_addr.s6_addr32[0], BPF_SIZE(si->code), off,
10888 			tmp_reg);
10889 		break;
10890 
10891 	case offsetof(struct bpf_sock_addr, user_port):
10892 		/* To get port we need to know sa_family first and then treat
10893 		 * sockaddr as either sockaddr_in or sockaddr_in6.
10894 		 * Though we can simplify since port field has same offset and
10895 		 * size in both structures.
10896 		 * Here we check this invariant and use just one of the
10897 		 * structures if it's true.
10898 		 */
10899 		BUILD_BUG_ON(offsetof(struct sockaddr_in, sin_port) !=
10900 			     offsetof(struct sockaddr_in6, sin6_port));
10901 		BUILD_BUG_ON(sizeof_field(struct sockaddr_in, sin_port) !=
10902 			     sizeof_field(struct sockaddr_in6, sin6_port));
10903 		/* Account for sin6_port being smaller than user_port. */
10904 		port_size = min(port_size, BPF_LDST_BYTES(si));
10905 		SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10906 			struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
10907 			sin6_port, bytes_to_bpf_size(port_size), 0, tmp_reg);
10908 		break;
10909 
10910 	case offsetof(struct bpf_sock_addr, family):
10911 		SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10912 					    struct sock, sk, sk_family);
10913 		break;
10914 
10915 	case offsetof(struct bpf_sock_addr, type):
10916 		SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10917 					    struct sock, sk, sk_type);
10918 		break;
10919 
10920 	case offsetof(struct bpf_sock_addr, protocol):
10921 		SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
10922 					    struct sock, sk, sk_protocol);
10923 		break;
10924 
10925 	case offsetof(struct bpf_sock_addr, msg_src_ip4):
10926 		/* Treat t_ctx as struct in_addr for msg_src_ip4. */
10927 		SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10928 			struct bpf_sock_addr_kern, struct in_addr, t_ctx,
10929 			s_addr, BPF_SIZE(si->code), 0, tmp_reg);
10930 		break;
10931 
10932 	case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
10933 				msg_src_ip6[3]):
10934 		off = si->off;
10935 		off -= offsetof(struct bpf_sock_addr, msg_src_ip6[0]);
10936 		/* Treat t_ctx as struct in6_addr for msg_src_ip6. */
10937 		SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
10938 			struct bpf_sock_addr_kern, struct in6_addr, t_ctx,
10939 			s6_addr32[0], BPF_SIZE(si->code), off, tmp_reg);
10940 		break;
10941 	case offsetof(struct bpf_sock_addr, sk):
10942 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_addr_kern, sk),
10943 				      si->dst_reg, si->src_reg,
10944 				      offsetof(struct bpf_sock_addr_kern, sk));
10945 		break;
10946 	}
10947 
10948 	return insn - insn_buf;
10949 }
10950 
10951 static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
10952 				       const struct bpf_insn *si,
10953 				       struct bpf_insn *insn_buf,
10954 				       struct bpf_prog *prog,
10955 				       u32 *target_size)
10956 {
10957 	struct bpf_insn *insn = insn_buf;
10958 	int off;
10959 
10960 /* Helper macro for adding read access to tcp_sock or sock fields. */
10961 #define SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)			      \
10962 	do {								      \
10963 		int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 2;     \
10964 		BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) >		      \
10965 			     sizeof_field(struct bpf_sock_ops, BPF_FIELD));   \
10966 		if (si->dst_reg == reg || si->src_reg == reg)		      \
10967 			reg--;						      \
10968 		if (si->dst_reg == reg || si->src_reg == reg)		      \
10969 			reg--;						      \
10970 		if (si->dst_reg == si->src_reg) {			      \
10971 			*insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg,	      \
10972 					  offsetof(struct bpf_sock_ops_kern,  \
10973 					  temp));			      \
10974 			fullsock_reg = reg;				      \
10975 			jmp += 2;					      \
10976 		}							      \
10977 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \
10978 						struct bpf_sock_ops_kern,     \
10979 						is_locked_tcp_sock),	      \
10980 				      fullsock_reg, si->src_reg,	      \
10981 				      offsetof(struct bpf_sock_ops_kern,      \
10982 					       is_locked_tcp_sock));	      \
10983 		*insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp);	      \
10984 		if (si->dst_reg == si->src_reg)				      \
10985 			*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,	      \
10986 				      offsetof(struct bpf_sock_ops_kern,      \
10987 				      temp));				      \
10988 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \
10989 						struct bpf_sock_ops_kern, sk),\
10990 				      si->dst_reg, si->src_reg,		      \
10991 				      offsetof(struct bpf_sock_ops_kern, sk));\
10992 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(OBJ,		      \
10993 						       OBJ_FIELD),	      \
10994 				      si->dst_reg, si->dst_reg,		      \
10995 				      offsetof(OBJ, OBJ_FIELD));	      \
10996 		if (si->dst_reg == si->src_reg)	{			      \
10997 			*insn++ = BPF_JMP_A(2);				      \
10998 			*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,	      \
10999 				      offsetof(struct bpf_sock_ops_kern,      \
11000 				      temp));				      \
11001 			*insn++ = BPF_MOV64_IMM(si->dst_reg, 0);	      \
11002 		}							      \
11003 	} while (0)
11004 
11005 #define SOCK_OPS_GET_SK()							      \
11006 	do {								      \
11007 		int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 1;     \
11008 		if (si->dst_reg == reg || si->src_reg == reg)		      \
11009 			reg--;						      \
11010 		if (si->dst_reg == reg || si->src_reg == reg)		      \
11011 			reg--;						      \
11012 		if (si->dst_reg == si->src_reg) {			      \
11013 			*insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg,	      \
11014 					  offsetof(struct bpf_sock_ops_kern,  \
11015 					  temp));			      \
11016 			fullsock_reg = reg;				      \
11017 			jmp += 2;					      \
11018 		}							      \
11019 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \
11020 						struct bpf_sock_ops_kern,     \
11021 						is_fullsock),		      \
11022 				      fullsock_reg, si->src_reg,	      \
11023 				      offsetof(struct bpf_sock_ops_kern,      \
11024 					       is_fullsock));		      \
11025 		*insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp);	      \
11026 		if (si->dst_reg == si->src_reg)				      \
11027 			*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,	      \
11028 				      offsetof(struct bpf_sock_ops_kern,      \
11029 				      temp));				      \
11030 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \
11031 						struct bpf_sock_ops_kern, sk),\
11032 				      si->dst_reg, si->src_reg,		      \
11033 				      offsetof(struct bpf_sock_ops_kern, sk));\
11034 		if (si->dst_reg == si->src_reg)	{			      \
11035 			*insn++ = BPF_JMP_A(2);				      \
11036 			*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,	      \
11037 				      offsetof(struct bpf_sock_ops_kern,      \
11038 				      temp));				      \
11039 			*insn++ = BPF_MOV64_IMM(si->dst_reg, 0);	      \
11040 		}							      \
11041 	} while (0)
11042 
11043 #define SOCK_OPS_GET_TCP_SOCK_FIELD(FIELD) \
11044 		SOCK_OPS_GET_FIELD(FIELD, FIELD, struct tcp_sock)
11045 
11046 /* Helper macro for adding write access to tcp_sock or sock fields.
11047  * The macro is called with two registers, dst_reg which contains a pointer
11048  * to ctx (context) and src_reg which contains the value that should be
11049  * stored. However, we need an additional register since we cannot overwrite
11050  * dst_reg because it may be used later in the program.
11051  * Instead we "borrow" one of the other register. We first save its value
11052  * into a new (temp) field in bpf_sock_ops_kern, use it, and then restore
11053  * it at the end of the macro.
11054  */
11055 #define SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)			      \
11056 	do {								      \
11057 		int reg = BPF_REG_9;					      \
11058 		BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) >		      \
11059 			     sizeof_field(struct bpf_sock_ops, BPF_FIELD));   \
11060 		if (si->dst_reg == reg || si->src_reg == reg)		      \
11061 			reg--;						      \
11062 		if (si->dst_reg == reg || si->src_reg == reg)		      \
11063 			reg--;						      \
11064 		*insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, reg,		      \
11065 				      offsetof(struct bpf_sock_ops_kern,      \
11066 					       temp));			      \
11067 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \
11068 						struct bpf_sock_ops_kern,     \
11069 						is_locked_tcp_sock),	      \
11070 				      reg, si->dst_reg,			      \
11071 				      offsetof(struct bpf_sock_ops_kern,      \
11072 					       is_locked_tcp_sock));	      \
11073 		*insn++ = BPF_JMP_IMM(BPF_JEQ, reg, 0, 2);		      \
11074 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \
11075 						struct bpf_sock_ops_kern, sk),\
11076 				      reg, si->dst_reg,			      \
11077 				      offsetof(struct bpf_sock_ops_kern, sk));\
11078 		*insn++ = BPF_RAW_INSN(BPF_FIELD_SIZEOF(OBJ, OBJ_FIELD) |     \
11079 				       BPF_MEM | BPF_CLASS(si->code),	      \
11080 				       reg, si->src_reg,		      \
11081 				       offsetof(OBJ, OBJ_FIELD),	      \
11082 				       si->imm);			      \
11083 		*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->dst_reg,		      \
11084 				      offsetof(struct bpf_sock_ops_kern,      \
11085 					       temp));			      \
11086 	} while (0)
11087 
11088 #define SOCK_OPS_GET_OR_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ, TYPE)	      \
11089 	do {								      \
11090 		if (TYPE == BPF_WRITE)					      \
11091 			SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \
11092 		else							      \
11093 			SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \
11094 	} while (0)
11095 
11096 	switch (si->off) {
11097 	case offsetof(struct bpf_sock_ops, op):
11098 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
11099 						       op),
11100 				      si->dst_reg, si->src_reg,
11101 				      offsetof(struct bpf_sock_ops_kern, op));
11102 		break;
11103 
11104 	case offsetof(struct bpf_sock_ops, replylong[0]) ...
11105 	     offsetof(struct bpf_sock_ops, replylong[3]):
11106 		BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, reply) !=
11107 			     sizeof_field(struct bpf_sock_ops_kern, reply));
11108 		BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, replylong) !=
11109 			     sizeof_field(struct bpf_sock_ops_kern, replylong));
11110 		off = si->off;
11111 		off -= offsetof(struct bpf_sock_ops, replylong[0]);
11112 		off += offsetof(struct bpf_sock_ops_kern, replylong[0]);
11113 		if (type == BPF_WRITE)
11114 			*insn++ = BPF_EMIT_STORE(BPF_W, si, off);
11115 		else
11116 			*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
11117 					      off);
11118 		break;
11119 
11120 	case offsetof(struct bpf_sock_ops, family):
11121 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
11122 
11123 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11124 					      struct bpf_sock_ops_kern, sk),
11125 				      si->dst_reg, si->src_reg,
11126 				      offsetof(struct bpf_sock_ops_kern, sk));
11127 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11128 				      offsetof(struct sock_common, skc_family));
11129 		break;
11130 
11131 	case offsetof(struct bpf_sock_ops, remote_ip4):
11132 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
11133 
11134 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11135 						struct bpf_sock_ops_kern, sk),
11136 				      si->dst_reg, si->src_reg,
11137 				      offsetof(struct bpf_sock_ops_kern, sk));
11138 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11139 				      offsetof(struct sock_common, skc_daddr));
11140 		break;
11141 
11142 	case offsetof(struct bpf_sock_ops, local_ip4):
11143 		BUILD_BUG_ON(sizeof_field(struct sock_common,
11144 					  skc_rcv_saddr) != 4);
11145 
11146 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11147 					      struct bpf_sock_ops_kern, sk),
11148 				      si->dst_reg, si->src_reg,
11149 				      offsetof(struct bpf_sock_ops_kern, sk));
11150 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11151 				      offsetof(struct sock_common,
11152 					       skc_rcv_saddr));
11153 		break;
11154 
11155 	case offsetof(struct bpf_sock_ops, remote_ip6[0]) ...
11156 	     offsetof(struct bpf_sock_ops, remote_ip6[3]):
11157 #if IS_ENABLED(CONFIG_IPV6)
11158 		BUILD_BUG_ON(sizeof_field(struct sock_common,
11159 					  skc_v6_daddr.s6_addr32[0]) != 4);
11160 
11161 		off = si->off;
11162 		off -= offsetof(struct bpf_sock_ops, remote_ip6[0]);
11163 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11164 						struct bpf_sock_ops_kern, sk),
11165 				      si->dst_reg, si->src_reg,
11166 				      offsetof(struct bpf_sock_ops_kern, sk));
11167 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11168 				      offsetof(struct sock_common,
11169 					       skc_v6_daddr.s6_addr32[0]) +
11170 				      off);
11171 #else
11172 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11173 #endif
11174 		break;
11175 
11176 	case offsetof(struct bpf_sock_ops, local_ip6[0]) ...
11177 	     offsetof(struct bpf_sock_ops, local_ip6[3]):
11178 #if IS_ENABLED(CONFIG_IPV6)
11179 		BUILD_BUG_ON(sizeof_field(struct sock_common,
11180 					  skc_v6_rcv_saddr.s6_addr32[0]) != 4);
11181 
11182 		off = si->off;
11183 		off -= offsetof(struct bpf_sock_ops, local_ip6[0]);
11184 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11185 						struct bpf_sock_ops_kern, sk),
11186 				      si->dst_reg, si->src_reg,
11187 				      offsetof(struct bpf_sock_ops_kern, sk));
11188 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11189 				      offsetof(struct sock_common,
11190 					       skc_v6_rcv_saddr.s6_addr32[0]) +
11191 				      off);
11192 #else
11193 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11194 #endif
11195 		break;
11196 
11197 	case offsetof(struct bpf_sock_ops, remote_port):
11198 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
11199 
11200 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11201 						struct bpf_sock_ops_kern, sk),
11202 				      si->dst_reg, si->src_reg,
11203 				      offsetof(struct bpf_sock_ops_kern, sk));
11204 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11205 				      offsetof(struct sock_common, skc_dport));
11206 #ifndef __BIG_ENDIAN_BITFIELD
11207 		*insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
11208 #endif
11209 		break;
11210 
11211 	case offsetof(struct bpf_sock_ops, local_port):
11212 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
11213 
11214 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11215 						struct bpf_sock_ops_kern, sk),
11216 				      si->dst_reg, si->src_reg,
11217 				      offsetof(struct bpf_sock_ops_kern, sk));
11218 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11219 				      offsetof(struct sock_common, skc_num));
11220 		break;
11221 
11222 	case offsetof(struct bpf_sock_ops, is_fullsock):
11223 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11224 						struct bpf_sock_ops_kern,
11225 						is_fullsock),
11226 				      si->dst_reg, si->src_reg,
11227 				      offsetof(struct bpf_sock_ops_kern,
11228 					       is_fullsock));
11229 		break;
11230 
11231 	case offsetof(struct bpf_sock_ops, state):
11232 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_state) != 1);
11233 
11234 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11235 						struct bpf_sock_ops_kern, sk),
11236 				      si->dst_reg, si->src_reg,
11237 				      offsetof(struct bpf_sock_ops_kern, sk));
11238 		*insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->dst_reg,
11239 				      offsetof(struct sock_common, skc_state));
11240 		break;
11241 
11242 	case offsetof(struct bpf_sock_ops, rtt_min):
11243 		SOCK_OPS_GET_FIELD(rtt_min, rtt_min.s[0].v, struct tcp_sock);
11244 		break;
11245 
11246 	case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags):
11247 		SOCK_OPS_GET_FIELD(bpf_sock_ops_cb_flags, bpf_sock_ops_cb_flags,
11248 				   struct tcp_sock);
11249 		break;
11250 
11251 	case offsetof(struct bpf_sock_ops, sk_txhash):
11252 		SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,
11253 					  struct sock, type);
11254 		break;
11255 	case offsetof(struct bpf_sock_ops, snd_cwnd):
11256 		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_cwnd);
11257 		break;
11258 	case offsetof(struct bpf_sock_ops, srtt_us):
11259 		SOCK_OPS_GET_TCP_SOCK_FIELD(srtt_us);
11260 		break;
11261 	case offsetof(struct bpf_sock_ops, snd_ssthresh):
11262 		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_ssthresh);
11263 		break;
11264 	case offsetof(struct bpf_sock_ops, rcv_nxt):
11265 		SOCK_OPS_GET_TCP_SOCK_FIELD(rcv_nxt);
11266 		break;
11267 	case offsetof(struct bpf_sock_ops, snd_nxt):
11268 		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_nxt);
11269 		break;
11270 	case offsetof(struct bpf_sock_ops, snd_una):
11271 		SOCK_OPS_GET_TCP_SOCK_FIELD(snd_una);
11272 		break;
11273 	case offsetof(struct bpf_sock_ops, mss_cache):
11274 		SOCK_OPS_GET_TCP_SOCK_FIELD(mss_cache);
11275 		break;
11276 	case offsetof(struct bpf_sock_ops, ecn_flags):
11277 		SOCK_OPS_GET_TCP_SOCK_FIELD(ecn_flags);
11278 		break;
11279 	case offsetof(struct bpf_sock_ops, rate_delivered):
11280 		SOCK_OPS_GET_TCP_SOCK_FIELD(rate_delivered);
11281 		break;
11282 	case offsetof(struct bpf_sock_ops, rate_interval_us):
11283 		SOCK_OPS_GET_TCP_SOCK_FIELD(rate_interval_us);
11284 		break;
11285 	case offsetof(struct bpf_sock_ops, packets_out):
11286 		SOCK_OPS_GET_TCP_SOCK_FIELD(packets_out);
11287 		break;
11288 	case offsetof(struct bpf_sock_ops, retrans_out):
11289 		SOCK_OPS_GET_TCP_SOCK_FIELD(retrans_out);
11290 		break;
11291 	case offsetof(struct bpf_sock_ops, total_retrans):
11292 		SOCK_OPS_GET_TCP_SOCK_FIELD(total_retrans);
11293 		break;
11294 	case offsetof(struct bpf_sock_ops, segs_in):
11295 		SOCK_OPS_GET_TCP_SOCK_FIELD(segs_in);
11296 		break;
11297 	case offsetof(struct bpf_sock_ops, data_segs_in):
11298 		SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_in);
11299 		break;
11300 	case offsetof(struct bpf_sock_ops, segs_out):
11301 		SOCK_OPS_GET_TCP_SOCK_FIELD(segs_out);
11302 		break;
11303 	case offsetof(struct bpf_sock_ops, data_segs_out):
11304 		SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_out);
11305 		break;
11306 	case offsetof(struct bpf_sock_ops, lost_out):
11307 		SOCK_OPS_GET_TCP_SOCK_FIELD(lost_out);
11308 		break;
11309 	case offsetof(struct bpf_sock_ops, sacked_out):
11310 		SOCK_OPS_GET_TCP_SOCK_FIELD(sacked_out);
11311 		break;
11312 	case offsetof(struct bpf_sock_ops, bytes_received):
11313 		SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_received);
11314 		break;
11315 	case offsetof(struct bpf_sock_ops, bytes_acked):
11316 		SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_acked);
11317 		break;
11318 	case offsetof(struct bpf_sock_ops, sk):
11319 		SOCK_OPS_GET_SK();
11320 		break;
11321 	case offsetof(struct bpf_sock_ops, skb_data_end):
11322 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
11323 						       skb_data_end),
11324 				      si->dst_reg, si->src_reg,
11325 				      offsetof(struct bpf_sock_ops_kern,
11326 					       skb_data_end));
11327 		break;
11328 	case offsetof(struct bpf_sock_ops, skb_data):
11329 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
11330 						       skb),
11331 				      si->dst_reg, si->src_reg,
11332 				      offsetof(struct bpf_sock_ops_kern,
11333 					       skb));
11334 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
11335 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
11336 				      si->dst_reg, si->dst_reg,
11337 				      offsetof(struct sk_buff, data));
11338 		break;
11339 	case offsetof(struct bpf_sock_ops, skb_len):
11340 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
11341 						       skb),
11342 				      si->dst_reg, si->src_reg,
11343 				      offsetof(struct bpf_sock_ops_kern,
11344 					       skb));
11345 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
11346 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
11347 				      si->dst_reg, si->dst_reg,
11348 				      offsetof(struct sk_buff, len));
11349 		break;
11350 	case offsetof(struct bpf_sock_ops, skb_tcp_flags):
11351 		off = offsetof(struct sk_buff, cb);
11352 		off += offsetof(struct tcp_skb_cb, tcp_flags);
11353 		*target_size = sizeof_field(struct tcp_skb_cb, tcp_flags);
11354 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
11355 						       skb),
11356 				      si->dst_reg, si->src_reg,
11357 				      offsetof(struct bpf_sock_ops_kern,
11358 					       skb));
11359 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
11360 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_skb_cb,
11361 						       tcp_flags),
11362 				      si->dst_reg, si->dst_reg, off);
11363 		break;
11364 	case offsetof(struct bpf_sock_ops, skb_hwtstamp): {
11365 		struct bpf_insn *jmp_on_null_skb;
11366 
11367 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
11368 						       skb),
11369 				      si->dst_reg, si->src_reg,
11370 				      offsetof(struct bpf_sock_ops_kern,
11371 					       skb));
11372 		/* Reserve one insn to test skb == NULL */
11373 		jmp_on_null_skb = insn++;
11374 		insn = bpf_convert_shinfo_access(si->dst_reg, si->dst_reg, insn);
11375 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
11376 				      bpf_target_off(struct skb_shared_info,
11377 						     hwtstamps, 8,
11378 						     target_size));
11379 		*jmp_on_null_skb = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0,
11380 					       insn - jmp_on_null_skb - 1);
11381 		break;
11382 	}
11383 	}
11384 	return insn - insn_buf;
11385 }
11386 
11387 /* data_end = skb->data + skb_headlen() */
11388 static struct bpf_insn *bpf_convert_data_end_access(const struct bpf_insn *si,
11389 						    struct bpf_insn *insn)
11390 {
11391 	int reg;
11392 	int temp_reg_off = offsetof(struct sk_buff, cb) +
11393 			   offsetof(struct sk_skb_cb, temp_reg);
11394 
11395 	if (si->src_reg == si->dst_reg) {
11396 		/* We need an extra register, choose and save a register. */
11397 		reg = BPF_REG_9;
11398 		if (si->src_reg == reg || si->dst_reg == reg)
11399 			reg--;
11400 		if (si->src_reg == reg || si->dst_reg == reg)
11401 			reg--;
11402 		*insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, temp_reg_off);
11403 	} else {
11404 		reg = si->dst_reg;
11405 	}
11406 
11407 	/* reg = skb->data */
11408 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
11409 			      reg, si->src_reg,
11410 			      offsetof(struct sk_buff, data));
11411 	/* AX = skb->len */
11412 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
11413 			      BPF_REG_AX, si->src_reg,
11414 			      offsetof(struct sk_buff, len));
11415 	/* reg = skb->data + skb->len */
11416 	*insn++ = BPF_ALU64_REG(BPF_ADD, reg, BPF_REG_AX);
11417 	/* AX = skb->data_len */
11418 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data_len),
11419 			      BPF_REG_AX, si->src_reg,
11420 			      offsetof(struct sk_buff, data_len));
11421 
11422 	/* reg = skb->data + skb->len - skb->data_len */
11423 	*insn++ = BPF_ALU64_REG(BPF_SUB, reg, BPF_REG_AX);
11424 
11425 	if (si->src_reg == si->dst_reg) {
11426 		/* Restore the saved register */
11427 		*insn++ = BPF_MOV64_REG(BPF_REG_AX, si->src_reg);
11428 		*insn++ = BPF_MOV64_REG(si->dst_reg, reg);
11429 		*insn++ = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_AX, temp_reg_off);
11430 	}
11431 
11432 	return insn;
11433 }
11434 
11435 static u32 sk_skb_convert_ctx_access(enum bpf_access_type type,
11436 				     const struct bpf_insn *si,
11437 				     struct bpf_insn *insn_buf,
11438 				     struct bpf_prog *prog, u32 *target_size)
11439 {
11440 	struct bpf_insn *insn = insn_buf;
11441 	int off;
11442 
11443 	switch (si->off) {
11444 	case offsetof(struct __sk_buff, data_end):
11445 		insn = bpf_convert_data_end_access(si, insn);
11446 		break;
11447 	case offsetof(struct __sk_buff, cb[0]) ...
11448 	     offsetofend(struct __sk_buff, cb[4]) - 1:
11449 		BUILD_BUG_ON(sizeof_field(struct sk_skb_cb, data) < 20);
11450 		BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
11451 			      offsetof(struct sk_skb_cb, data)) %
11452 			     sizeof(__u64));
11453 
11454 		prog->cb_access = 1;
11455 		off  = si->off;
11456 		off -= offsetof(struct __sk_buff, cb[0]);
11457 		off += offsetof(struct sk_buff, cb);
11458 		off += offsetof(struct sk_skb_cb, data);
11459 		if (type == BPF_WRITE)
11460 			*insn++ = BPF_EMIT_STORE(BPF_SIZE(si->code), si, off);
11461 		else
11462 			*insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
11463 					      si->src_reg, off);
11464 		break;
11465 
11466 
11467 	default:
11468 		return bpf_convert_ctx_access(type, si, insn_buf, prog,
11469 					      target_size);
11470 	}
11471 
11472 	return insn - insn_buf;
11473 }
11474 
11475 static u32 sk_msg_convert_ctx_access(enum bpf_access_type type,
11476 				     const struct bpf_insn *si,
11477 				     struct bpf_insn *insn_buf,
11478 				     struct bpf_prog *prog, u32 *target_size)
11479 {
11480 	struct bpf_insn *insn = insn_buf;
11481 #if IS_ENABLED(CONFIG_IPV6)
11482 	int off;
11483 #endif
11484 
11485 	/* convert ctx uses the fact sg element is first in struct */
11486 	BUILD_BUG_ON(offsetof(struct sk_msg, sg) != 0);
11487 
11488 	switch (si->off) {
11489 	case offsetof(struct sk_msg_md, data):
11490 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data),
11491 				      si->dst_reg, si->src_reg,
11492 				      offsetof(struct sk_msg, data));
11493 		break;
11494 	case offsetof(struct sk_msg_md, data_end):
11495 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data_end),
11496 				      si->dst_reg, si->src_reg,
11497 				      offsetof(struct sk_msg, data_end));
11498 		break;
11499 	case offsetof(struct sk_msg_md, family):
11500 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);
11501 
11502 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11503 					      struct sk_msg, sk),
11504 				      si->dst_reg, si->src_reg,
11505 				      offsetof(struct sk_msg, sk));
11506 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11507 				      offsetof(struct sock_common, skc_family));
11508 		break;
11509 
11510 	case offsetof(struct sk_msg_md, remote_ip4):
11511 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);
11512 
11513 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11514 						struct sk_msg, sk),
11515 				      si->dst_reg, si->src_reg,
11516 				      offsetof(struct sk_msg, sk));
11517 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11518 				      offsetof(struct sock_common, skc_daddr));
11519 		break;
11520 
11521 	case offsetof(struct sk_msg_md, local_ip4):
11522 		BUILD_BUG_ON(sizeof_field(struct sock_common,
11523 					  skc_rcv_saddr) != 4);
11524 
11525 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11526 					      struct sk_msg, sk),
11527 				      si->dst_reg, si->src_reg,
11528 				      offsetof(struct sk_msg, sk));
11529 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11530 				      offsetof(struct sock_common,
11531 					       skc_rcv_saddr));
11532 		break;
11533 
11534 	case offsetof(struct sk_msg_md, remote_ip6[0]) ...
11535 	     offsetof(struct sk_msg_md, remote_ip6[3]):
11536 #if IS_ENABLED(CONFIG_IPV6)
11537 		BUILD_BUG_ON(sizeof_field(struct sock_common,
11538 					  skc_v6_daddr.s6_addr32[0]) != 4);
11539 
11540 		off = si->off;
11541 		off -= offsetof(struct sk_msg_md, remote_ip6[0]);
11542 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11543 						struct sk_msg, sk),
11544 				      si->dst_reg, si->src_reg,
11545 				      offsetof(struct sk_msg, sk));
11546 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11547 				      offsetof(struct sock_common,
11548 					       skc_v6_daddr.s6_addr32[0]) +
11549 				      off);
11550 #else
11551 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11552 #endif
11553 		break;
11554 
11555 	case offsetof(struct sk_msg_md, local_ip6[0]) ...
11556 	     offsetof(struct sk_msg_md, local_ip6[3]):
11557 #if IS_ENABLED(CONFIG_IPV6)
11558 		BUILD_BUG_ON(sizeof_field(struct sock_common,
11559 					  skc_v6_rcv_saddr.s6_addr32[0]) != 4);
11560 
11561 		off = si->off;
11562 		off -= offsetof(struct sk_msg_md, local_ip6[0]);
11563 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11564 						struct sk_msg, sk),
11565 				      si->dst_reg, si->src_reg,
11566 				      offsetof(struct sk_msg, sk));
11567 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
11568 				      offsetof(struct sock_common,
11569 					       skc_v6_rcv_saddr.s6_addr32[0]) +
11570 				      off);
11571 #else
11572 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
11573 #endif
11574 		break;
11575 
11576 	case offsetof(struct sk_msg_md, remote_port):
11577 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);
11578 
11579 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11580 						struct sk_msg, sk),
11581 				      si->dst_reg, si->src_reg,
11582 				      offsetof(struct sk_msg, sk));
11583 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11584 				      offsetof(struct sock_common, skc_dport));
11585 #ifndef __BIG_ENDIAN_BITFIELD
11586 		*insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
11587 #endif
11588 		break;
11589 
11590 	case offsetof(struct sk_msg_md, local_port):
11591 		BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);
11592 
11593 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
11594 						struct sk_msg, sk),
11595 				      si->dst_reg, si->src_reg,
11596 				      offsetof(struct sk_msg, sk));
11597 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
11598 				      offsetof(struct sock_common, skc_num));
11599 		break;
11600 
11601 	case offsetof(struct sk_msg_md, size):
11602 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_sg, size),
11603 				      si->dst_reg, si->src_reg,
11604 				      offsetof(struct sk_msg_sg, size));
11605 		break;
11606 
11607 	case offsetof(struct sk_msg_md, sk):
11608 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, sk),
11609 				      si->dst_reg, si->src_reg,
11610 				      offsetof(struct sk_msg, sk));
11611 		break;
11612 	}
11613 
11614 	return insn - insn_buf;
11615 }
11616 
11617 const struct bpf_verifier_ops sk_filter_verifier_ops = {
11618 	.get_func_proto		= sk_filter_func_proto,
11619 	.is_valid_access	= sk_filter_is_valid_access,
11620 	.convert_ctx_access	= bpf_convert_ctx_access,
11621 	.gen_ld_abs		= bpf_gen_ld_abs,
11622 };
11623 
11624 const struct bpf_prog_ops sk_filter_prog_ops = {
11625 	.test_run		= bpf_prog_test_run_skb,
11626 };
11627 
11628 const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
11629 	.get_func_proto		= tc_cls_act_func_proto,
11630 	.is_valid_access	= tc_cls_act_is_valid_access,
11631 	.convert_ctx_access	= tc_cls_act_convert_ctx_access,
11632 	.gen_prologue		= tc_cls_act_prologue,
11633 	.gen_ld_abs		= bpf_gen_ld_abs,
11634 	.btf_struct_access	= tc_cls_act_btf_struct_access,
11635 };
11636 
11637 const struct bpf_prog_ops tc_cls_act_prog_ops = {
11638 	.test_run		= bpf_prog_test_run_skb,
11639 };
11640 
11641 const struct bpf_verifier_ops xdp_verifier_ops = {
11642 	.get_func_proto		= xdp_func_proto,
11643 	.is_valid_access	= xdp_is_valid_access,
11644 	.convert_ctx_access	= xdp_convert_ctx_access,
11645 	.gen_prologue		= bpf_noop_prologue,
11646 	.btf_struct_access	= xdp_btf_struct_access,
11647 };
11648 
11649 const struct bpf_prog_ops xdp_prog_ops = {
11650 	.test_run		= bpf_prog_test_run_xdp,
11651 };
11652 
11653 const struct bpf_verifier_ops cg_skb_verifier_ops = {
11654 	.get_func_proto		= cg_skb_func_proto,
11655 	.is_valid_access	= cg_skb_is_valid_access,
11656 	.convert_ctx_access	= bpf_convert_ctx_access,
11657 };
11658 
11659 const struct bpf_prog_ops cg_skb_prog_ops = {
11660 	.test_run		= bpf_prog_test_run_skb,
11661 };
11662 
11663 const struct bpf_verifier_ops lwt_in_verifier_ops = {
11664 	.get_func_proto		= lwt_in_func_proto,
11665 	.is_valid_access	= lwt_is_valid_access,
11666 	.convert_ctx_access	= bpf_convert_ctx_access,
11667 };
11668 
11669 const struct bpf_prog_ops lwt_in_prog_ops = {
11670 	.test_run		= bpf_prog_test_run_skb,
11671 };
11672 
11673 const struct bpf_verifier_ops lwt_out_verifier_ops = {
11674 	.get_func_proto		= lwt_out_func_proto,
11675 	.is_valid_access	= lwt_is_valid_access,
11676 	.convert_ctx_access	= bpf_convert_ctx_access,
11677 };
11678 
11679 const struct bpf_prog_ops lwt_out_prog_ops = {
11680 	.test_run		= bpf_prog_test_run_skb,
11681 };
11682 
11683 const struct bpf_verifier_ops lwt_xmit_verifier_ops = {
11684 	.get_func_proto		= lwt_xmit_func_proto,
11685 	.is_valid_access	= lwt_is_valid_access,
11686 	.convert_ctx_access	= bpf_convert_ctx_access,
11687 	.gen_prologue		= tc_cls_act_prologue,
11688 };
11689 
11690 const struct bpf_prog_ops lwt_xmit_prog_ops = {
11691 	.test_run		= bpf_prog_test_run_skb,
11692 };
11693 
11694 const struct bpf_verifier_ops lwt_seg6local_verifier_ops = {
11695 	.get_func_proto		= lwt_seg6local_func_proto,
11696 	.is_valid_access	= lwt_is_valid_access,
11697 	.convert_ctx_access	= bpf_convert_ctx_access,
11698 };
11699 
11700 const struct bpf_prog_ops lwt_seg6local_prog_ops = {
11701 };
11702 
11703 const struct bpf_verifier_ops cg_sock_verifier_ops = {
11704 	.get_func_proto		= sock_filter_func_proto,
11705 	.is_valid_access	= sock_filter_is_valid_access,
11706 	.convert_ctx_access	= bpf_sock_convert_ctx_access,
11707 };
11708 
11709 const struct bpf_prog_ops cg_sock_prog_ops = {
11710 };
11711 
11712 const struct bpf_verifier_ops cg_sock_addr_verifier_ops = {
11713 	.get_func_proto		= sock_addr_func_proto,
11714 	.is_valid_access	= sock_addr_is_valid_access,
11715 	.convert_ctx_access	= sock_addr_convert_ctx_access,
11716 };
11717 
11718 const struct bpf_prog_ops cg_sock_addr_prog_ops = {
11719 };
11720 
11721 const struct bpf_verifier_ops sock_ops_verifier_ops = {
11722 	.get_func_proto		= sock_ops_func_proto,
11723 	.is_valid_access	= sock_ops_is_valid_access,
11724 	.convert_ctx_access	= sock_ops_convert_ctx_access,
11725 };
11726 
11727 const struct bpf_prog_ops sock_ops_prog_ops = {
11728 };
11729 
11730 const struct bpf_verifier_ops sk_skb_verifier_ops = {
11731 	.get_func_proto		= sk_skb_func_proto,
11732 	.is_valid_access	= sk_skb_is_valid_access,
11733 	.convert_ctx_access	= sk_skb_convert_ctx_access,
11734 	.gen_prologue		= sk_skb_prologue,
11735 };
11736 
11737 const struct bpf_prog_ops sk_skb_prog_ops = {
11738 };
11739 
11740 const struct bpf_verifier_ops sk_msg_verifier_ops = {
11741 	.get_func_proto		= sk_msg_func_proto,
11742 	.is_valid_access	= sk_msg_is_valid_access,
11743 	.convert_ctx_access	= sk_msg_convert_ctx_access,
11744 	.gen_prologue		= bpf_noop_prologue,
11745 };
11746 
11747 const struct bpf_prog_ops sk_msg_prog_ops = {
11748 };
11749 
11750 const struct bpf_verifier_ops flow_dissector_verifier_ops = {
11751 	.get_func_proto		= flow_dissector_func_proto,
11752 	.is_valid_access	= flow_dissector_is_valid_access,
11753 	.convert_ctx_access	= flow_dissector_convert_ctx_access,
11754 };
11755 
11756 const struct bpf_prog_ops flow_dissector_prog_ops = {
11757 	.test_run		= bpf_prog_test_run_flow_dissector,
11758 };
11759 
11760 int sk_detach_filter(struct sock *sk)
11761 {
11762 	int ret = -ENOENT;
11763 	struct sk_filter *filter;
11764 
11765 	if (sock_flag(sk, SOCK_FILTER_LOCKED))
11766 		return -EPERM;
11767 
11768 	filter = rcu_dereference_protected(sk->sk_filter,
11769 					   lockdep_sock_is_held(sk));
11770 	if (filter) {
11771 		RCU_INIT_POINTER(sk->sk_filter, NULL);
11772 		sk_filter_uncharge(sk, filter);
11773 		ret = 0;
11774 	}
11775 
11776 	return ret;
11777 }
11778 EXPORT_SYMBOL_GPL(sk_detach_filter);
11779 
11780 int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len)
11781 {
11782 	struct sock_fprog_kern *fprog;
11783 	struct sk_filter *filter;
11784 	int ret = 0;
11785 
11786 	sockopt_lock_sock(sk);
11787 	filter = rcu_dereference_protected(sk->sk_filter,
11788 					   lockdep_sock_is_held(sk));
11789 	if (!filter)
11790 		goto out;
11791 
11792 	/* We're copying the filter that has been originally attached,
11793 	 * so no conversion/decode needed anymore. eBPF programs that
11794 	 * have no original program cannot be dumped through this.
11795 	 */
11796 	ret = -EACCES;
11797 	fprog = filter->prog->orig_prog;
11798 	if (!fprog)
11799 		goto out;
11800 
11801 	ret = fprog->len;
11802 	if (!len)
11803 		/* User space only enquires number of filter blocks. */
11804 		goto out;
11805 
11806 	ret = -EINVAL;
11807 	if (len < fprog->len)
11808 		goto out;
11809 
11810 	ret = -EFAULT;
11811 	if (copy_to_sockptr(optval, fprog->filter, bpf_classic_proglen(fprog)))
11812 		goto out;
11813 
11814 	/* Instead of bytes, the API requests to return the number
11815 	 * of filter blocks.
11816 	 */
11817 	ret = fprog->len;
11818 out:
11819 	sockopt_release_sock(sk);
11820 	return ret;
11821 }
11822 
11823 #ifdef CONFIG_INET
11824 static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern,
11825 				    struct sock_reuseport *reuse,
11826 				    struct sock *sk, struct sk_buff *skb,
11827 				    struct sock *migrating_sk,
11828 				    u32 hash)
11829 {
11830 	reuse_kern->skb = skb;
11831 	reuse_kern->sk = sk;
11832 	reuse_kern->selected_sk = NULL;
11833 	reuse_kern->migrating_sk = migrating_sk;
11834 	reuse_kern->data_end = skb->data + skb_headlen(skb);
11835 	reuse_kern->hash = hash;
11836 	reuse_kern->reuseport_id = reuse->reuseport_id;
11837 	reuse_kern->bind_inany = reuse->bind_inany;
11838 }
11839 
11840 struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
11841 				  struct bpf_prog *prog, struct sk_buff *skb,
11842 				  struct sock *migrating_sk,
11843 				  u32 hash)
11844 {
11845 	struct sk_reuseport_kern reuse_kern;
11846 	enum sk_action action;
11847 
11848 	bpf_init_reuseport_kern(&reuse_kern, reuse, sk, skb, migrating_sk, hash);
11849 	action = bpf_prog_run(prog, &reuse_kern);
11850 
11851 	if (action == SK_PASS)
11852 		return reuse_kern.selected_sk;
11853 	else
11854 		return ERR_PTR(-ECONNREFUSED);
11855 }
11856 
11857 BPF_CALL_4(sk_select_reuseport, struct sk_reuseport_kern *, reuse_kern,
11858 	   struct bpf_map *, map, void *, key, u32, flags)
11859 {
11860 	bool is_sockarray = map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
11861 	struct sock_reuseport *reuse;
11862 	struct sock *selected_sk;
11863 	int err;
11864 
11865 	selected_sk = map->ops->map_lookup_elem(map, key);
11866 	if (!selected_sk)
11867 		return -ENOENT;
11868 
11869 	reuse = rcu_dereference(selected_sk->sk_reuseport_cb);
11870 	if (!reuse) {
11871 		/* reuseport_array has only sk with non NULL sk_reuseport_cb.
11872 		 * The only (!reuse) case here is - the sk has already been
11873 		 * unhashed (e.g. by close()), so treat it as -ENOENT.
11874 		 *
11875 		 * Other maps (e.g. sock_map) do not provide this guarantee and
11876 		 * the sk may never be in the reuseport group to begin with.
11877 		 */
11878 		err = is_sockarray ? -ENOENT : -EINVAL;
11879 		goto error;
11880 	}
11881 
11882 	if (unlikely(reuse->reuseport_id != reuse_kern->reuseport_id)) {
11883 		struct sock *sk = reuse_kern->sk;
11884 
11885 		if (sk->sk_protocol != selected_sk->sk_protocol) {
11886 			err = -EPROTOTYPE;
11887 		} else if (sk->sk_family != selected_sk->sk_family) {
11888 			err = -EAFNOSUPPORT;
11889 		} else {
11890 			/* Catch all. Likely bound to a different sockaddr. */
11891 			err = -EBADFD;
11892 		}
11893 		goto error;
11894 	}
11895 
11896 	reuse_kern->selected_sk = selected_sk;
11897 
11898 	return 0;
11899 error:
11900 	/* Lookup in sock_map can return TCP ESTABLISHED sockets. */
11901 	if (sk_is_refcounted(selected_sk))
11902 		sock_put(selected_sk);
11903 
11904 	return err;
11905 }
11906 
11907 static const struct bpf_func_proto sk_select_reuseport_proto = {
11908 	.func           = sk_select_reuseport,
11909 	.gpl_only       = false,
11910 	.ret_type       = RET_INTEGER,
11911 	.arg1_type	= ARG_PTR_TO_CTX,
11912 	.arg2_type      = ARG_CONST_MAP_PTR,
11913 	.arg3_type      = ARG_PTR_TO_MAP_KEY,
11914 	.arg4_type	= ARG_ANYTHING,
11915 };
11916 
11917 BPF_CALL_4(sk_reuseport_load_bytes,
11918 	   const struct sk_reuseport_kern *, reuse_kern, u32, offset,
11919 	   void *, to, u32, len)
11920 {
11921 	return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
11922 }
11923 
11924 static const struct bpf_func_proto sk_reuseport_load_bytes_proto = {
11925 	.func		= sk_reuseport_load_bytes,
11926 	.gpl_only	= false,
11927 	.ret_type	= RET_INTEGER,
11928 	.arg1_type	= ARG_PTR_TO_CTX,
11929 	.arg2_type	= ARG_ANYTHING,
11930 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
11931 	.arg4_type	= ARG_MEM_SIZE,
11932 };
11933 
11934 BPF_CALL_5(sk_reuseport_load_bytes_relative,
11935 	   const struct sk_reuseport_kern *, reuse_kern, u32, offset,
11936 	   void *, to, u32, len, u32, start_header)
11937 {
11938 	return ____bpf_skb_load_bytes_relative(reuse_kern->skb, offset, to,
11939 					       len, start_header);
11940 }
11941 
11942 static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = {
11943 	.func		= sk_reuseport_load_bytes_relative,
11944 	.gpl_only	= false,
11945 	.ret_type	= RET_INTEGER,
11946 	.arg1_type	= ARG_PTR_TO_CTX,
11947 	.arg2_type	= ARG_ANYTHING,
11948 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
11949 	.arg4_type	= ARG_MEM_SIZE,
11950 	.arg5_type	= ARG_ANYTHING,
11951 };
11952 
11953 static const struct bpf_func_proto *
11954 sk_reuseport_func_proto(enum bpf_func_id func_id,
11955 			const struct bpf_prog *prog)
11956 {
11957 	switch (func_id) {
11958 	case BPF_FUNC_sk_select_reuseport:
11959 		return &sk_select_reuseport_proto;
11960 	case BPF_FUNC_skb_load_bytes:
11961 		return &sk_reuseport_load_bytes_proto;
11962 	case BPF_FUNC_skb_load_bytes_relative:
11963 		return &sk_reuseport_load_bytes_relative_proto;
11964 	case BPF_FUNC_get_socket_cookie:
11965 		return &bpf_get_socket_ptr_cookie_proto;
11966 	case BPF_FUNC_ktime_get_coarse_ns:
11967 		return &bpf_ktime_get_coarse_ns_proto;
11968 	default:
11969 		return bpf_base_func_proto(func_id, prog);
11970 	}
11971 }
11972 
11973 static bool
11974 sk_reuseport_is_valid_access(int off, int size,
11975 			     enum bpf_access_type type,
11976 			     const struct bpf_prog *prog,
11977 			     struct bpf_insn_access_aux *info)
11978 {
11979 	const u32 size_default = sizeof(__u32);
11980 
11981 	if (off < 0 || off >= sizeof(struct sk_reuseport_md) ||
11982 	    off % size || type != BPF_READ)
11983 		return false;
11984 
11985 	switch (off) {
11986 	case offsetof(struct sk_reuseport_md, data):
11987 		info->reg_type = PTR_TO_PACKET;
11988 		return size == sizeof(__u64);
11989 
11990 	case offsetof(struct sk_reuseport_md, data_end):
11991 		info->reg_type = PTR_TO_PACKET_END;
11992 		return size == sizeof(__u64);
11993 
11994 	case offsetof(struct sk_reuseport_md, hash):
11995 		return size == size_default;
11996 
11997 	case offsetof(struct sk_reuseport_md, sk):
11998 		info->reg_type = PTR_TO_SOCKET;
11999 		return size == sizeof(__u64);
12000 
12001 	case offsetof(struct sk_reuseport_md, migrating_sk):
12002 		info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
12003 		return size == sizeof(__u64);
12004 
12005 	/* Fields that allow narrowing */
12006 	case bpf_ctx_range(struct sk_reuseport_md, eth_protocol):
12007 		if (size < sizeof_field(struct sk_buff, protocol))
12008 			return false;
12009 		fallthrough;
12010 	case bpf_ctx_range(struct sk_reuseport_md, ip_protocol):
12011 	case bpf_ctx_range(struct sk_reuseport_md, bind_inany):
12012 	case bpf_ctx_range(struct sk_reuseport_md, len):
12013 		bpf_ctx_record_field_size(info, size_default);
12014 		return bpf_ctx_narrow_access_ok(off, size, size_default);
12015 
12016 	default:
12017 		return false;
12018 	}
12019 }
12020 
12021 #define SK_REUSEPORT_LOAD_FIELD(F) ({					\
12022 	*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_reuseport_kern, F), \
12023 			      si->dst_reg, si->src_reg,			\
12024 			      bpf_target_off(struct sk_reuseport_kern, F, \
12025 					     sizeof_field(struct sk_reuseport_kern, F), \
12026 					     target_size));		\
12027 	})
12028 
12029 #define SK_REUSEPORT_LOAD_SKB_FIELD(SKB_FIELD)				\
12030 	SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern,		\
12031 				    struct sk_buff,			\
12032 				    skb,				\
12033 				    SKB_FIELD)
12034 
12035 #define SK_REUSEPORT_LOAD_SK_FIELD(SK_FIELD)				\
12036 	SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern,		\
12037 				    struct sock,			\
12038 				    sk,					\
12039 				    SK_FIELD)
12040 
12041 static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type,
12042 					   const struct bpf_insn *si,
12043 					   struct bpf_insn *insn_buf,
12044 					   struct bpf_prog *prog,
12045 					   u32 *target_size)
12046 {
12047 	struct bpf_insn *insn = insn_buf;
12048 
12049 	switch (si->off) {
12050 	case offsetof(struct sk_reuseport_md, data):
12051 		SK_REUSEPORT_LOAD_SKB_FIELD(data);
12052 		break;
12053 
12054 	case offsetof(struct sk_reuseport_md, len):
12055 		SK_REUSEPORT_LOAD_SKB_FIELD(len);
12056 		break;
12057 
12058 	case offsetof(struct sk_reuseport_md, eth_protocol):
12059 		SK_REUSEPORT_LOAD_SKB_FIELD(protocol);
12060 		break;
12061 
12062 	case offsetof(struct sk_reuseport_md, ip_protocol):
12063 		SK_REUSEPORT_LOAD_SK_FIELD(sk_protocol);
12064 		break;
12065 
12066 	case offsetof(struct sk_reuseport_md, data_end):
12067 		SK_REUSEPORT_LOAD_FIELD(data_end);
12068 		break;
12069 
12070 	case offsetof(struct sk_reuseport_md, hash):
12071 		SK_REUSEPORT_LOAD_FIELD(hash);
12072 		break;
12073 
12074 	case offsetof(struct sk_reuseport_md, bind_inany):
12075 		SK_REUSEPORT_LOAD_FIELD(bind_inany);
12076 		break;
12077 
12078 	case offsetof(struct sk_reuseport_md, sk):
12079 		SK_REUSEPORT_LOAD_FIELD(sk);
12080 		break;
12081 
12082 	case offsetof(struct sk_reuseport_md, migrating_sk):
12083 		SK_REUSEPORT_LOAD_FIELD(migrating_sk);
12084 		break;
12085 	}
12086 
12087 	return insn - insn_buf;
12088 }
12089 
12090 const struct bpf_verifier_ops sk_reuseport_verifier_ops = {
12091 	.get_func_proto		= sk_reuseport_func_proto,
12092 	.is_valid_access	= sk_reuseport_is_valid_access,
12093 	.convert_ctx_access	= sk_reuseport_convert_ctx_access,
12094 };
12095 
12096 const struct bpf_prog_ops sk_reuseport_prog_ops = {
12097 };
12098 
12099 DEFINE_STATIC_KEY_FALSE(bpf_sk_lookup_enabled);
12100 EXPORT_SYMBOL(bpf_sk_lookup_enabled);
12101 
12102 BPF_CALL_3(bpf_sk_lookup_assign, struct bpf_sk_lookup_kern *, ctx,
12103 	   struct sock *, sk, u64, flags)
12104 {
12105 	if (unlikely(flags & ~(BPF_SK_LOOKUP_F_REPLACE |
12106 			       BPF_SK_LOOKUP_F_NO_REUSEPORT)))
12107 		return -EINVAL;
12108 	if (unlikely(sk && sk_is_refcounted(sk)))
12109 		return -ESOCKTNOSUPPORT; /* reject non-RCU freed sockets */
12110 	if (unlikely(sk && sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN))
12111 		return -ESOCKTNOSUPPORT; /* only accept TCP socket in LISTEN */
12112 	if (unlikely(sk && sk_is_udp(sk) && sk->sk_state != TCP_CLOSE))
12113 		return -ESOCKTNOSUPPORT; /* only accept UDP socket in CLOSE */
12114 
12115 	/* Check if socket is suitable for packet L3/L4 protocol */
12116 	if (sk && sk->sk_protocol != ctx->protocol)
12117 		return -EPROTOTYPE;
12118 	if (sk && sk->sk_family != ctx->family &&
12119 	    (sk->sk_family == AF_INET || ipv6_only_sock(sk)))
12120 		return -EAFNOSUPPORT;
12121 
12122 	if (ctx->selected_sk && !(flags & BPF_SK_LOOKUP_F_REPLACE))
12123 		return -EEXIST;
12124 
12125 	/* Select socket as lookup result */
12126 	ctx->selected_sk = sk;
12127 	ctx->no_reuseport = flags & BPF_SK_LOOKUP_F_NO_REUSEPORT;
12128 	return 0;
12129 }
12130 
12131 static const struct bpf_func_proto bpf_sk_lookup_assign_proto = {
12132 	.func		= bpf_sk_lookup_assign,
12133 	.gpl_only	= false,
12134 	.ret_type	= RET_INTEGER,
12135 	.arg1_type	= ARG_PTR_TO_CTX,
12136 	.arg2_type	= ARG_PTR_TO_SOCKET_OR_NULL,
12137 	.arg3_type	= ARG_ANYTHING,
12138 };
12139 
12140 static const struct bpf_func_proto *
12141 sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
12142 {
12143 	switch (func_id) {
12144 	case BPF_FUNC_perf_event_output:
12145 		return &bpf_event_output_data_proto;
12146 	case BPF_FUNC_sk_assign:
12147 		return &bpf_sk_lookup_assign_proto;
12148 	case BPF_FUNC_sk_release:
12149 		return &bpf_sk_release_proto;
12150 	default:
12151 		return bpf_sk_base_func_proto(func_id, prog);
12152 	}
12153 }
12154 
12155 static bool sk_lookup_is_valid_access(int off, int size,
12156 				      enum bpf_access_type type,
12157 				      const struct bpf_prog *prog,
12158 				      struct bpf_insn_access_aux *info)
12159 {
12160 	if (off < 0 || off >= sizeof(struct bpf_sk_lookup))
12161 		return false;
12162 	if (off % size != 0)
12163 		return false;
12164 	if (type != BPF_READ)
12165 		return false;
12166 
12167 	switch (off) {
12168 	case bpf_ctx_range_ptr(struct bpf_sk_lookup, sk):
12169 		info->reg_type = PTR_TO_SOCKET_OR_NULL;
12170 		return size == sizeof(__u64);
12171 
12172 	case bpf_ctx_range(struct bpf_sk_lookup, family):
12173 	case bpf_ctx_range(struct bpf_sk_lookup, protocol):
12174 	case bpf_ctx_range(struct bpf_sk_lookup, remote_ip4):
12175 	case bpf_ctx_range(struct bpf_sk_lookup, local_ip4):
12176 	case bpf_ctx_range_till(struct bpf_sk_lookup, remote_ip6[0], remote_ip6[3]):
12177 	case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]):
12178 	case bpf_ctx_range(struct bpf_sk_lookup, local_port):
12179 	case bpf_ctx_range(struct bpf_sk_lookup, ingress_ifindex):
12180 		bpf_ctx_record_field_size(info, sizeof(__u32));
12181 		return bpf_ctx_narrow_access_ok(off, size, sizeof(__u32));
12182 
12183 	case bpf_ctx_range(struct bpf_sk_lookup, remote_port):
12184 		/* Allow 4-byte access to 2-byte field for backward compatibility */
12185 		if (size == sizeof(__u32))
12186 			return true;
12187 		bpf_ctx_record_field_size(info, sizeof(__be16));
12188 		return bpf_ctx_narrow_access_ok(off, size, sizeof(__be16));
12189 
12190 	case offsetofend(struct bpf_sk_lookup, remote_port) ...
12191 	     offsetof(struct bpf_sk_lookup, local_ip4) - 1:
12192 		/* Allow access to zero padding for backward compatibility */
12193 		bpf_ctx_record_field_size(info, sizeof(__u16));
12194 		return bpf_ctx_narrow_access_ok(off, size, sizeof(__u16));
12195 
12196 	default:
12197 		return false;
12198 	}
12199 }
12200 
12201 static u32 sk_lookup_convert_ctx_access(enum bpf_access_type type,
12202 					const struct bpf_insn *si,
12203 					struct bpf_insn *insn_buf,
12204 					struct bpf_prog *prog,
12205 					u32 *target_size)
12206 {
12207 	struct bpf_insn *insn = insn_buf;
12208 
12209 	switch (si->off) {
12210 	case offsetof(struct bpf_sk_lookup, sk):
12211 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
12212 				      offsetof(struct bpf_sk_lookup_kern, selected_sk));
12213 		break;
12214 
12215 	case offsetof(struct bpf_sk_lookup, family):
12216 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
12217 				      bpf_target_off(struct bpf_sk_lookup_kern,
12218 						     family, 2, target_size));
12219 		break;
12220 
12221 	case offsetof(struct bpf_sk_lookup, protocol):
12222 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
12223 				      bpf_target_off(struct bpf_sk_lookup_kern,
12224 						     protocol, 2, target_size));
12225 		break;
12226 
12227 	case offsetof(struct bpf_sk_lookup, remote_ip4):
12228 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
12229 				      bpf_target_off(struct bpf_sk_lookup_kern,
12230 						     v4.saddr, 4, target_size));
12231 		break;
12232 
12233 	case offsetof(struct bpf_sk_lookup, local_ip4):
12234 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
12235 				      bpf_target_off(struct bpf_sk_lookup_kern,
12236 						     v4.daddr, 4, target_size));
12237 		break;
12238 
12239 	case bpf_ctx_range_till(struct bpf_sk_lookup,
12240 				remote_ip6[0], remote_ip6[3]): {
12241 #if IS_ENABLED(CONFIG_IPV6)
12242 		int off = si->off;
12243 
12244 		off -= offsetof(struct bpf_sk_lookup, remote_ip6[0]);
12245 		off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
12246 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
12247 				      offsetof(struct bpf_sk_lookup_kern, v6.saddr));
12248 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
12249 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
12250 #else
12251 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
12252 #endif
12253 		break;
12254 	}
12255 	case bpf_ctx_range_till(struct bpf_sk_lookup,
12256 				local_ip6[0], local_ip6[3]): {
12257 #if IS_ENABLED(CONFIG_IPV6)
12258 		int off = si->off;
12259 
12260 		off -= offsetof(struct bpf_sk_lookup, local_ip6[0]);
12261 		off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
12262 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
12263 				      offsetof(struct bpf_sk_lookup_kern, v6.daddr));
12264 		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
12265 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
12266 #else
12267 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
12268 #endif
12269 		break;
12270 	}
12271 	case offsetof(struct bpf_sk_lookup, remote_port):
12272 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
12273 				      bpf_target_off(struct bpf_sk_lookup_kern,
12274 						     sport, 2, target_size));
12275 		break;
12276 
12277 	case offsetofend(struct bpf_sk_lookup, remote_port):
12278 		*target_size = 2;
12279 		*insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
12280 		break;
12281 
12282 	case offsetof(struct bpf_sk_lookup, local_port):
12283 		*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
12284 				      bpf_target_off(struct bpf_sk_lookup_kern,
12285 						     dport, 2, target_size));
12286 		break;
12287 
12288 	case offsetof(struct bpf_sk_lookup, ingress_ifindex):
12289 		*insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
12290 				      bpf_target_off(struct bpf_sk_lookup_kern,
12291 						     ingress_ifindex, 4, target_size));
12292 		break;
12293 	}
12294 
12295 	return insn - insn_buf;
12296 }
12297 
12298 const struct bpf_prog_ops sk_lookup_prog_ops = {
12299 	.test_run = bpf_prog_test_run_sk_lookup,
12300 };
12301 
12302 const struct bpf_verifier_ops sk_lookup_verifier_ops = {
12303 	.get_func_proto		= sk_lookup_func_proto,
12304 	.is_valid_access	= sk_lookup_is_valid_access,
12305 	.convert_ctx_access	= sk_lookup_convert_ctx_access,
12306 };
12307 
12308 #endif /* CONFIG_INET */
12309 
12310 DEFINE_BPF_DISPATCHER(xdp)
12311 
12312 void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog)
12313 {
12314 	bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(xdp), prev_prog, prog);
12315 }
12316 
12317 BTF_ID_LIST_GLOBAL(btf_sock_ids, MAX_BTF_SOCK_TYPE)
12318 #define BTF_SOCK_TYPE(name, type) BTF_ID(struct, type)
12319 BTF_SOCK_TYPE_xxx
12320 #undef BTF_SOCK_TYPE
12321 
12322 BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
12323 {
12324 	/* tcp6_sock type is not generated in dwarf and hence btf,
12325 	 * trigger an explicit type generation here.
12326 	 */
12327 	BTF_TYPE_EMIT(struct tcp6_sock);
12328 	if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
12329 	    sk->sk_type == SOCK_STREAM && sk->sk_family == AF_INET6)
12330 		return (unsigned long)sk;
12331 
12332 	return (unsigned long)NULL;
12333 }
12334 
12335 const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
12336 	.func			= bpf_skc_to_tcp6_sock,
12337 	.gpl_only		= false,
12338 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
12339 	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
12340 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP6],
12341 };
12342 
12343 BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
12344 {
12345 	if (sk && sk_fullsock(sk) && sk_is_tcp(sk))
12346 		return (unsigned long)sk;
12347 
12348 	return (unsigned long)NULL;
12349 }
12350 
12351 const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = {
12352 	.func			= bpf_skc_to_tcp_sock,
12353 	.gpl_only		= false,
12354 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
12355 	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
12356 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP],
12357 };
12358 
12359 BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
12360 {
12361 	/* BTF types for tcp_timewait_sock and inet_timewait_sock are not
12362 	 * generated if CONFIG_INET=n. Trigger an explicit generation here.
12363 	 */
12364 	BTF_TYPE_EMIT(struct inet_timewait_sock);
12365 	BTF_TYPE_EMIT(struct tcp_timewait_sock);
12366 
12367 #ifdef CONFIG_INET
12368 	if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
12369 		return (unsigned long)sk;
12370 #endif
12371 
12372 #if IS_ENABLED(CONFIG_IPV6)
12373 	if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_TIME_WAIT)
12374 		return (unsigned long)sk;
12375 #endif
12376 
12377 	return (unsigned long)NULL;
12378 }
12379 
12380 const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = {
12381 	.func			= bpf_skc_to_tcp_timewait_sock,
12382 	.gpl_only		= false,
12383 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
12384 	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
12385 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP_TW],
12386 };
12387 
12388 BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk)
12389 {
12390 	/* Ensure BTF for the structs exists even with CONFIG_INET=n. */
12391 	BTF_TYPE_EMIT(struct inet_request_sock);
12392 	BTF_TYPE_EMIT(struct tcp_request_sock);
12393 
12394 #ifdef CONFIG_INET
12395 	if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV)
12396 		return (unsigned long)sk;
12397 #endif
12398 
12399 #if IS_ENABLED(CONFIG_IPV6)
12400 	if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_NEW_SYN_RECV)
12401 		return (unsigned long)sk;
12402 #endif
12403 
12404 	return (unsigned long)NULL;
12405 }
12406 
12407 const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto = {
12408 	.func			= bpf_skc_to_tcp_request_sock,
12409 	.gpl_only		= false,
12410 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
12411 	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
12412 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP_REQ],
12413 };
12414 
12415 BPF_CALL_1(bpf_skc_to_udp6_sock, struct sock *, sk)
12416 {
12417 	/* udp6_sock type is not generated in dwarf and hence btf,
12418 	 * trigger an explicit type generation here.
12419 	 */
12420 	BTF_TYPE_EMIT(struct udp6_sock);
12421 	if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_UDP &&
12422 	    sk->sk_type == SOCK_DGRAM && sk->sk_family == AF_INET6)
12423 		return (unsigned long)sk;
12424 
12425 	return (unsigned long)NULL;
12426 }
12427 
12428 const struct bpf_func_proto bpf_skc_to_udp6_sock_proto = {
12429 	.func			= bpf_skc_to_udp6_sock,
12430 	.gpl_only		= false,
12431 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
12432 	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
12433 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_UDP6],
12434 };
12435 
12436 BPF_CALL_1(bpf_skc_to_unix_sock, struct sock *, sk)
12437 {
12438 	/* unix_sock type is not generated in dwarf and hence btf,
12439 	 * trigger an explicit type generation here.
12440 	 */
12441 	BTF_TYPE_EMIT(struct unix_sock);
12442 	if (sk && sk_is_unix(sk))
12443 		return (unsigned long)sk;
12444 
12445 	return (unsigned long)NULL;
12446 }
12447 
12448 const struct bpf_func_proto bpf_skc_to_unix_sock_proto = {
12449 	.func			= bpf_skc_to_unix_sock,
12450 	.gpl_only		= false,
12451 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
12452 	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
12453 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_UNIX],
12454 };
12455 
12456 BPF_CALL_1(bpf_skc_to_mptcp_sock, struct sock *, sk)
12457 {
12458 	BTF_TYPE_EMIT(struct mptcp_sock);
12459 	return (unsigned long)bpf_mptcp_sock_from_subflow(sk);
12460 }
12461 
12462 const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto = {
12463 	.func		= bpf_skc_to_mptcp_sock,
12464 	.gpl_only	= false,
12465 	.ret_type	= RET_PTR_TO_BTF_ID_OR_NULL,
12466 	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
12467 	.ret_btf_id	= &btf_sock_ids[BTF_SOCK_TYPE_MPTCP],
12468 };
12469 
12470 BPF_CALL_1(bpf_sock_from_file, struct file *, file)
12471 {
12472 	return (unsigned long)sock_from_file(file);
12473 }
12474 
12475 BTF_ID_LIST(bpf_sock_from_file_btf_ids)
12476 BTF_ID(struct, socket)
12477 BTF_ID(struct, file)
12478 
12479 const struct bpf_func_proto bpf_sock_from_file_proto = {
12480 	.func		= bpf_sock_from_file,
12481 	.gpl_only	= false,
12482 	.ret_type	= RET_PTR_TO_BTF_ID_OR_NULL,
12483 	.ret_btf_id	= &bpf_sock_from_file_btf_ids[0],
12484 	.arg1_type	= ARG_PTR_TO_BTF_ID,
12485 	.arg1_btf_id	= &bpf_sock_from_file_btf_ids[1],
12486 };
12487 
12488 static const struct bpf_func_proto *
12489 bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
12490 {
12491 	const struct bpf_func_proto *func;
12492 
12493 	switch (func_id) {
12494 	case BPF_FUNC_skc_to_tcp6_sock:
12495 		func = &bpf_skc_to_tcp6_sock_proto;
12496 		break;
12497 	case BPF_FUNC_skc_to_tcp_sock:
12498 		func = &bpf_skc_to_tcp_sock_proto;
12499 		break;
12500 	case BPF_FUNC_skc_to_tcp_timewait_sock:
12501 		func = &bpf_skc_to_tcp_timewait_sock_proto;
12502 		break;
12503 	case BPF_FUNC_skc_to_tcp_request_sock:
12504 		func = &bpf_skc_to_tcp_request_sock_proto;
12505 		break;
12506 	case BPF_FUNC_skc_to_udp6_sock:
12507 		func = &bpf_skc_to_udp6_sock_proto;
12508 		break;
12509 	case BPF_FUNC_skc_to_unix_sock:
12510 		func = &bpf_skc_to_unix_sock_proto;
12511 		break;
12512 	case BPF_FUNC_skc_to_mptcp_sock:
12513 		func = &bpf_skc_to_mptcp_sock_proto;
12514 		break;
12515 	case BPF_FUNC_ktime_get_coarse_ns:
12516 		return &bpf_ktime_get_coarse_ns_proto;
12517 	default:
12518 		return bpf_base_func_proto(func_id, prog);
12519 	}
12520 
12521 	if (!bpf_token_capable(prog->aux->token, CAP_PERFMON))
12522 		return NULL;
12523 
12524 	return func;
12525 }
12526 
12527 /**
12528  * bpf_skb_meta_pointer() - Gets a mutable pointer within the skb metadata area.
12529  * @skb: socket buffer carrying the metadata
12530  * @offset: offset into the metadata area, must be <= skb_metadata_len()
12531  */
12532 void *bpf_skb_meta_pointer(struct sk_buff *skb, u32 offset)
12533 {
12534 	return skb_metadata_end(skb) - skb_metadata_len(skb) + offset;
12535 }
12536 
12537 int __bpf_skb_meta_store_bytes(struct sk_buff *skb, u32 offset,
12538 			       const void *from, u32 len, u64 flags)
12539 {
12540 	if (unlikely(flags))
12541 		return -EINVAL;
12542 	if (unlikely(bpf_try_make_writable_from(skb, 0, &from)))
12543 		return -EFAULT;
12544 
12545 	memmove(bpf_skb_meta_pointer(skb, offset), from, len);
12546 	return 0;
12547 }
12548 
12549 __bpf_kfunc_start_defs();
12550 __bpf_kfunc int bpf_dynptr_from_skb(struct __sk_buff *s, u64 flags,
12551 				    struct bpf_dynptr *ptr__uninit)
12552 {
12553 	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12554 	struct sk_buff *skb = (struct sk_buff *)s;
12555 
12556 	if (flags) {
12557 		bpf_dynptr_set_null(ptr);
12558 		return -EINVAL;
12559 	}
12560 
12561 	bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);
12562 
12563 	return 0;
12564 }
12565 
12566 /**
12567  * bpf_dynptr_from_skb_meta() - Initialize a dynptr to the skb metadata area.
12568  * @skb_: socket buffer carrying the metadata
12569  * @flags: future use, must be zero
12570  * @ptr__uninit: dynptr to initialize
12571  *
12572  * Set up a dynptr for access to the metadata area earlier allocated from the
12573  * XDP context with bpf_xdp_adjust_meta(). Serves as an alternative to
12574  * &__sk_buff->data_meta.
12575  *
12576  * Return:
12577  * * %0         - dynptr ready to use
12578  * * %-EINVAL   - invalid flags, dynptr set to null
12579  */
12580 __bpf_kfunc int bpf_dynptr_from_skb_meta(struct __sk_buff *skb_, u64 flags,
12581 					 struct bpf_dynptr *ptr__uninit)
12582 {
12583 	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12584 	struct sk_buff *skb = (struct sk_buff *)skb_;
12585 
12586 	if (flags) {
12587 		bpf_dynptr_set_null(ptr);
12588 		return -EINVAL;
12589 	}
12590 
12591 	bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB_META, 0, skb_metadata_len(skb));
12592 
12593 	return 0;
12594 }
12595 
12596 __bpf_kfunc int bpf_dynptr_from_xdp(struct xdp_md *x, u64 flags,
12597 				    struct bpf_dynptr *ptr__uninit)
12598 {
12599 	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12600 	struct xdp_buff *xdp = (struct xdp_buff *)x;
12601 
12602 	if (flags) {
12603 		bpf_dynptr_set_null(ptr);
12604 		return -EINVAL;
12605 	}
12606 
12607 	bpf_dynptr_init(ptr, xdp, BPF_DYNPTR_TYPE_XDP, 0, xdp_get_buff_len(xdp));
12608 
12609 	return 0;
12610 }
12611 
12612 __bpf_kfunc int bpf_sock_addr_set_sun_path(struct bpf_sock_addr_kern *sa_kern,
12613 					   const u8 *sun_path, u32 sun_path__sz)
12614 {
12615 	struct sockaddr_un *un;
12616 
12617 	if (sa_kern->sk->sk_family != AF_UNIX)
12618 		return -EINVAL;
12619 
12620 	/* We do not allow changing the address to unnamed or larger than the
12621 	 * maximum allowed address size for a unix sockaddr.
12622 	 */
12623 	if (sun_path__sz == 0 || sun_path__sz > UNIX_PATH_MAX)
12624 		return -EINVAL;
12625 
12626 	un = (struct sockaddr_un *)sa_kern->uaddr;
12627 	memcpy(un->sun_path, sun_path, sun_path__sz);
12628 	sa_kern->uaddrlen = offsetof(struct sockaddr_un, sun_path) + sun_path__sz;
12629 
12630 	return 0;
12631 }
12632 
12633 __bpf_kfunc int bpf_sk_assign_tcp_reqsk(struct __sk_buff *s, struct sock *sk,
12634 					struct bpf_tcp_req_attrs *attrs, int attrs__sz)
12635 {
12636 #if IS_ENABLED(CONFIG_SYN_COOKIES)
12637 	struct sk_buff *skb = (struct sk_buff *)s;
12638 	const struct request_sock_ops *ops;
12639 	struct inet_request_sock *ireq;
12640 	struct tcp_request_sock *treq;
12641 	struct request_sock *req;
12642 	struct net *net;
12643 	__u16 min_mss;
12644 	u32 tsoff = 0;
12645 
12646 	if (attrs__sz != sizeof(*attrs) ||
12647 	    attrs->reserved[0] || attrs->reserved[1] || attrs->reserved[2])
12648 		return -EINVAL;
12649 
12650 	if (!skb_at_tc_ingress(skb))
12651 		return -EINVAL;
12652 
12653 	net = dev_net(skb->dev);
12654 	if (net != sock_net(sk))
12655 		return -ENETUNREACH;
12656 
12657 	switch (skb->protocol) {
12658 	case htons(ETH_P_IP):
12659 		ops = &tcp_request_sock_ops;
12660 		min_mss = 536;
12661 		break;
12662 #if IS_ENABLED(CONFIG_IPV6)
12663 	case htons(ETH_P_IPV6):
12664 		ops = &tcp6_request_sock_ops;
12665 		min_mss = IPV6_MIN_MTU - 60;
12666 		break;
12667 #endif
12668 	default:
12669 		return -EINVAL;
12670 	}
12671 
12672 	if (sk->sk_type != SOCK_STREAM || sk->sk_state != TCP_LISTEN ||
12673 	    sk_is_mptcp(sk))
12674 		return -EINVAL;
12675 
12676 	if (attrs->mss < min_mss)
12677 		return -EINVAL;
12678 
12679 	if (attrs->wscale_ok) {
12680 		if (!READ_ONCE(net->ipv4.sysctl_tcp_window_scaling))
12681 			return -EINVAL;
12682 
12683 		if (attrs->snd_wscale > TCP_MAX_WSCALE ||
12684 		    attrs->rcv_wscale > TCP_MAX_WSCALE)
12685 			return -EINVAL;
12686 	}
12687 
12688 	if (attrs->sack_ok && !READ_ONCE(net->ipv4.sysctl_tcp_sack))
12689 		return -EINVAL;
12690 
12691 	if (attrs->tstamp_ok) {
12692 		if (!READ_ONCE(net->ipv4.sysctl_tcp_timestamps))
12693 			return -EINVAL;
12694 
12695 		tsoff = attrs->rcv_tsecr - tcp_ns_to_ts(attrs->usec_ts_ok, tcp_clock_ns());
12696 	}
12697 
12698 	req = inet_reqsk_alloc(ops, sk, false);
12699 	if (!req)
12700 		return -ENOMEM;
12701 
12702 	ireq = inet_rsk(req);
12703 	treq = tcp_rsk(req);
12704 
12705 	req->rsk_listener = sk;
12706 	req->syncookie = 1;
12707 	req->mss = attrs->mss;
12708 	req->ts_recent = attrs->rcv_tsval;
12709 
12710 	ireq->snd_wscale = attrs->snd_wscale;
12711 	ireq->rcv_wscale = attrs->rcv_wscale;
12712 	ireq->tstamp_ok	= !!attrs->tstamp_ok;
12713 	ireq->sack_ok = !!attrs->sack_ok;
12714 	ireq->wscale_ok = !!attrs->wscale_ok;
12715 	ireq->ecn_ok = !!attrs->ecn_ok;
12716 
12717 	treq->req_usec_ts = !!attrs->usec_ts_ok;
12718 	treq->ts_off = tsoff;
12719 
12720 	skb_orphan(skb);
12721 	skb->sk = req_to_sk(req);
12722 	skb->destructor = sock_pfree;
12723 
12724 	return 0;
12725 #else
12726 	return -EOPNOTSUPP;
12727 #endif
12728 }
12729 
12730 __bpf_kfunc int bpf_sock_ops_enable_tx_tstamp(struct bpf_sock_ops_kern *skops,
12731 					      u64 flags)
12732 {
12733 	struct sk_buff *skb;
12734 
12735 	if (skops->op != BPF_SOCK_OPS_TSTAMP_SENDMSG_CB)
12736 		return -EOPNOTSUPP;
12737 
12738 	if (flags)
12739 		return -EINVAL;
12740 
12741 	skb = skops->skb;
12742 	skb_shinfo(skb)->tx_flags |= SKBTX_BPF;
12743 	TCP_SKB_CB(skb)->txstamp_ack |= TSTAMP_ACK_BPF;
12744 	skb_shinfo(skb)->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
12745 
12746 	return 0;
12747 }
12748 
12749 /**
12750  * bpf_xdp_pull_data() - Pull in non-linear xdp data.
12751  * @x: &xdp_md associated with the XDP buffer
12752  * @len: length of data to be made directly accessible in the linear part
12753  *
12754  * Pull in data in case the XDP buffer associated with @x is non-linear and
12755  * not all @len are in the linear data area.
12756  *
12757  * Direct packet access allows reading and writing linear XDP data through
12758  * packet pointers (i.e., &xdp_md->data + offsets). The amount of data which
12759  * ends up in the linear part of the xdp_buff depends on the NIC and its
12760  * configuration. When a frag-capable XDP program wants to directly access
12761  * headers that may be in the non-linear area, call this kfunc to make sure
12762  * the data is available in the linear area. Alternatively, use dynptr or
12763  * bpf_xdp_{load,store}_bytes() to access data without pulling.
12764  *
12765  * This kfunc can also be used with bpf_xdp_adjust_head() to decapsulate
12766  * headers in the non-linear data area.
12767  *
12768  * A call to this kfunc may reduce headroom. If there is not enough tailroom
12769  * in the linear data area, metadata and data will be shifted down.
12770  *
12771  * A call to this kfunc is susceptible to change the buffer geometry.
12772  * Therefore, at load time, all checks on pointers previously done by the
12773  * verifier are invalidated and must be performed again, if the kfunc is used
12774  * in combination with direct packet access.
12775  *
12776  * Return:
12777  * * %0         - success
12778  * * %-EINVAL   - invalid len
12779  */
12780 __bpf_kfunc int bpf_xdp_pull_data(struct xdp_md *x, u32 len)
12781 {
12782 	struct xdp_buff *xdp = (struct xdp_buff *)x;
12783 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
12784 	int i, delta, shift, headroom, tailroom, n_frags_free = 0;
12785 	void *data_hard_end = xdp_data_hard_end(xdp);
12786 	int data_len = xdp->data_end - xdp->data;
12787 	void *start;
12788 
12789 	if (len <= data_len)
12790 		return 0;
12791 
12792 	if (unlikely(len > xdp_get_buff_len(xdp)))
12793 		return -EINVAL;
12794 
12795 	start = xdp_data_meta_unsupported(xdp) ? xdp->data : xdp->data_meta;
12796 
12797 	headroom = start - xdp->data_hard_start - sizeof(struct xdp_frame);
12798 	tailroom = data_hard_end - xdp->data_end;
12799 
12800 	delta = len - data_len;
12801 	if (unlikely(delta > tailroom + headroom))
12802 		return -EINVAL;
12803 
12804 	shift = delta - tailroom;
12805 	if (shift > 0) {
12806 		memmove(start - shift, start, xdp->data_end - start);
12807 
12808 		xdp->data_meta -= shift;
12809 		xdp->data -= shift;
12810 		xdp->data_end -= shift;
12811 	}
12812 
12813 	for (i = 0; i < sinfo->nr_frags && delta; i++) {
12814 		skb_frag_t *frag = &sinfo->frags[i];
12815 		u32 shrink = min_t(u32, delta, skb_frag_size(frag));
12816 
12817 		memcpy(xdp->data_end, skb_frag_address(frag), shrink);
12818 
12819 		xdp->data_end += shrink;
12820 		sinfo->xdp_frags_size -= shrink;
12821 		delta -= shrink;
12822 		if (bpf_xdp_shrink_data(xdp, frag, shrink, false))
12823 			n_frags_free++;
12824 	}
12825 
12826 	if (unlikely(n_frags_free)) {
12827 		memmove(sinfo->frags, sinfo->frags + n_frags_free,
12828 			(sinfo->nr_frags - n_frags_free) * sizeof(skb_frag_t));
12829 
12830 		sinfo->nr_frags -= n_frags_free;
12831 
12832 		if (!sinfo->nr_frags) {
12833 			xdp_buff_clear_frags_flag(xdp);
12834 			xdp_buff_clear_frag_pfmemalloc(xdp);
12835 		}
12836 	}
12837 
12838 	return 0;
12839 }
12840 
12841 /**
12842  * bpf_icmp_send - Send an ICMP control message
12843  * @skb_ctx: Packet that triggered the control message
12844  * @type: ICMP type (only ICMP_DEST_UNREACH/ICMPV6_DEST_UNREACH supported)
12845  * @code: ICMP code (0-15 except ICMP_FRAG_NEEDED for IPv4, 0-6 for IPv6)
12846  *
12847  * Sends an ICMP control message in response to the packet. The original packet
12848  * is cloned before sending the ICMP message, so the BPF program can still let
12849  * the packet pass if desired.
12850  *
12851  * Currently only ICMP_DEST_UNREACH (IPv4) and ICMPV6_DEST_UNREACH (IPv6) are
12852  * supported.
12853  *
12854  * Return: 0 on success (send attempt), negative error code on failure:
12855  *         -EBUSY: Recursion detected
12856  *         -EPROTONOSUPPORT: Non-IP protocol
12857  *         -EOPNOTSUPP: Unsupported ICMP type
12858  *         -EINVAL: Invalid code parameter
12859  *         -ENETUNREACH: No usable route/dst for the ICMP reply
12860  *         -ENOMEM: Memory allocation failed
12861  */
12862 __bpf_kfunc int bpf_icmp_send(struct __sk_buff *skb_ctx, int type, int code)
12863 {
12864 	struct sk_buff *skb = (struct sk_buff *)skb_ctx;
12865 	struct sock *sk;
12866 
12867 	sk = skb_to_full_sk(skb);
12868 	if (sk && sk->sk_kern_sock &&
12869 	    (sk->sk_protocol == IPPROTO_ICMP || sk->sk_protocol == IPPROTO_ICMPV6))
12870 		return -EBUSY;
12871 
12872 	if (!skb_valid_dst(skb))
12873 		return -ENETUNREACH;
12874 
12875 	switch (skb->protocol) {
12876 #if IS_ENABLED(CONFIG_INET)
12877 	case htons(ETH_P_IP): {
12878 		struct sk_buff *nskb;
12879 
12880 		if (type != ICMP_DEST_UNREACH)
12881 			return -EOPNOTSUPP;
12882 		if (code < 0 || code > NR_ICMP_UNREACH ||
12883 		    code == ICMP_FRAG_NEEDED) /* needs a valid next-hop MTU */
12884 			return -EINVAL;
12885 
12886 		nskb = skb_clone(skb, GFP_ATOMIC);
12887 		if (!nskb)
12888 			return -ENOMEM;
12889 
12890 		memset(IPCB(nskb), 0, sizeof(*IPCB(nskb)));
12891 		icmp_send(nskb, type, code, 0);
12892 		consume_skb(nskb);
12893 		break;
12894 	}
12895 #endif
12896 #if IS_ENABLED(CONFIG_IPV6)
12897 	case htons(ETH_P_IPV6): {
12898 		struct sk_buff *nskb;
12899 
12900 		if (type != ICMPV6_DEST_UNREACH)
12901 			return -EOPNOTSUPP;
12902 		if (code < 0 || code > ICMPV6_REJECT_ROUTE)
12903 			return -EINVAL;
12904 
12905 		nskb = skb_clone(skb, GFP_ATOMIC);
12906 		if (!nskb)
12907 			return -ENOMEM;
12908 
12909 		memset(IP6CB(nskb), 0, sizeof(*IP6CB(nskb)));
12910 		icmpv6_send(nskb, type, code, 0);
12911 		consume_skb(nskb);
12912 		break;
12913 	}
12914 #endif
12915 	default:
12916 		return -EPROTONOSUPPORT;
12917 	}
12918 
12919 	return 0;
12920 }
12921 
12922 __bpf_kfunc_end_defs();
12923 
12924 int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
12925 			       struct bpf_dynptr *ptr__uninit)
12926 {
12927 	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
12928 	int err;
12929 
12930 	err = bpf_dynptr_from_skb(skb, flags, ptr__uninit);
12931 	if (err)
12932 		return err;
12933 
12934 	bpf_dynptr_set_rdonly(ptr);
12935 
12936 	return 0;
12937 }
12938 
12939 BTF_KFUNCS_START(bpf_kfunc_check_set_skb)
12940 BTF_ID_FLAGS(func, bpf_dynptr_from_skb)
12941 BTF_KFUNCS_END(bpf_kfunc_check_set_skb)
12942 
12943 BTF_KFUNCS_START(bpf_kfunc_check_set_skb_meta)
12944 BTF_ID_FLAGS(func, bpf_dynptr_from_skb_meta)
12945 BTF_KFUNCS_END(bpf_kfunc_check_set_skb_meta)
12946 
12947 BTF_KFUNCS_START(bpf_kfunc_check_set_xdp)
12948 BTF_ID_FLAGS(func, bpf_dynptr_from_xdp)
12949 BTF_ID_FLAGS(func, bpf_xdp_pull_data)
12950 BTF_KFUNCS_END(bpf_kfunc_check_set_xdp)
12951 
12952 BTF_KFUNCS_START(bpf_kfunc_check_set_sock_addr)
12953 BTF_ID_FLAGS(func, bpf_sock_addr_set_sun_path)
12954 BTF_KFUNCS_END(bpf_kfunc_check_set_sock_addr)
12955 
12956 BTF_KFUNCS_START(bpf_kfunc_check_set_tcp_reqsk)
12957 BTF_ID_FLAGS(func, bpf_sk_assign_tcp_reqsk)
12958 BTF_KFUNCS_END(bpf_kfunc_check_set_tcp_reqsk)
12959 
12960 BTF_KFUNCS_START(bpf_kfunc_check_set_sock_ops)
12961 BTF_ID_FLAGS(func, bpf_sock_ops_enable_tx_tstamp)
12962 BTF_KFUNCS_END(bpf_kfunc_check_set_sock_ops)
12963 
12964 BTF_KFUNCS_START(bpf_kfunc_check_set_icmp_send)
12965 BTF_ID_FLAGS(func, bpf_icmp_send)
12966 BTF_KFUNCS_END(bpf_kfunc_check_set_icmp_send)
12967 
12968 static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
12969 	.owner = THIS_MODULE,
12970 	.set = &bpf_kfunc_check_set_skb,
12971 };
12972 
12973 static const struct btf_kfunc_id_set bpf_kfunc_set_skb_meta = {
12974 	.owner = THIS_MODULE,
12975 	.set = &bpf_kfunc_check_set_skb_meta,
12976 };
12977 
12978 static const struct btf_kfunc_id_set bpf_kfunc_set_xdp = {
12979 	.owner = THIS_MODULE,
12980 	.set = &bpf_kfunc_check_set_xdp,
12981 };
12982 
12983 static const struct btf_kfunc_id_set bpf_kfunc_set_sock_addr = {
12984 	.owner = THIS_MODULE,
12985 	.set = &bpf_kfunc_check_set_sock_addr,
12986 };
12987 
12988 static const struct btf_kfunc_id_set bpf_kfunc_set_tcp_reqsk = {
12989 	.owner = THIS_MODULE,
12990 	.set = &bpf_kfunc_check_set_tcp_reqsk,
12991 };
12992 
12993 static const struct btf_kfunc_id_set bpf_kfunc_set_sock_ops = {
12994 	.owner = THIS_MODULE,
12995 	.set = &bpf_kfunc_check_set_sock_ops,
12996 };
12997 
12998 static const struct btf_kfunc_id_set bpf_kfunc_set_icmp_send = {
12999 	.owner = THIS_MODULE,
13000 	.set = &bpf_kfunc_check_set_icmp_send,
13001 };
13002 
13003 static int __init bpf_kfunc_init(void)
13004 {
13005 	int ret;
13006 
13007 	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb);
13008 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb);
13009 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SK_SKB, &bpf_kfunc_set_skb);
13010 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCKET_FILTER, &bpf_kfunc_set_skb);
13011 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_skb);
13012 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
13013 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
13014 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
13015 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
13016 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb);
13017 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb);
13018 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_kfunc_set_skb);
13019 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb_meta);
13020 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb_meta);
13021 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
13022 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
13023 					       &bpf_kfunc_set_sock_addr);
13024 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_tcp_reqsk);
13025 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_icmp_send);
13026 	return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCK_OPS, &bpf_kfunc_set_sock_ops);
13027 }
13028 late_initcall(bpf_kfunc_init);
13029 
13030 __bpf_kfunc_start_defs();
13031 
13032 /* bpf_sock_destroy: Destroy the given socket with ECONNABORTED error code.
13033  *
13034  * The function expects a non-NULL pointer to a socket, and invokes the
13035  * protocol specific socket destroy handlers.
13036  *
13037  * The helper can only be called from BPF contexts that have acquired the socket
13038  * locks.
13039  *
13040  * Parameters:
13041  * @sock: Pointer to socket to be destroyed
13042  *
13043  * Return:
13044  * On error, may return EOPNOTSUPP, or whatever the protocol specific
13045  * destroy handler returns.
13046  * EOPNOTSUPP if protocol specific destroy handler is not supported.
13047  * 0 otherwise
13048  */
13049 __bpf_kfunc int bpf_sock_destroy(struct sock_common *sock)
13050 {
13051 	struct sock *sk = (struct sock *)sock;
13052 
13053 	/* The locking semantics that allow for synchronous execution of the
13054 	 * destroy handlers are only supported for TCP and UDP.
13055 	 * Supporting protocols will need to acquire sock lock in the BPF context
13056 	 * prior to invoking this kfunc.
13057 	 */
13058 	if (!sk->sk_prot->diag_destroy)
13059 		return -EOPNOTSUPP;
13060 
13061 	if (sk_fullsock(sk) &&
13062 	    sk->sk_protocol != IPPROTO_TCP &&
13063 	    sk->sk_protocol != IPPROTO_UDP)
13064 		return -EOPNOTSUPP;
13065 
13066 	return sk->sk_prot->diag_destroy(sk, ECONNABORTED);
13067 }
13068 
13069 __bpf_kfunc_end_defs();
13070 
13071 BTF_KFUNCS_START(bpf_sk_iter_kfunc_ids)
13072 BTF_ID_FLAGS(func, bpf_sock_destroy)
13073 BTF_KFUNCS_END(bpf_sk_iter_kfunc_ids)
13074 
13075 BTF_SET_START(bpf_sk_iter_locked_ids)
13076 #if defined(CONFIG_INET) && defined(CONFIG_PROC_FS)
13077 BTF_ID(func, bpf_iter_tcp)
13078 BTF_ID(func, bpf_iter_udp)
13079 #endif
13080 BTF_SET_END(bpf_sk_iter_locked_ids)
13081 
13082 static int tracing_iter_filter(const struct bpf_prog *prog, u32 kfunc_id)
13083 {
13084 	if (btf_id_set8_contains(&bpf_sk_iter_kfunc_ids, kfunc_id) &&
13085 	    (prog->expected_attach_type != BPF_TRACE_ITER ||
13086 	     !btf_id_set_contains(&bpf_sk_iter_locked_ids, prog->aux->attach_btf_id)))
13087 		return -EACCES;
13088 
13089 	return 0;
13090 }
13091 
13092 static const struct btf_kfunc_id_set bpf_sk_iter_kfunc_set = {
13093 	.owner = THIS_MODULE,
13094 	.set   = &bpf_sk_iter_kfunc_ids,
13095 	.filter = tracing_iter_filter,
13096 };
13097 
13098 static int init_subsystem(void)
13099 {
13100 	return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_sk_iter_kfunc_set);
13101 }
13102 late_initcall(init_subsystem);
13103