1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */
3
4 #ifndef _LINUX_SKMSG_H
5 #define _LINUX_SKMSG_H
6
7 #include <linux/bpf.h>
8 #include <linux/filter.h>
9 #include <linux/scatterlist.h>
10 #include <linux/skbuff.h>
11
12 #include <net/sock.h>
13 #include <net/tcp.h>
14 #include <net/strparser.h>
15
16 #define MAX_MSG_FRAGS MAX_SKB_FRAGS
17 #define NR_MSG_FRAG_IDS (MAX_MSG_FRAGS + 1)
18
19 enum __sk_action {
20 __SK_DROP = 0,
21 __SK_PASS,
22 __SK_REDIRECT,
23 __SK_NONE,
24 };
25
26 struct sk_msg_sg {
27 u32 start;
28 u32 curr;
29 u32 end;
30 u32 size;
31 u32 copybreak;
32 DECLARE_BITMAP(copy, MAX_MSG_FRAGS + 2);
33 /* The extra two elements:
34 * 1) used for chaining the front and sections when the list becomes
35 * partitioned (e.g. end < start). The crypto APIs require the
36 * chaining;
37 * 2) to chain tailer SG entries after the message.
38 */
39 struct scatterlist data[MAX_MSG_FRAGS + 2];
40 };
41
42 /* UAPI in filter.c depends on struct sk_msg_sg being first element. */
43 struct sk_msg {
44 struct sk_msg_sg sg;
45 void *data;
46 void *data_end;
47 u32 apply_bytes;
48 u32 cork_bytes;
49 u32 flags;
50 struct sk_buff *skb;
51 struct sock *sk_redir;
52 struct sock *sk;
53 struct list_head list;
54 };
55
56 struct sk_psock_progs {
57 struct bpf_prog *msg_parser;
58 struct bpf_prog *stream_parser;
59 struct bpf_prog *stream_verdict;
60 struct bpf_prog *skb_verdict;
61 struct bpf_link *msg_parser_link;
62 struct bpf_link *stream_parser_link;
63 struct bpf_link *stream_verdict_link;
64 struct bpf_link *skb_verdict_link;
65 };
66
67 enum sk_psock_state_bits {
68 SK_PSOCK_TX_ENABLED,
69 SK_PSOCK_RX_STRP_ENABLED,
70 };
71
72 struct sk_psock_link {
73 struct list_head list;
74 struct bpf_map *map;
75 void *link_raw;
76 };
77
78 struct sk_psock_work_state {
79 u32 len;
80 u32 off;
81 };
82
83 struct sk_psock {
84 struct sock *sk;
85 struct sock *sk_redir;
86 u32 apply_bytes;
87 u32 cork_bytes;
88 u32 eval;
89 bool redir_ingress; /* undefined if sk_redir is null */
90 struct sk_msg *cork;
91 struct sk_psock_progs progs;
92 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
93 struct strparser strp;
94 u32 copied_seq;
95 u32 ingress_bytes;
96 #endif
97 struct sk_buff_head ingress_skb;
98 struct list_head ingress_msg;
99 spinlock_t ingress_lock;
100 /** @msg_tot_len: Total bytes queued in ingress_msg list. */
101 u32 msg_tot_len;
102 unsigned long state;
103 struct list_head link;
104 spinlock_t link_lock;
105 refcount_t refcnt;
106 void (*saved_unhash)(struct sock *sk);
107 void (*saved_destroy)(struct sock *sk);
108 void (*saved_close)(struct sock *sk, long timeout);
109 void (*saved_write_space)(struct sock *sk);
110 void (*saved_data_ready)(struct sock *sk);
111 /* psock_update_sk_prot may be called with restore=false many times
112 * so the handler must be safe for this case. It will be called
113 * exactly once with restore=true when the psock is being destroyed
114 * and psock refcnt is zero, but before an RCU grace period.
115 */
116 int (*psock_update_sk_prot)(struct sock *sk, struct sk_psock *psock,
117 bool restore);
118 struct proto *sk_proto;
119 struct mutex work_mutex;
120 struct sk_psock_work_state work_state;
121 struct delayed_work work;
122 struct sock *sk_pair;
123 struct rcu_work rwork;
124 };
125
126 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
127 int elem_first_coalesce);
128 int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src,
129 u32 off, u32 len);
130 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
131 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
132 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
133 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
134 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
135 u32 bytes);
136
137 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
138 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
139
140 int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
141 struct sk_msg *msg, u32 bytes);
142 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from,
143 struct sk_msg *msg, u32 bytes);
144 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
145 int len, int flags);
146 int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
147 int len, int flags, int *copied_from_self);
148 bool sk_msg_is_readable(struct sock *sk);
149
sk_msg_check_to_free(struct sk_msg * msg,u32 i,u32 bytes)150 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)
151 {
152 WARN_ON(i == msg->sg.end && bytes);
153 }
154
sk_msg_apply_bytes(struct sk_psock * psock,u32 bytes)155 static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes)
156 {
157 if (psock->apply_bytes) {
158 if (psock->apply_bytes < bytes)
159 psock->apply_bytes = 0;
160 else
161 psock->apply_bytes -= bytes;
162 }
163 }
164
sk_msg_iter_dist(u32 start,u32 end)165 static inline u32 sk_msg_iter_dist(u32 start, u32 end)
166 {
167 return end >= start ? end - start : end + (NR_MSG_FRAG_IDS - start);
168 }
169
170 #define sk_msg_iter_var_prev(var) \
171 do { \
172 if (var == 0) \
173 var = NR_MSG_FRAG_IDS - 1; \
174 else \
175 var--; \
176 } while (0)
177
178 #define sk_msg_iter_var_next(var) \
179 do { \
180 var++; \
181 if (var == NR_MSG_FRAG_IDS) \
182 var = 0; \
183 } while (0)
184
185 #define sk_msg_iter_prev(msg, which) \
186 sk_msg_iter_var_prev(msg->sg.which)
187
188 #define sk_msg_iter_next(msg, which) \
189 sk_msg_iter_var_next(msg->sg.which)
190
sk_msg_init(struct sk_msg * msg)191 static inline void sk_msg_init(struct sk_msg *msg)
192 {
193 BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);
194 memset(msg, 0, sizeof(*msg));
195 sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);
196 }
197
sk_msg_xfer(struct sk_msg * dst,struct sk_msg * src,int which,u32 size)198 static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src,
199 int which, u32 size)
200 {
201 dst->sg.data[which] = src->sg.data[which];
202 dst->sg.data[which].length = size;
203 dst->sg.size += size;
204 src->sg.size -= size;
205 src->sg.data[which].length -= size;
206 src->sg.data[which].offset += size;
207 }
208
sk_msg_xfer_full(struct sk_msg * dst,struct sk_msg * src)209 static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src)
210 {
211 memcpy(dst, src, sizeof(*src));
212 sk_msg_init(src);
213 }
214
sk_msg_full(const struct sk_msg * msg)215 static inline bool sk_msg_full(const struct sk_msg *msg)
216 {
217 return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;
218 }
219
sk_msg_elem_used(const struct sk_msg * msg)220 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)
221 {
222 return sk_msg_iter_dist(msg->sg.start, msg->sg.end);
223 }
224
sk_msg_elem(struct sk_msg * msg,int which)225 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)
226 {
227 return &msg->sg.data[which];
228 }
229
sk_msg_elem_cpy(struct sk_msg * msg,int which)230 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)
231 {
232 return msg->sg.data[which];
233 }
234
sk_msg_page(struct sk_msg * msg,int which)235 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)
236 {
237 return sg_page(sk_msg_elem(msg, which));
238 }
239
sk_msg_to_ingress(const struct sk_msg * msg)240 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)
241 {
242 return msg->flags & BPF_F_INGRESS;
243 }
244
sk_msg_compute_data_pointers(struct sk_msg * msg)245 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)
246 {
247 struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);
248
249 if (test_bit(msg->sg.start, msg->sg.copy)) {
250 msg->data = NULL;
251 msg->data_end = NULL;
252 } else {
253 msg->data = sg_virt(sge);
254 msg->data_end = msg->data + sge->length;
255 }
256 }
257
sk_msg_page_add(struct sk_msg * msg,struct page * page,u32 len,u32 offset)258 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,
259 u32 len, u32 offset)
260 {
261 struct scatterlist *sge;
262
263 get_page(page);
264 sge = sk_msg_elem(msg, msg->sg.end);
265 sg_set_page(sge, page, len, offset);
266 sg_unmark_end(sge);
267
268 __set_bit(msg->sg.end, msg->sg.copy);
269 msg->sg.size += len;
270 sk_msg_iter_next(msg, end);
271 }
272
sk_msg_sg_copy(struct sk_msg * msg,u32 i,bool copy_state)273 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)
274 {
275 do {
276 if (copy_state)
277 __set_bit(i, msg->sg.copy);
278 else
279 __clear_bit(i, msg->sg.copy);
280 sk_msg_iter_var_next(i);
281 if (i == msg->sg.end)
282 break;
283 } while (1);
284 }
285
sk_msg_sg_copy_set(struct sk_msg * msg,u32 start)286 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)
287 {
288 sk_msg_sg_copy(msg, start, true);
289 }
290
sk_msg_sg_copy_clear(struct sk_msg * msg,u32 start)291 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)
292 {
293 sk_msg_sg_copy(msg, start, false);
294 }
295
sk_psock(const struct sock * sk)296 static inline struct sk_psock *sk_psock(const struct sock *sk)
297 {
298 return __rcu_dereference_sk_user_data_with_flags(sk,
299 SK_USER_DATA_PSOCK);
300 }
301
sk_psock_set_state(struct sk_psock * psock,enum sk_psock_state_bits bit)302 static inline void sk_psock_set_state(struct sk_psock *psock,
303 enum sk_psock_state_bits bit)
304 {
305 set_bit(bit, &psock->state);
306 }
307
sk_psock_clear_state(struct sk_psock * psock,enum sk_psock_state_bits bit)308 static inline void sk_psock_clear_state(struct sk_psock *psock,
309 enum sk_psock_state_bits bit)
310 {
311 clear_bit(bit, &psock->state);
312 }
313
sk_psock_test_state(const struct sk_psock * psock,enum sk_psock_state_bits bit)314 static inline bool sk_psock_test_state(const struct sk_psock *psock,
315 enum sk_psock_state_bits bit)
316 {
317 return test_bit(bit, &psock->state);
318 }
319
sock_drop(struct sock * sk,struct sk_buff * skb)320 static inline void sock_drop(struct sock *sk, struct sk_buff *skb)
321 {
322 sk_drops_skbadd(sk, skb);
323 kfree_skb(skb);
324 }
325
sk_psock_get_msg_len_nolock(struct sk_psock * psock)326 static inline u32 sk_psock_get_msg_len_nolock(struct sk_psock *psock)
327 {
328 /* Used by ioctl to read msg_tot_len only; lock-free for performance */
329 return READ_ONCE(psock->msg_tot_len);
330 }
331
sk_psock_msg_len_add_locked(struct sk_psock * psock,int diff)332 static inline void sk_psock_msg_len_add_locked(struct sk_psock *psock, int diff)
333 {
334 /* Use WRITE_ONCE to ensure correct read in sk_psock_get_msg_len_nolock().
335 * ingress_lock should be held to prevent concurrent updates to msg_tot_len
336 */
337 WRITE_ONCE(psock->msg_tot_len, psock->msg_tot_len + diff);
338 }
339
sk_psock_msg_len_add(struct sk_psock * psock,int diff)340 static inline void sk_psock_msg_len_add(struct sk_psock *psock, int diff)
341 {
342 spin_lock_bh(&psock->ingress_lock);
343 sk_psock_msg_len_add_locked(psock, diff);
344 spin_unlock_bh(&psock->ingress_lock);
345 }
346
sk_psock_queue_msg(struct sk_psock * psock,struct sk_msg * msg)347 static inline bool sk_psock_queue_msg(struct sk_psock *psock,
348 struct sk_msg *msg)
349 {
350 bool ret;
351
352 spin_lock_bh(&psock->ingress_lock);
353 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
354 list_add_tail(&msg->list, &psock->ingress_msg);
355 sk_psock_msg_len_add_locked(psock, msg->sg.size);
356 ret = true;
357 } else {
358 sk_msg_free(psock->sk, msg);
359 kfree(msg);
360 ret = false;
361 }
362 spin_unlock_bh(&psock->ingress_lock);
363 return ret;
364 }
365
sk_psock_dequeue_msg(struct sk_psock * psock)366 static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock)
367 {
368 struct sk_msg *msg;
369
370 spin_lock_bh(&psock->ingress_lock);
371 msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
372 if (msg) {
373 list_del(&msg->list);
374 sk_psock_msg_len_add_locked(psock, -msg->sg.size);
375 }
376 spin_unlock_bh(&psock->ingress_lock);
377 return msg;
378 }
379
sk_psock_peek_msg_locked(struct sk_psock * psock)380 static inline struct sk_msg *sk_psock_peek_msg_locked(struct sk_psock *psock)
381 {
382 return list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);
383 }
384
sk_psock_peek_msg(struct sk_psock * psock)385 static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock)
386 {
387 struct sk_msg *msg;
388
389 spin_lock_bh(&psock->ingress_lock);
390 msg = sk_psock_peek_msg_locked(psock);
391 spin_unlock_bh(&psock->ingress_lock);
392 return msg;
393 }
394
sk_psock_next_msg(struct sk_psock * psock,struct sk_msg * msg)395 static inline struct sk_msg *sk_psock_next_msg(struct sk_psock *psock,
396 struct sk_msg *msg)
397 {
398 struct sk_msg *ret;
399
400 spin_lock_bh(&psock->ingress_lock);
401 if (list_is_last(&msg->list, &psock->ingress_msg))
402 ret = NULL;
403 else
404 ret = list_next_entry(msg, list);
405 spin_unlock_bh(&psock->ingress_lock);
406 return ret;
407 }
408
sk_psock_queue_empty(const struct sk_psock * psock)409 static inline bool sk_psock_queue_empty(const struct sk_psock *psock)
410 {
411 return psock ? list_empty(&psock->ingress_msg) : true;
412 }
413
kfree_sk_msg(struct sk_msg * msg)414 static inline void kfree_sk_msg(struct sk_msg *msg)
415 {
416 if (msg->skb)
417 consume_skb(msg->skb);
418 kfree(msg);
419 }
420
sk_psock_report_error(struct sk_psock * psock,int err)421 static inline void sk_psock_report_error(struct sk_psock *psock, int err)
422 {
423 struct sock *sk = psock->sk;
424
425 sk->sk_err = err;
426 sk_error_report(sk);
427 }
428
429 struct sk_psock *sk_psock_init(struct sock *sk, int node);
430 void sk_psock_stop(struct sk_psock *psock);
431
432 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER)
433 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock);
434 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock);
435 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock);
436 #else
sk_psock_init_strp(struct sock * sk,struct sk_psock * psock)437 static inline int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock)
438 {
439 return -EOPNOTSUPP;
440 }
441
sk_psock_start_strp(struct sock * sk,struct sk_psock * psock)442 static inline void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
443 {
444 }
445
sk_psock_stop_strp(struct sock * sk,struct sk_psock * psock)446 static inline void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock)
447 {
448 }
449 #endif
450
451 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock);
452 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock);
453
454 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock,
455 struct sk_msg *msg);
456
457 /*
458 * This specialized allocator has to be a macro for its allocations to be
459 * accounted separately (to have a separate alloc_tag). The typecast is
460 * intentional to enforce typesafety.
461 */
462 #define sk_psock_init_link() \
463 kzalloc_obj(struct sk_psock_link, GFP_ATOMIC | __GFP_NOWARN)
464
sk_psock_free_link(struct sk_psock_link * link)465 static inline void sk_psock_free_link(struct sk_psock_link *link)
466 {
467 kfree(link);
468 }
469
470 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock);
471
sk_psock_cork_free(struct sk_psock * psock)472 static inline void sk_psock_cork_free(struct sk_psock *psock)
473 {
474 if (psock->cork) {
475 sk_msg_free(psock->sk, psock->cork);
476 kfree(psock->cork);
477 psock->cork = NULL;
478 }
479 }
480
sk_psock_restore_proto(struct sock * sk,struct sk_psock * psock)481 static inline void sk_psock_restore_proto(struct sock *sk,
482 struct sk_psock *psock)
483 {
484 if (psock->psock_update_sk_prot)
485 psock->psock_update_sk_prot(sk, psock, true);
486 }
487
sk_psock_get(struct sock * sk)488 static inline struct sk_psock *sk_psock_get(struct sock *sk)
489 {
490 struct sk_psock *psock;
491
492 rcu_read_lock();
493 psock = sk_psock(sk);
494 if (psock && !refcount_inc_not_zero(&psock->refcnt))
495 psock = NULL;
496 rcu_read_unlock();
497 return psock;
498 }
499
500 void sk_psock_drop(struct sock *sk, struct sk_psock *psock);
501
sk_psock_put(struct sock * sk,struct sk_psock * psock)502 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock)
503 {
504 if (refcount_dec_and_test(&psock->refcnt))
505 sk_psock_drop(sk, psock);
506 }
507
sk_psock_data_ready(struct sock * sk,struct sk_psock * psock)508 static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock)
509 {
510 read_lock_bh(&sk->sk_callback_lock);
511 if (psock->saved_data_ready)
512 psock->saved_data_ready(sk);
513 else
514 sk->sk_data_ready(sk);
515 read_unlock_bh(&sk->sk_callback_lock);
516 }
517
psock_set_prog(struct bpf_prog ** pprog,struct bpf_prog * prog)518 static inline void psock_set_prog(struct bpf_prog **pprog,
519 struct bpf_prog *prog)
520 {
521 prog = xchg(pprog, prog);
522 if (prog)
523 bpf_prog_put(prog);
524 }
525
psock_replace_prog(struct bpf_prog ** pprog,struct bpf_prog * prog,struct bpf_prog * old)526 static inline int psock_replace_prog(struct bpf_prog **pprog,
527 struct bpf_prog *prog,
528 struct bpf_prog *old)
529 {
530 if (cmpxchg(pprog, old, prog) != old)
531 return -ENOENT;
532
533 if (old)
534 bpf_prog_put(old);
535
536 return 0;
537 }
538
psock_progs_drop(struct sk_psock_progs * progs)539 static inline void psock_progs_drop(struct sk_psock_progs *progs)
540 {
541 psock_set_prog(&progs->msg_parser, NULL);
542 psock_set_prog(&progs->stream_parser, NULL);
543 psock_set_prog(&progs->stream_verdict, NULL);
544 psock_set_prog(&progs->skb_verdict, NULL);
545 }
546
547 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb);
548
sk_psock_strp_enabled(struct sk_psock * psock)549 static inline bool sk_psock_strp_enabled(struct sk_psock *psock)
550 {
551 if (!psock)
552 return false;
553 return !!psock->saved_data_ready;
554 }
555
556 /* for tcp only, sk is locked */
sk_psock_msg_inq(struct sock * sk)557 static inline ssize_t sk_psock_msg_inq(struct sock *sk)
558 {
559 struct sk_psock *psock;
560 ssize_t inq = 0;
561
562 psock = sk_psock_get(sk);
563 if (likely(psock)) {
564 inq = sk_psock_get_msg_len_nolock(psock);
565 sk_psock_put(sk, psock);
566 }
567 return inq;
568 }
569
570 /* for udp only, sk is not locked */
sk_msg_first_len(struct sock * sk)571 static inline ssize_t sk_msg_first_len(struct sock *sk)
572 {
573 struct sk_psock *psock;
574 struct sk_msg *msg;
575 ssize_t inq = 0;
576
577 psock = sk_psock_get(sk);
578 if (likely(psock)) {
579 spin_lock_bh(&psock->ingress_lock);
580 msg = sk_psock_peek_msg_locked(psock);
581 if (msg)
582 inq = msg->sg.size;
583 spin_unlock_bh(&psock->ingress_lock);
584 sk_psock_put(sk, psock);
585 }
586 return inq;
587 }
588
589 #if IS_ENABLED(CONFIG_NET_SOCK_MSG)
590
591 #define BPF_F_STRPARSER (1UL << 1)
592
593 /* We only have two bits so far. */
594 #define BPF_F_PTR_MASK ~(BPF_F_INGRESS | BPF_F_STRPARSER)
595
skb_bpf_strparser(const struct sk_buff * skb)596 static inline bool skb_bpf_strparser(const struct sk_buff *skb)
597 {
598 unsigned long sk_redir = skb->_sk_redir;
599
600 return sk_redir & BPF_F_STRPARSER;
601 }
602
skb_bpf_set_strparser(struct sk_buff * skb)603 static inline void skb_bpf_set_strparser(struct sk_buff *skb)
604 {
605 skb->_sk_redir |= BPF_F_STRPARSER;
606 }
607
skb_bpf_ingress(const struct sk_buff * skb)608 static inline bool skb_bpf_ingress(const struct sk_buff *skb)
609 {
610 unsigned long sk_redir = skb->_sk_redir;
611
612 return sk_redir & BPF_F_INGRESS;
613 }
614
skb_bpf_set_ingress(struct sk_buff * skb)615 static inline void skb_bpf_set_ingress(struct sk_buff *skb)
616 {
617 skb->_sk_redir |= BPF_F_INGRESS;
618 }
619
skb_bpf_set_redir(struct sk_buff * skb,struct sock * sk_redir,bool ingress)620 static inline void skb_bpf_set_redir(struct sk_buff *skb, struct sock *sk_redir,
621 bool ingress)
622 {
623 skb->_sk_redir = (unsigned long)sk_redir;
624 if (ingress)
625 skb->_sk_redir |= BPF_F_INGRESS;
626 }
627
skb_bpf_redirect_fetch(const struct sk_buff * skb)628 static inline struct sock *skb_bpf_redirect_fetch(const struct sk_buff *skb)
629 {
630 unsigned long sk_redir = skb->_sk_redir;
631
632 return (struct sock *)(sk_redir & BPF_F_PTR_MASK);
633 }
634
skb_bpf_redirect_clear(struct sk_buff * skb)635 static inline void skb_bpf_redirect_clear(struct sk_buff *skb)
636 {
637 skb->_sk_redir = 0;
638 }
639 #endif /* CONFIG_NET_SOCK_MSG */
640 #endif /* _LINUX_SKMSG_H */
641