1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ 3 4 #ifndef _LINUX_SKMSG_H 5 #define _LINUX_SKMSG_H 6 7 #include <linux/bitops.h> 8 #include <linux/bpf.h> 9 #include <linux/filter.h> 10 #include <linux/scatterlist.h> 11 #include <linux/skbuff.h> 12 13 #include <net/sock.h> 14 #include <net/tcp.h> 15 #include <net/strparser.h> 16 17 #define MAX_MSG_FRAGS MAX_SKB_FRAGS 18 #define NR_MSG_FRAG_IDS (MAX_MSG_FRAGS + 1) 19 20 enum __sk_action { 21 __SK_DROP = 0, 22 __SK_PASS, 23 __SK_REDIRECT, 24 __SK_NONE, 25 }; 26 27 struct sk_msg_sg { 28 u32 start; 29 u32 curr; 30 u32 end; 31 u32 size; 32 u32 copybreak; 33 DECLARE_BITMAP(copy, MAX_MSG_FRAGS + 2); 34 /* The extra two elements: 35 * 1) used for chaining the front and sections when the list becomes 36 * partitioned (e.g. end < start). The crypto APIs require the 37 * chaining; 38 * 2) to chain tailer SG entries after the message. 39 */ 40 struct scatterlist data[MAX_MSG_FRAGS + 2]; 41 }; 42 43 /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ 44 struct sk_msg { 45 struct sk_msg_sg sg; 46 void *data; 47 void *data_end; 48 u32 apply_bytes; 49 u32 cork_bytes; 50 u32 flags; 51 struct sk_buff *skb; 52 struct sock *sk_redir; 53 struct sock *sk; 54 struct list_head list; 55 }; 56 57 struct sk_psock_progs { 58 struct bpf_prog *msg_parser; 59 struct bpf_prog *stream_parser; 60 struct bpf_prog *stream_verdict; 61 struct bpf_prog *skb_verdict; 62 struct bpf_link *msg_parser_link; 63 struct bpf_link *stream_parser_link; 64 struct bpf_link *stream_verdict_link; 65 struct bpf_link *skb_verdict_link; 66 }; 67 68 enum sk_psock_state_bits { 69 SK_PSOCK_TX_ENABLED, 70 SK_PSOCK_RX_STRP_ENABLED, 71 }; 72 73 struct sk_psock_link { 74 struct list_head list; 75 struct bpf_map *map; 76 void *link_raw; 77 }; 78 79 struct sk_psock_work_state { 80 u32 len; 81 u32 off; 82 }; 83 84 struct sk_psock { 85 struct sock *sk; 86 struct sock *sk_redir; 87 u32 apply_bytes; 88 u32 cork_bytes; 89 u32 eval; 90 bool redir_ingress; /* undefined if sk_redir is null */ 91 struct sk_msg *cork; 92 struct sk_psock_progs progs; 93 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) 94 struct strparser strp; 95 u32 copied_seq; 96 u32 ingress_bytes; 97 #endif 98 struct sk_buff_head ingress_skb; 99 struct list_head ingress_msg; 100 spinlock_t ingress_lock; 101 /** @msg_tot_len: Total bytes queued in ingress_msg list. */ 102 u32 msg_tot_len; 103 unsigned long state; 104 struct list_head link; 105 spinlock_t link_lock; 106 refcount_t refcnt; 107 void (*saved_unhash)(struct sock *sk); 108 void (*saved_destroy)(struct sock *sk); 109 void (*saved_close)(struct sock *sk, long timeout); 110 void (*saved_write_space)(struct sock *sk); 111 void (*saved_data_ready)(struct sock *sk); 112 /* psock_update_sk_prot may be called with restore=false many times 113 * so the handler must be safe for this case. It will be called 114 * exactly once with restore=true when the psock is being destroyed 115 * and psock refcnt is zero, but before an RCU grace period. 116 */ 117 int (*psock_update_sk_prot)(struct sock *sk, struct sk_psock *psock, 118 bool restore); 119 struct proto *sk_proto; 120 struct mutex work_mutex; 121 struct sk_psock_work_state work_state; 122 struct delayed_work work; 123 struct sock *sk_pair; 124 struct rcu_work rwork; 125 }; 126 127 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, 128 int elem_first_coalesce); 129 int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, 130 u32 off, u32 len); 131 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len); 132 int sk_msg_free(struct sock *sk, struct sk_msg *msg); 133 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg); 134 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes); 135 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg, 136 u32 bytes); 137 138 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes); 139 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes); 140 141 int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, 142 struct sk_msg *msg, u32 bytes); 143 int sk_msg_memcopy_from_iter(struct sock *sk, struct iov_iter *from, 144 struct sk_msg *msg, u32 bytes); 145 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, 146 int len, int flags); 147 int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, 148 int len, int flags, int *copied_from_self); 149 bool sk_msg_is_readable(struct sock *sk); 150 151 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes) 152 { 153 WARN_ON(i == msg->sg.end && bytes); 154 } 155 156 static inline void sk_msg_apply_bytes(struct sk_psock *psock, u32 bytes) 157 { 158 if (psock->apply_bytes) { 159 if (psock->apply_bytes < bytes) 160 psock->apply_bytes = 0; 161 else 162 psock->apply_bytes -= bytes; 163 } 164 } 165 166 static inline u32 sk_msg_iter_dist(u32 start, u32 end) 167 { 168 return end >= start ? end - start : end + (NR_MSG_FRAG_IDS - start); 169 } 170 171 #define sk_msg_iter_var_prev(var) \ 172 do { \ 173 if (var == 0) \ 174 var = NR_MSG_FRAG_IDS - 1; \ 175 else \ 176 var--; \ 177 } while (0) 178 179 #define sk_msg_iter_var_next(var) \ 180 do { \ 181 var++; \ 182 if (var == NR_MSG_FRAG_IDS) \ 183 var = 0; \ 184 } while (0) 185 186 #define sk_msg_iter_prev(msg, which) \ 187 sk_msg_iter_var_prev(msg->sg.which) 188 189 #define sk_msg_iter_next(msg, which) \ 190 sk_msg_iter_var_next(msg->sg.which) 191 192 static inline void sk_msg_init(struct sk_msg *msg) 193 { 194 BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS); 195 memset(msg, 0, sizeof(*msg)); 196 sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS); 197 } 198 199 static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src, 200 int which, u32 size) 201 { 202 dst->sg.data[which] = src->sg.data[which]; 203 __assign_bit(which, dst->sg.copy, test_bit(which, src->sg.copy)); 204 dst->sg.data[which].length = size; 205 dst->sg.size += size; 206 src->sg.size -= size; 207 src->sg.data[which].length -= size; 208 src->sg.data[which].offset += size; 209 if (!src->sg.data[which].length) 210 __clear_bit(which, src->sg.copy); 211 } 212 213 static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src) 214 { 215 memcpy(dst, src, sizeof(*src)); 216 sk_msg_init(src); 217 } 218 219 static inline bool sk_msg_full(const struct sk_msg *msg) 220 { 221 return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS; 222 } 223 224 static inline u32 sk_msg_elem_used(const struct sk_msg *msg) 225 { 226 return sk_msg_iter_dist(msg->sg.start, msg->sg.end); 227 } 228 229 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which) 230 { 231 return &msg->sg.data[which]; 232 } 233 234 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which) 235 { 236 return msg->sg.data[which]; 237 } 238 239 static inline struct page *sk_msg_page(struct sk_msg *msg, int which) 240 { 241 return sg_page(sk_msg_elem(msg, which)); 242 } 243 244 static inline bool sk_msg_to_ingress(const struct sk_msg *msg) 245 { 246 return msg->flags & BPF_F_INGRESS; 247 } 248 249 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) 250 { 251 struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start); 252 253 if (test_bit(msg->sg.start, msg->sg.copy)) { 254 msg->data = NULL; 255 msg->data_end = NULL; 256 } else { 257 msg->data = sg_virt(sge); 258 msg->data_end = msg->data + sge->length; 259 } 260 } 261 262 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, 263 u32 len, u32 offset) 264 { 265 struct scatterlist *sge; 266 267 get_page(page); 268 sge = sk_msg_elem(msg, msg->sg.end); 269 sg_set_page(sge, page, len, offset); 270 sg_unmark_end(sge); 271 272 __set_bit(msg->sg.end, msg->sg.copy); 273 msg->sg.size += len; 274 sk_msg_iter_next(msg, end); 275 } 276 277 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) 278 { 279 do { 280 __assign_bit(i, msg->sg.copy, copy_state); 281 sk_msg_iter_var_next(i); 282 if (i == msg->sg.end) 283 break; 284 } while (1); 285 } 286 287 static inline void sk_msg_sg_copy_assign(struct sk_msg *dst, u32 dst_i, 288 const struct sk_msg *src, u32 src_i) 289 { 290 __assign_bit(dst_i, dst->sg.copy, test_bit(src_i, src->sg.copy)); 291 } 292 293 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start) 294 { 295 sk_msg_sg_copy(msg, start, true); 296 } 297 298 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start) 299 { 300 sk_msg_sg_copy(msg, start, false); 301 } 302 303 static inline struct sk_psock *sk_psock(const struct sock *sk) 304 { 305 return __rcu_dereference_sk_user_data_with_flags(sk, 306 SK_USER_DATA_PSOCK); 307 } 308 309 static inline void sk_psock_set_state(struct sk_psock *psock, 310 enum sk_psock_state_bits bit) 311 { 312 set_bit(bit, &psock->state); 313 } 314 315 static inline void sk_psock_clear_state(struct sk_psock *psock, 316 enum sk_psock_state_bits bit) 317 { 318 clear_bit(bit, &psock->state); 319 } 320 321 static inline bool sk_psock_test_state(const struct sk_psock *psock, 322 enum sk_psock_state_bits bit) 323 { 324 return test_bit(bit, &psock->state); 325 } 326 327 static inline void sock_drop(struct sock *sk, struct sk_buff *skb) 328 { 329 sk_drops_skbadd(sk, skb); 330 kfree_skb(skb); 331 } 332 333 static inline u32 sk_psock_get_msg_len_nolock(struct sk_psock *psock) 334 { 335 /* Used by ioctl to read msg_tot_len only; lock-free for performance */ 336 return READ_ONCE(psock->msg_tot_len); 337 } 338 339 static inline void sk_psock_msg_len_add_locked(struct sk_psock *psock, int diff) 340 { 341 /* Use WRITE_ONCE to ensure correct read in sk_psock_get_msg_len_nolock(). 342 * ingress_lock should be held to prevent concurrent updates to msg_tot_len 343 */ 344 WRITE_ONCE(psock->msg_tot_len, psock->msg_tot_len + diff); 345 } 346 347 static inline void sk_psock_msg_len_add(struct sk_psock *psock, int diff) 348 { 349 spin_lock_bh(&psock->ingress_lock); 350 sk_psock_msg_len_add_locked(psock, diff); 351 spin_unlock_bh(&psock->ingress_lock); 352 } 353 354 static inline bool sk_psock_queue_msg(struct sk_psock *psock, 355 struct sk_msg *msg) 356 { 357 bool ret; 358 359 spin_lock_bh(&psock->ingress_lock); 360 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { 361 list_add_tail(&msg->list, &psock->ingress_msg); 362 sk_psock_msg_len_add_locked(psock, msg->sg.size); 363 ret = true; 364 } else { 365 sk_msg_free(psock->sk, msg); 366 kfree(msg); 367 ret = false; 368 } 369 spin_unlock_bh(&psock->ingress_lock); 370 return ret; 371 } 372 373 static inline struct sk_msg *sk_psock_dequeue_msg(struct sk_psock *psock) 374 { 375 struct sk_msg *msg; 376 377 spin_lock_bh(&psock->ingress_lock); 378 msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); 379 if (msg) { 380 list_del(&msg->list); 381 sk_psock_msg_len_add_locked(psock, -msg->sg.size); 382 } 383 spin_unlock_bh(&psock->ingress_lock); 384 return msg; 385 } 386 387 static inline struct sk_msg *sk_psock_peek_msg_locked(struct sk_psock *psock) 388 { 389 return list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list); 390 } 391 392 static inline struct sk_msg *sk_psock_peek_msg(struct sk_psock *psock) 393 { 394 struct sk_msg *msg; 395 396 spin_lock_bh(&psock->ingress_lock); 397 msg = sk_psock_peek_msg_locked(psock); 398 spin_unlock_bh(&psock->ingress_lock); 399 return msg; 400 } 401 402 static inline struct sk_msg *sk_psock_next_msg(struct sk_psock *psock, 403 struct sk_msg *msg) 404 { 405 struct sk_msg *ret; 406 407 spin_lock_bh(&psock->ingress_lock); 408 if (list_is_last(&msg->list, &psock->ingress_msg)) 409 ret = NULL; 410 else 411 ret = list_next_entry(msg, list); 412 spin_unlock_bh(&psock->ingress_lock); 413 return ret; 414 } 415 416 static inline bool sk_psock_queue_empty(const struct sk_psock *psock) 417 { 418 return psock ? list_empty(&psock->ingress_msg) : true; 419 } 420 421 static inline void kfree_sk_msg(struct sk_msg *msg) 422 { 423 if (msg->skb) 424 consume_skb(msg->skb); 425 kfree(msg); 426 } 427 428 static inline void sk_psock_report_error(struct sk_psock *psock, int err) 429 { 430 struct sock *sk = psock->sk; 431 432 sk->sk_err = err; 433 sk_error_report(sk); 434 } 435 436 struct sk_psock *sk_psock_init(struct sock *sk, int node); 437 void sk_psock_stop(struct sk_psock *psock); 438 439 #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) 440 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock); 441 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock); 442 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock); 443 #else 444 static inline int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) 445 { 446 return -EOPNOTSUPP; 447 } 448 449 static inline void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) 450 { 451 } 452 453 static inline void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) 454 { 455 } 456 #endif 457 458 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock); 459 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock); 460 461 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, 462 struct sk_msg *msg); 463 464 /* 465 * This specialized allocator has to be a macro for its allocations to be 466 * accounted separately (to have a separate alloc_tag). The typecast is 467 * intentional to enforce typesafety. 468 */ 469 #define sk_psock_init_link() \ 470 kzalloc_obj(struct sk_psock_link, GFP_ATOMIC | __GFP_NOWARN) 471 472 static inline void sk_psock_free_link(struct sk_psock_link *link) 473 { 474 kfree(link); 475 } 476 477 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock); 478 479 static inline void sk_psock_cork_free(struct sk_psock *psock) 480 { 481 if (psock->cork) { 482 sk_msg_free(psock->sk, psock->cork); 483 kfree(psock->cork); 484 psock->cork = NULL; 485 } 486 } 487 488 static inline void sk_psock_restore_proto(struct sock *sk, 489 struct sk_psock *psock) 490 { 491 if (psock->psock_update_sk_prot) 492 psock->psock_update_sk_prot(sk, psock, true); 493 } 494 495 static inline struct sk_psock *sk_psock_get(struct sock *sk) 496 { 497 struct sk_psock *psock; 498 499 rcu_read_lock(); 500 psock = sk_psock(sk); 501 if (psock && !refcount_inc_not_zero(&psock->refcnt)) 502 psock = NULL; 503 rcu_read_unlock(); 504 return psock; 505 } 506 507 void sk_psock_drop(struct sock *sk, struct sk_psock *psock); 508 509 static inline void sk_psock_put(struct sock *sk, struct sk_psock *psock) 510 { 511 if (refcount_dec_and_test(&psock->refcnt)) 512 sk_psock_drop(sk, psock); 513 } 514 515 static inline void sk_psock_data_ready(struct sock *sk, struct sk_psock *psock) 516 { 517 read_lock_bh(&sk->sk_callback_lock); 518 if (psock->saved_data_ready) 519 psock->saved_data_ready(sk); 520 else 521 sk->sk_data_ready(sk); 522 read_unlock_bh(&sk->sk_callback_lock); 523 } 524 525 static inline void psock_set_prog(struct bpf_prog **pprog, 526 struct bpf_prog *prog) 527 { 528 prog = xchg(pprog, prog); 529 if (prog) 530 bpf_prog_put(prog); 531 } 532 533 static inline int psock_replace_prog(struct bpf_prog **pprog, 534 struct bpf_prog *prog, 535 struct bpf_prog *old) 536 { 537 if (cmpxchg(pprog, old, prog) != old) 538 return -ENOENT; 539 540 if (old) 541 bpf_prog_put(old); 542 543 return 0; 544 } 545 546 static inline void psock_progs_drop(struct sk_psock_progs *progs) 547 { 548 psock_set_prog(&progs->msg_parser, NULL); 549 psock_set_prog(&progs->stream_parser, NULL); 550 psock_set_prog(&progs->stream_verdict, NULL); 551 psock_set_prog(&progs->skb_verdict, NULL); 552 } 553 554 /* for tcp only, sk is locked */ 555 static inline ssize_t sk_psock_msg_inq(struct sock *sk) 556 { 557 struct sk_psock *psock; 558 ssize_t inq = 0; 559 560 psock = sk_psock_get(sk); 561 if (likely(psock)) { 562 inq = sk_psock_get_msg_len_nolock(psock); 563 sk_psock_put(sk, psock); 564 } 565 return inq; 566 } 567 568 /* for udp only, sk is not locked */ 569 static inline ssize_t sk_msg_first_len(struct sock *sk) 570 { 571 struct sk_psock *psock; 572 struct sk_msg *msg; 573 ssize_t inq = 0; 574 575 psock = sk_psock_get(sk); 576 if (likely(psock)) { 577 spin_lock_bh(&psock->ingress_lock); 578 msg = sk_psock_peek_msg_locked(psock); 579 if (msg) 580 inq = msg->sg.size; 581 spin_unlock_bh(&psock->ingress_lock); 582 sk_psock_put(sk, psock); 583 } 584 return inq; 585 } 586 587 #if IS_ENABLED(CONFIG_NET_SOCK_MSG) 588 589 #define BPF_F_STRPARSER (1UL << 1) 590 591 /* We only have two bits so far. */ 592 #define BPF_F_PTR_MASK ~(BPF_F_INGRESS | BPF_F_STRPARSER) 593 594 static inline bool skb_bpf_strparser(const struct sk_buff *skb) 595 { 596 unsigned long sk_redir = skb->_sk_redir; 597 598 return sk_redir & BPF_F_STRPARSER; 599 } 600 601 static inline void skb_bpf_set_strparser(struct sk_buff *skb) 602 { 603 skb->_sk_redir |= BPF_F_STRPARSER; 604 } 605 606 static inline bool skb_bpf_ingress(const struct sk_buff *skb) 607 { 608 unsigned long sk_redir = skb->_sk_redir; 609 610 return sk_redir & BPF_F_INGRESS; 611 } 612 613 static inline void skb_bpf_set_ingress(struct sk_buff *skb) 614 { 615 skb->_sk_redir |= BPF_F_INGRESS; 616 } 617 618 static inline void skb_bpf_set_redir(struct sk_buff *skb, struct sock *sk_redir, 619 bool ingress) 620 { 621 skb->_sk_redir = (unsigned long)sk_redir; 622 if (ingress) 623 skb->_sk_redir |= BPF_F_INGRESS; 624 } 625 626 static inline struct sock *skb_bpf_redirect_fetch(const struct sk_buff *skb) 627 { 628 unsigned long sk_redir = skb->_sk_redir; 629 630 return (struct sock *)(sk_redir & BPF_F_PTR_MASK); 631 } 632 633 static inline void skb_bpf_redirect_clear(struct sk_buff *skb) 634 { 635 skb->_sk_redir = 0; 636 } 637 #endif /* CONFIG_NET_SOCK_MSG */ 638 #endif /* _LINUX_SKMSG_H */ 639