Lines Matching refs:psock
412 int __sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, in __sk_msg_recvmsg() argument
421 msg_rx = sk_psock_peek_msg(psock); in __sk_msg_recvmsg()
461 sk_psock_msg_len_add(psock, -copy); in __sk_msg_recvmsg()
482 msg_rx = sk_psock_next_msg(psock, msg_rx); in __sk_msg_recvmsg()
490 msg_rx = sk_psock_dequeue_msg(psock); in __sk_msg_recvmsg()
493 msg_rx = sk_psock_peek_msg(psock); in __sk_msg_recvmsg()
500 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, in sk_msg_recvmsg() argument
503 return __sk_msg_recvmsg(sk, psock, msg, len, flags, NULL); in sk_msg_recvmsg()
509 struct sk_psock *psock; in sk_msg_is_readable() local
513 psock = sk_psock(sk); in sk_msg_is_readable()
514 if (likely(psock)) in sk_msg_is_readable()
515 empty = list_empty(&psock->ingress_msg); in sk_msg_is_readable()
546 struct sk_psock *psock, in sk_psock_skb_ingress_enqueue() argument
574 psock->ingress_bytes += len; in sk_psock_skb_ingress_enqueue()
582 sk_psock_queue_msg(psock, msg); in sk_psock_skb_ingress_enqueue()
583 sk_psock_data_ready(sk, psock); in sk_psock_skb_ingress_enqueue()
587 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb,
590 static int sk_psock_skb_ingress(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_skb_ingress() argument
593 struct sock *sk = psock->sk; in sk_psock_skb_ingress()
602 return sk_psock_skb_ingress_self(psock, skb, off, len, true); in sk_psock_skb_ingress()
614 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, true); in sk_psock_skb_ingress()
624 static int sk_psock_skb_ingress_self(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_skb_ingress_self() argument
628 struct sock *sk = psock->sk; in sk_psock_skb_ingress_self()
640 err = sk_psock_skb_ingress_enqueue(skb, off, len, psock, sk, msg, take_ref); in sk_psock_skb_ingress_self()
646 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_handle_skb() argument
650 if (!sock_writeable(psock->sk)) in sk_psock_handle_skb()
652 return skb_send_sock(psock->sk, skb, off, len); in sk_psock_handle_skb()
655 return sk_psock_skb_ingress(psock, skb, off, len); in sk_psock_handle_skb()
658 static void sk_psock_skb_state(struct sk_psock *psock, in sk_psock_skb_state() argument
662 spin_lock_bh(&psock->ingress_lock); in sk_psock_skb_state()
663 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_skb_state()
667 spin_unlock_bh(&psock->ingress_lock); in sk_psock_skb_state()
673 struct sk_psock *psock = container_of(dwork, struct sk_psock, work); in sk_psock_backlog() local
674 struct sk_psock_work_state *state = &psock->work_state; in sk_psock_backlog()
684 if (!sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_backlog()
692 if (!sk_psock_get(psock->sk)) in sk_psock_backlog()
694 mutex_lock(&psock->work_mutex); in sk_psock_backlog()
695 while ((skb = skb_peek(&psock->ingress_skb))) { in sk_psock_backlog()
715 if (!sock_flag(psock->sk, SOCK_DEAD)) in sk_psock_backlog()
716 ret = sk_psock_handle_skb(psock, skb, off, in sk_psock_backlog()
720 sk_psock_skb_state(psock, state, len, off); in sk_psock_backlog()
722 skb_bpf_set_redir(skb, psock->sk, ingress); in sk_psock_backlog()
726 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_backlog()
727 schedule_delayed_work(&psock->work, 1); in sk_psock_backlog()
731 sk_psock_report_error(psock, ret ? -ret : EPIPE); in sk_psock_backlog()
732 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_backlog()
740 sk_psock_skb_state(psock, state, 0, 0); in sk_psock_backlog()
741 skb = skb_dequeue(&psock->ingress_skb); in sk_psock_backlog()
745 mutex_unlock(&psock->work_mutex); in sk_psock_backlog()
746 sk_psock_put(psock->sk, psock); in sk_psock_backlog()
751 struct sk_psock *psock; in sk_psock_init() local
757 psock = ERR_PTR(-EINVAL); in sk_psock_init()
762 psock = ERR_PTR(-EBUSY); in sk_psock_init()
766 psock = kzalloc_node(sizeof(*psock), GFP_ATOMIC | __GFP_NOWARN, node); in sk_psock_init()
767 if (!psock) { in sk_psock_init()
768 psock = ERR_PTR(-ENOMEM); in sk_psock_init()
773 psock->sk = sk; in sk_psock_init()
774 psock->eval = __SK_NONE; in sk_psock_init()
775 psock->sk_proto = prot; in sk_psock_init()
776 psock->saved_unhash = prot->unhash; in sk_psock_init()
777 psock->saved_destroy = prot->destroy; in sk_psock_init()
778 psock->saved_close = prot->close; in sk_psock_init()
779 psock->saved_write_space = sk->sk_write_space; in sk_psock_init()
781 INIT_LIST_HEAD(&psock->link); in sk_psock_init()
782 spin_lock_init(&psock->link_lock); in sk_psock_init()
784 INIT_DELAYED_WORK(&psock->work, sk_psock_backlog); in sk_psock_init()
785 mutex_init(&psock->work_mutex); in sk_psock_init()
786 INIT_LIST_HEAD(&psock->ingress_msg); in sk_psock_init()
787 spin_lock_init(&psock->ingress_lock); in sk_psock_init()
788 skb_queue_head_init(&psock->ingress_skb); in sk_psock_init()
790 sk_psock_set_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_init()
791 refcount_set(&psock->refcnt, 1); in sk_psock_init()
793 __rcu_assign_sk_user_data_with_flags(sk, psock, in sk_psock_init()
800 return psock; in sk_psock_init()
804 struct sk_psock_link *sk_psock_link_pop(struct sk_psock *psock) in sk_psock_link_pop() argument
808 spin_lock_bh(&psock->link_lock); in sk_psock_link_pop()
809 link = list_first_entry_or_null(&psock->link, struct sk_psock_link, in sk_psock_link_pop()
813 spin_unlock_bh(&psock->link_lock); in sk_psock_link_pop()
817 static void __sk_psock_purge_ingress_msg(struct sk_psock *psock) in __sk_psock_purge_ingress_msg() argument
821 list_for_each_entry_safe(msg, tmp, &psock->ingress_msg, list) { in __sk_psock_purge_ingress_msg()
824 atomic_sub(msg->sg.size, &psock->sk->sk_rmem_alloc); in __sk_psock_purge_ingress_msg()
825 sk_psock_msg_len_add(psock, -msg->sg.size); in __sk_psock_purge_ingress_msg()
826 sk_msg_free(psock->sk, msg); in __sk_psock_purge_ingress_msg()
829 WARN_ON_ONCE(psock->msg_tot_len); in __sk_psock_purge_ingress_msg()
832 static void __sk_psock_zap_ingress(struct sk_psock *psock) in __sk_psock_zap_ingress() argument
836 while ((skb = skb_dequeue(&psock->ingress_skb)) != NULL) { in __sk_psock_zap_ingress()
838 sock_drop(psock->sk, skb); in __sk_psock_zap_ingress()
840 __sk_psock_purge_ingress_msg(psock); in __sk_psock_zap_ingress()
843 static void sk_psock_link_destroy(struct sk_psock *psock) in sk_psock_link_destroy() argument
847 list_for_each_entry_safe(link, tmp, &psock->link, list) { in sk_psock_link_destroy()
853 void sk_psock_stop(struct sk_psock *psock) in sk_psock_stop() argument
855 spin_lock_bh(&psock->ingress_lock); in sk_psock_stop()
856 sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); in sk_psock_stop()
857 sk_psock_cork_free(psock); in sk_psock_stop()
858 spin_unlock_bh(&psock->ingress_lock); in sk_psock_stop()
861 static void sk_psock_done_strp(struct sk_psock *psock);
865 struct sk_psock *psock = container_of(to_rcu_work(work), in sk_psock_destroy() local
869 sk_psock_done_strp(psock); in sk_psock_destroy()
871 cancel_delayed_work_sync(&psock->work); in sk_psock_destroy()
872 __sk_psock_zap_ingress(psock); in sk_psock_destroy()
873 mutex_destroy(&psock->work_mutex); in sk_psock_destroy()
875 psock_progs_drop(&psock->progs); in sk_psock_destroy()
877 sk_psock_link_destroy(psock); in sk_psock_destroy()
878 sk_psock_cork_free(psock); in sk_psock_destroy()
880 if (psock->sk_redir) in sk_psock_destroy()
881 sock_put(psock->sk_redir); in sk_psock_destroy()
882 if (psock->sk_pair) in sk_psock_destroy()
883 sock_put(psock->sk_pair); in sk_psock_destroy()
884 sock_put(psock->sk); in sk_psock_destroy()
885 kfree(psock); in sk_psock_destroy()
888 void sk_psock_drop(struct sock *sk, struct sk_psock *psock) in sk_psock_drop() argument
891 sk_psock_restore_proto(sk, psock); in sk_psock_drop()
893 if (psock->progs.stream_parser) in sk_psock_drop()
894 sk_psock_stop_strp(sk, psock); in sk_psock_drop()
895 else if (psock->progs.stream_verdict || psock->progs.skb_verdict) in sk_psock_drop()
896 sk_psock_stop_verdict(sk, psock); in sk_psock_drop()
899 sk_psock_stop(psock); in sk_psock_drop()
901 INIT_RCU_WORK(&psock->rwork, sk_psock_destroy); in sk_psock_drop()
902 queue_rcu_work(system_percpu_wq, &psock->rwork); in sk_psock_drop()
919 int sk_psock_msg_verdict(struct sock *sk, struct sk_psock *psock, in sk_psock_msg_verdict() argument
926 prog = READ_ONCE(psock->progs.msg_parser); in sk_psock_msg_verdict()
937 psock->apply_bytes = msg->apply_bytes; in sk_psock_msg_verdict()
939 if (psock->sk_redir) { in sk_psock_msg_verdict()
940 sock_put(psock->sk_redir); in sk_psock_msg_verdict()
941 psock->sk_redir = NULL; in sk_psock_msg_verdict()
947 psock->redir_ingress = sk_msg_to_ingress(msg); in sk_psock_msg_verdict()
948 psock->sk_redir = msg->sk_redir; in sk_psock_msg_verdict()
949 sock_hold(psock->sk_redir); in sk_psock_msg_verdict()
1009 int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb) in sk_psock_tls_strp_read() argument
1015 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_tls_strp_read()
1017 skb->sk = psock->sk; in sk_psock_tls_strp_read()
1024 sk_psock_tls_verdict_apply(skb, psock, ret); in sk_psock_tls_strp_read()
1030 static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, in sk_psock_verdict_apply() argument
1040 sk_other = psock->sk; in sk_psock_verdict_apply()
1042 !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_verdict_apply()
1053 if (skb_queue_empty(&psock->ingress_skb)) { in sk_psock_verdict_apply()
1062 err = sk_psock_skb_ingress_self(psock, skb, off, len, false); in sk_psock_verdict_apply()
1065 spin_lock_bh(&psock->ingress_lock); in sk_psock_verdict_apply()
1066 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) { in sk_psock_verdict_apply()
1067 skb_queue_tail(&psock->ingress_skb, skb); in sk_psock_verdict_apply()
1068 schedule_delayed_work(&psock->work, 0); in sk_psock_verdict_apply()
1071 spin_unlock_bh(&psock->ingress_lock); in sk_psock_verdict_apply()
1077 tcp_eat_skb(psock->sk, skb); in sk_psock_verdict_apply()
1078 err = sk_psock_skb_redirect(psock, skb); in sk_psock_verdict_apply()
1084 tcp_eat_skb(psock->sk, skb); in sk_psock_verdict_apply()
1085 sock_drop(psock->sk, skb); in sk_psock_verdict_apply()
1093 struct sk_psock *psock; in sk_psock_write_space() local
1097 psock = sk_psock(sk); in sk_psock_write_space()
1098 if (likely(psock)) { in sk_psock_write_space()
1099 if (sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) in sk_psock_write_space()
1100 schedule_delayed_work(&psock->work, 0); in sk_psock_write_space()
1101 write_space = psock->saved_write_space; in sk_psock_write_space()
1111 struct sk_psock *psock; in sk_psock_strp_read() local
1118 psock = sk_psock(sk); in sk_psock_strp_read()
1119 if (unlikely(!psock)) { in sk_psock_strp_read()
1123 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_strp_read()
1133 sk_psock_verdict_apply(psock, skb, ret); in sk_psock_strp_read()
1145 struct sk_psock *psock = container_of(strp, struct sk_psock, strp); in sk_psock_strp_parse() local
1150 prog = READ_ONCE(psock->progs.stream_parser); in sk_psock_strp_parse()
1152 skb->sk = psock->sk; in sk_psock_strp_parse()
1163 struct sk_psock *psock; in sk_psock_strp_data_ready() local
1168 psock = sk_psock(sk); in sk_psock_strp_data_ready()
1169 if (likely(psock)) { in sk_psock_strp_data_ready()
1171 psock->saved_data_ready(sk); in sk_psock_strp_data_ready()
1174 strp_data_ready(&psock->strp); in sk_psock_strp_data_ready()
1181 int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_init_strp() argument
1191 ret = strp_init(&psock->strp, sk, &cb); in sk_psock_init_strp()
1193 sk_psock_set_state(psock, SK_PSOCK_RX_STRP_ENABLED); in sk_psock_init_strp()
1196 psock->strp.cb.read_sock = tcp_bpf_strp_read_sock; in sk_psock_init_strp()
1197 psock->copied_seq = tcp_sk(sk)->copied_seq; in sk_psock_init_strp()
1202 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_start_strp() argument
1204 if (psock->saved_data_ready) in sk_psock_start_strp()
1207 psock->saved_data_ready = sk->sk_data_ready; in sk_psock_start_strp()
1212 void sk_psock_stop_strp(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_strp() argument
1214 psock_set_prog(&psock->progs.stream_parser, NULL); in sk_psock_stop_strp()
1216 if (!psock->saved_data_ready) in sk_psock_stop_strp()
1219 sk->sk_data_ready = psock->saved_data_ready; in sk_psock_stop_strp()
1220 psock->saved_data_ready = NULL; in sk_psock_stop_strp()
1221 strp_stop(&psock->strp); in sk_psock_stop_strp()
1224 static void sk_psock_done_strp(struct sk_psock *psock) in sk_psock_done_strp() argument
1227 if (sk_psock_test_state(psock, SK_PSOCK_RX_STRP_ENABLED)) in sk_psock_done_strp()
1228 strp_done(&psock->strp); in sk_psock_done_strp()
1231 static void sk_psock_done_strp(struct sk_psock *psock) in sk_psock_done_strp() argument
1238 struct sk_psock *psock; in sk_psock_verdict_recv() local
1244 psock = sk_psock(sk); in sk_psock_verdict_recv()
1245 if (unlikely(!psock)) { in sk_psock_verdict_recv()
1251 prog = READ_ONCE(psock->progs.stream_verdict); in sk_psock_verdict_recv()
1253 prog = READ_ONCE(psock->progs.skb_verdict); in sk_psock_verdict_recv()
1260 ret = sk_psock_verdict_apply(psock, skb, ret); in sk_psock_verdict_recv()
1283 struct sk_psock *psock; in sk_psock_verdict_data_ready() local
1286 psock = sk_psock(sk); in sk_psock_verdict_data_ready()
1287 if (psock) in sk_psock_verdict_data_ready()
1288 sk_psock_data_ready(sk, psock); in sk_psock_verdict_data_ready()
1293 void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_start_verdict() argument
1295 if (psock->saved_data_ready) in sk_psock_start_verdict()
1298 psock->saved_data_ready = sk->sk_data_ready; in sk_psock_start_verdict()
1303 void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock) in sk_psock_stop_verdict() argument
1305 psock_set_prog(&psock->progs.stream_verdict, NULL); in sk_psock_stop_verdict()
1306 psock_set_prog(&psock->progs.skb_verdict, NULL); in sk_psock_stop_verdict()
1308 if (!psock->saved_data_ready) in sk_psock_stop_verdict()
1311 sk->sk_data_ready = psock->saved_data_ready; in sk_psock_stop_verdict()
1312 psock->saved_data_ready = NULL; in sk_psock_stop_verdict()