Lines Matching refs:sk
31 static unsigned int mptcp_inq_hint(const struct sock *sk);
50 static void __mptcp_destroy_sock(struct sock *sk);
51 static void mptcp_check_send_data_fin(struct sock *sk);
64 static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk) in mptcp_fallback_tcp_ops() argument
66 unsigned short family = READ_ONCE(sk->sk_family); in mptcp_fallback_tcp_ops()
106 struct sock *sk = (struct sock *)msk; in __mptcp_socket_create() local
110 err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock); in __mptcp_socket_create()
114 msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; in __mptcp_socket_create()
115 WRITE_ONCE(msk->first, ssock->sk); in __mptcp_socket_create()
116 subflow = mptcp_subflow_ctx(ssock->sk); in __mptcp_socket_create()
118 sock_hold(ssock->sk); in __mptcp_socket_create()
124 mptcp_sock_graft(msk->first, sk->sk_socket); in __mptcp_socket_create()
135 struct sock *sk = (struct sock *)msk; in __mptcp_nmpc_sk() local
138 if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) in __mptcp_nmpc_sk()
150 static void mptcp_drop(struct sock *sk, struct sk_buff *skb) in mptcp_drop() argument
152 sk_drops_skbadd(sk, skb); in mptcp_drop()
156 static bool __mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, in __mptcp_try_coalesce() argument
160 int limit = READ_ONCE(sk->sk_rcvbuf); in __mptcp_try_coalesce()
175 static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, in mptcp_try_coalesce() argument
181 if (!__mptcp_try_coalesce(sk, to, from, &fragstolen, &delta)) in mptcp_try_coalesce()
188 atomic_add(delta, &sk->sk_rmem_alloc); in mptcp_try_coalesce()
189 sk_mem_charge(sk, delta); in mptcp_try_coalesce()
208 static bool mptcp_rcvbuf_grow(struct sock *sk, u32 newval) in mptcp_rcvbuf_grow() argument
210 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_rcvbuf_grow()
211 const struct net *net = sock_net(sk); in mptcp_rcvbuf_grow()
218 (sk->sk_userlocks & SOCK_RCVBUF_LOCK)) in mptcp_rcvbuf_grow()
231 rcvbuf = min_t(u32, mptcp_space_from_win(sk, rcvwin), cap); in mptcp_rcvbuf_grow()
232 if (rcvbuf > sk->sk_rcvbuf) { in mptcp_rcvbuf_grow()
233 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); in mptcp_rcvbuf_grow()
245 struct sock *sk = (struct sock *)msk; in mptcp_data_queue_ofo() local
258 mptcp_drop(sk, skb); in mptcp_data_queue_ofo()
262 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); in mptcp_data_queue_ofo()
267 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); in mptcp_data_queue_ofo()
279 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); in mptcp_data_queue_ofo()
280 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); in mptcp_data_queue_ofo()
286 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); in mptcp_data_queue_ofo()
304 mptcp_drop(sk, skb); in mptcp_data_queue_ofo()
305 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); in mptcp_data_queue_ofo()
320 mptcp_drop(sk, skb1); in mptcp_data_queue_ofo()
321 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); in mptcp_data_queue_ofo()
325 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); in mptcp_data_queue_ofo()
342 mptcp_drop(sk, skb1); in mptcp_data_queue_ofo()
343 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); in mptcp_data_queue_ofo()
351 skb_set_owner_r(skb, sk); in mptcp_data_queue_ofo()
376 static bool __mptcp_move_skb(struct sock *sk, struct sk_buff *skb) in __mptcp_move_skb() argument
379 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_move_skb()
382 mptcp_borrow_fwdmem(sk, skb); in __mptcp_move_skb()
388 tail = skb_peek_tail(&sk->sk_receive_queue); in __mptcp_move_skb()
389 if (tail && mptcp_try_coalesce(sk, tail, skb)) in __mptcp_move_skb()
392 skb_set_owner_r(skb, sk); in __mptcp_move_skb()
393 __skb_queue_tail(&sk->sk_receive_queue, skb); in __mptcp_move_skb()
403 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); in __mptcp_move_skb()
404 mptcp_drop(sk, skb); in __mptcp_move_skb()
408 static void mptcp_stop_rtx_timer(struct sock *sk) in mptcp_stop_rtx_timer() argument
410 sk_stop_timer(sk, &sk->mptcp_retransmit_timer); in mptcp_stop_rtx_timer()
411 mptcp_sk(sk)->timer_ival = 0; in mptcp_stop_rtx_timer()
414 static void mptcp_close_wake_up(struct sock *sk) in mptcp_close_wake_up() argument
416 if (sock_flag(sk, SOCK_DEAD)) in mptcp_close_wake_up()
419 sk->sk_state_change(sk); in mptcp_close_wake_up()
420 if (sk->sk_shutdown == SHUTDOWN_MASK || in mptcp_close_wake_up()
421 sk->sk_state == TCP_CLOSE) in mptcp_close_wake_up()
422 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); in mptcp_close_wake_up()
424 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); in mptcp_close_wake_up()
442 static bool mptcp_pending_data_fin_ack(struct sock *sk) in mptcp_pending_data_fin_ack() argument
444 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin_ack()
446 return ((1 << sk->sk_state) & in mptcp_pending_data_fin_ack()
451 static void mptcp_check_data_fin_ack(struct sock *sk) in mptcp_check_data_fin_ack() argument
453 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin_ack()
456 if (mptcp_pending_data_fin_ack(sk)) { in mptcp_check_data_fin_ack()
459 switch (sk->sk_state) { in mptcp_check_data_fin_ack()
461 mptcp_set_state(sk, TCP_FIN_WAIT2); in mptcp_check_data_fin_ack()
466 mptcp_set_state(sk, TCP_CLOSE); in mptcp_check_data_fin_ack()
470 mptcp_close_wake_up(sk); in mptcp_check_data_fin_ack()
475 static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) in mptcp_pending_data_fin() argument
477 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_pending_data_fin()
480 ((1 << inet_sk_state_load(sk)) & in mptcp_pending_data_fin()
495 static void mptcp_set_datafin_timeout(struct sock *sk) in mptcp_set_datafin_timeout() argument
497 struct inet_connection_sock *icsk = inet_csk(sk); in mptcp_set_datafin_timeout()
503 mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits; in mptcp_set_datafin_timeout()
506 static void __mptcp_set_timeout(struct sock *sk, long tout) in __mptcp_set_timeout() argument
508 mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; in __mptcp_set_timeout()
519 static void mptcp_set_timeout(struct sock *sk) in mptcp_set_timeout() argument
524 mptcp_for_each_subflow(mptcp_sk(sk), subflow) in mptcp_set_timeout()
526 __mptcp_set_timeout(sk, tout); in mptcp_set_timeout()
585 struct sock *sk = (struct sock *)msk; in mptcp_cleanup_rbuf() local
586 int space = __mptcp_space(sk); in mptcp_cleanup_rbuf()
590 rx_empty = !sk_rmem_alloc_get(sk) && copied; in mptcp_cleanup_rbuf()
600 static void mptcp_check_data_fin(struct sock *sk) in mptcp_check_data_fin() argument
602 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_data_fin()
618 if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { in mptcp_check_data_fin()
622 WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN); in mptcp_check_data_fin()
625 switch (sk->sk_state) { in mptcp_check_data_fin()
627 mptcp_set_state(sk, TCP_CLOSE_WAIT); in mptcp_check_data_fin()
630 mptcp_set_state(sk, TCP_CLOSING); in mptcp_check_data_fin()
634 mptcp_set_state(sk, TCP_CLOSE); in mptcp_check_data_fin()
644 mptcp_close_wake_up(sk); in mptcp_check_data_fin()
656 static void __mptcp_add_backlog(struct sock *sk, in __mptcp_add_backlog() argument
660 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_add_backlog()
662 struct sock *ssk = skb->sk; in __mptcp_add_backlog()
666 if (unlikely(sk->sk_state == TCP_CLOSE)) { in __mptcp_add_backlog()
676 ssk == tail->sk && in __mptcp_add_backlog()
677 __mptcp_try_coalesce(sk, tail, skb, &fragstolen, &delta)) { in __mptcp_add_backlog()
702 struct sock *sk = (struct sock *)msk; in __mptcp_move_skbs_from_subflow() local
742 if (own_msk && sk_rmem_alloc_get(sk) < sk->sk_rcvbuf) { in __mptcp_move_skbs_from_subflow()
744 ret |= __mptcp_move_skb(sk, skb); in __mptcp_move_skbs_from_subflow()
746 __mptcp_add_backlog(sk, subflow, skb); in __mptcp_move_skbs_from_subflow()
775 struct sock *sk = (struct sock *)msk; in __mptcp_ofo_queue() local
793 mptcp_drop(sk, skb); in __mptcp_ofo_queue()
794 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); in __mptcp_ofo_queue()
799 tail = skb_peek_tail(&sk->sk_receive_queue); in __mptcp_ofo_queue()
809 __skb_queue_tail(&sk->sk_receive_queue, skb); in __mptcp_ofo_queue()
818 static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) in __mptcp_subflow_error_report() argument
826 if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk))) in __mptcp_subflow_error_report()
839 if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD)) in __mptcp_subflow_error_report()
840 mptcp_set_state(sk, ssk_state); in __mptcp_subflow_error_report()
841 WRITE_ONCE(sk->sk_err, -err); in __mptcp_subflow_error_report()
845 sk_error_report(sk); in __mptcp_subflow_error_report()
849 void __mptcp_error_report(struct sock *sk) in __mptcp_error_report() argument
852 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_error_report()
855 if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) in __mptcp_error_report()
864 struct sock *sk = (struct sock *)msk; in move_skbs_to_msk() local
870 __mptcp_subflow_error_report(sk, ssk); in move_skbs_to_msk()
877 if (mptcp_pending_data_fin(sk, NULL)) in move_skbs_to_msk()
878 mptcp_schedule_work(sk); in move_skbs_to_msk()
882 void mptcp_data_ready(struct sock *sk, struct sock *ssk) in mptcp_data_ready() argument
885 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_data_ready()
894 mptcp_data_lock(sk); in mptcp_data_ready()
895 if (!sock_owned_by_user(sk)) { in mptcp_data_ready()
897 if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk)) in mptcp_data_ready()
898 sk->sk_data_ready(sk); in mptcp_data_ready()
902 mptcp_data_unlock(sk); in mptcp_data_ready()
914 struct sock *sk = (struct sock *)msk; in __mptcp_finish_join() local
916 if (sk->sk_state != TCP_ESTABLISHED) in __mptcp_finish_join()
929 mptcp_stop_tout_timer(sk); in __mptcp_finish_join()
930 __mptcp_propagate_sndbuf(sk, ssk); in __mptcp_finish_join()
934 static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list) in __mptcp_flush_join_list() argument
937 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_flush_join_list()
950 static bool mptcp_rtx_timer_pending(struct sock *sk) in mptcp_rtx_timer_pending() argument
952 return timer_pending(&sk->mptcp_retransmit_timer); in mptcp_rtx_timer_pending()
955 static void mptcp_reset_rtx_timer(struct sock *sk) in mptcp_reset_rtx_timer() argument
960 if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) in mptcp_reset_rtx_timer()
963 tout = mptcp_sk(sk)->timer_ival; in mptcp_reset_rtx_timer()
964 sk_reset_timer(sk, &sk->mptcp_retransmit_timer, jiffies + tout); in mptcp_reset_rtx_timer()
967 bool mptcp_schedule_work(struct sock *sk) in mptcp_schedule_work() argument
969 if (inet_sk_state_load(sk) == TCP_CLOSE) in mptcp_schedule_work()
976 sock_hold(sk); in mptcp_schedule_work()
978 if (schedule_work(&mptcp_sk(sk)->work)) in mptcp_schedule_work()
981 sock_put(sk); in mptcp_schedule_work()
1014 static void dfrag_uncharge(struct sock *sk, int len) in dfrag_uncharge() argument
1016 sk_mem_uncharge(sk, len); in dfrag_uncharge()
1017 sk_wmem_queued_add(sk, -len); in dfrag_uncharge()
1020 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) in dfrag_clear() argument
1025 dfrag_uncharge(sk, len); in dfrag_clear()
1030 static void __mptcp_clean_una(struct sock *sk) in __mptcp_clean_una() argument
1032 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clean_una()
1046 msk->first_pending = mptcp_send_next(sk); in __mptcp_clean_una()
1049 dfrag_clear(sk, dfrag); in __mptcp_clean_una()
1052 dfrag = mptcp_rtx_head(sk); in __mptcp_clean_una()
1070 dfrag_uncharge(sk, delta); in __mptcp_clean_una()
1079 if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) in __mptcp_clean_una()
1080 mptcp_stop_rtx_timer(sk); in __mptcp_clean_una()
1082 mptcp_reset_rtx_timer(sk); in __mptcp_clean_una()
1085 if (mptcp_pending_data_fin_ack(sk)) in __mptcp_clean_una()
1086 mptcp_schedule_work(sk); in __mptcp_clean_una()
1089 static void __mptcp_clean_una_wakeup(struct sock *sk) in __mptcp_clean_una_wakeup() argument
1091 lockdep_assert_held_once(&sk->sk_lock.slock); in __mptcp_clean_una_wakeup()
1093 __mptcp_clean_una(sk); in __mptcp_clean_una_wakeup()
1094 mptcp_write_space(sk); in __mptcp_clean_una_wakeup()
1097 static void mptcp_clean_una_wakeup(struct sock *sk) in mptcp_clean_una_wakeup() argument
1099 mptcp_data_lock(sk); in mptcp_clean_una_wakeup()
1100 __mptcp_clean_una_wakeup(sk); in mptcp_clean_una_wakeup()
1101 mptcp_data_unlock(sk); in mptcp_clean_una_wakeup()
1104 static void mptcp_enter_memory_pressure(struct sock *sk) in mptcp_enter_memory_pressure() argument
1107 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_enter_memory_pressure()
1120 __mptcp_sync_sndbuf(sk); in mptcp_enter_memory_pressure()
1126 static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) in mptcp_page_frag_refill() argument
1129 pfrag, sk->sk_allocation))) in mptcp_page_frag_refill()
1132 mptcp_enter_memory_pressure(sk); in mptcp_page_frag_refill()
1194 static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) in __mptcp_do_alloc_tx_skb() argument
1208 mptcp_enter_memory_pressure(sk); in __mptcp_do_alloc_tx_skb()
1213 static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) in __mptcp_alloc_tx_skb() argument
1217 skb = __mptcp_do_alloc_tx_skb(sk, gfp); in __mptcp_alloc_tx_skb()
1230 static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) in mptcp_alloc_tx_skb() argument
1232 gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation; in mptcp_alloc_tx_skb()
1234 return __mptcp_alloc_tx_skb(sk, ssk, gfp); in mptcp_alloc_tx_skb()
1270 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, in mptcp_sendmsg_frag() argument
1276 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_frag()
1326 skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); in mptcp_sendmsg_frag()
1459 struct sock *sk = (struct sock *)msk; in mptcp_subflow_get_send() local
1497 __mptcp_set_timeout(sk, tout); in mptcp_subflow_get_send()
1564 void mptcp_check_and_set_pending(struct sock *sk) in mptcp_check_and_set_pending() argument
1566 if (mptcp_send_head(sk)) { in mptcp_check_and_set_pending()
1567 mptcp_data_lock(sk); in mptcp_check_and_set_pending()
1568 mptcp_sk(sk)->cb_flags |= BIT(MPTCP_PUSH_PENDING); in mptcp_check_and_set_pending()
1569 mptcp_data_unlock(sk); in mptcp_check_and_set_pending()
1573 static int __subflow_push_pending(struct sock *sk, struct sock *ssk, in __subflow_push_pending() argument
1576 struct mptcp_sock *msk = mptcp_sk(sk); in __subflow_push_pending()
1580 while ((dfrag = mptcp_send_head(sk))) { in __subflow_push_pending()
1587 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info); in __subflow_push_pending()
1599 msk->first_pending = mptcp_send_next(sk); in __subflow_push_pending()
1607 mptcp_set_timeout(sk); in __subflow_push_pending()
1617 void __mptcp_push_pending(struct sock *sk, unsigned int flags) in __mptcp_push_pending() argument
1620 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_push_pending()
1627 while (mptcp_send_head(sk) && (push_count > 0)) { in __mptcp_push_pending()
1658 ret = __subflow_push_pending(sk, ssk, &info); in __mptcp_push_pending()
1679 if (!mptcp_rtx_timer_pending(sk)) in __mptcp_push_pending()
1680 mptcp_reset_rtx_timer(sk); in __mptcp_push_pending()
1681 mptcp_check_send_data_fin(sk); in __mptcp_push_pending()
1685 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first) in __mptcp_subflow_push_pending() argument
1687 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_subflow_push_pending()
1696 while (mptcp_send_head(sk) && keep_pushing) { in __mptcp_subflow_push_pending()
1705 ret = __subflow_push_pending(sk, ssk, &info); in __mptcp_subflow_push_pending()
1718 ret = __subflow_push_pending(sk, ssk, &info); in __mptcp_subflow_push_pending()
1743 if (!mptcp_rtx_timer_pending(sk)) in __mptcp_subflow_push_pending()
1744 mptcp_reset_rtx_timer(sk); in __mptcp_subflow_push_pending()
1748 mptcp_schedule_work(sk); in __mptcp_subflow_push_pending()
1752 static int mptcp_disconnect(struct sock *sk, int flags);
1754 static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, in mptcp_sendmsg_fastopen() argument
1758 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg_fastopen()
1788 ret = __inet_stream_connect(sk->sk_socket, msg->msg_name, in mptcp_sendmsg_fastopen()
1802 if (!mptcp_disconnect(sk, 0)) { in mptcp_sendmsg_fastopen()
1803 sk->sk_disconnects++; in mptcp_sendmsg_fastopen()
1804 sk->sk_socket->state = SS_UNCONNECTED; in mptcp_sendmsg_fastopen()
1807 inet_clear_bit(DEFER_CONNECT, sk); in mptcp_sendmsg_fastopen()
1812 static int do_copy_data_nocache(struct sock *sk, int copy, in do_copy_data_nocache() argument
1815 if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) { in do_copy_data_nocache()
1829 static u32 mptcp_send_limit(const struct sock *sk) in mptcp_send_limit() argument
1831 const struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_send_limit()
1834 if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf)) in mptcp_send_limit()
1837 limit = mptcp_notsent_lowat(sk); in mptcp_send_limit()
1862 static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) in mptcp_sendmsg() argument
1864 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_sendmsg()
1873 lock_sock(sk); in mptcp_sendmsg()
1877 if (unlikely(inet_test_bit(DEFER_CONNECT, sk) || in mptcp_sendmsg()
1881 ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn); in mptcp_sendmsg()
1889 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); in mptcp_sendmsg()
1891 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { in mptcp_sendmsg()
1892 ret = sk_stream_wait_connect(sk, &timeo); in mptcp_sendmsg()
1898 if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))) in mptcp_sendmsg()
1901 pfrag = sk_page_frag(sk); in mptcp_sendmsg()
1911 copy_limit = mptcp_send_limit(sk); in mptcp_sendmsg()
1918 dfrag = mptcp_pending_tail(sk); in mptcp_sendmsg()
1921 if (!mptcp_page_frag_refill(sk, pfrag)) in mptcp_sendmsg()
1938 if (!sk_wmem_schedule(sk, total_ts)) in mptcp_sendmsg()
1941 ret = do_copy_data_nocache(sk, psize, &msg->msg_iter, in mptcp_sendmsg()
1947 sk_forward_alloc_add(sk, -total_ts); in mptcp_sendmsg()
1957 sk_wmem_queued_add(sk, frag_truesize); in mptcp_sendmsg()
1971 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in mptcp_sendmsg()
1972 __mptcp_push_pending(sk, msg->msg_flags); in mptcp_sendmsg()
1973 ret = sk_stream_wait_memory(sk, &timeo); in mptcp_sendmsg()
1979 __mptcp_push_pending(sk, msg->msg_flags); in mptcp_sendmsg()
1982 release_sock(sk); in mptcp_sendmsg()
1989 copied = sk_stream_error(sk, msg->msg_flags, ret); in mptcp_sendmsg()
1995 static void mptcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) in mptcp_eat_recv_skb() argument
1999 skb->sk = NULL; in mptcp_eat_recv_skb()
2000 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); in mptcp_eat_recv_skb()
2001 sk_mem_uncharge(sk, skb->truesize); in mptcp_eat_recv_skb()
2002 __skb_unlink(skb, &sk->sk_receive_queue); in mptcp_eat_recv_skb()
2006 static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, in __mptcp_recvmsg_mskq() argument
2011 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_recvmsg_mskq()
2016 skb_queue_walk_safe(&sk->sk_receive_queue, skb, tmp) { in __mptcp_recvmsg_mskq()
2060 mptcp_eat_recv_skb(sk, skb); in __mptcp_recvmsg_mskq()
2093 struct sock *sk = (struct sock *)msk; in mptcp_rcv_space_adjust() local
2139 trace_mptcp_rcvbuf_grow(sk, time); in mptcp_rcv_space_adjust()
2140 if (mptcp_rcvbuf_grow(sk, msk->rcvq_space.copied)) { in mptcp_rcv_space_adjust()
2164 static bool __mptcp_move_skbs(struct sock *sk, struct list_head *skbs, u32 *delta) in __mptcp_move_skbs() argument
2167 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_move_skbs()
2173 if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf) in __mptcp_move_skbs()
2180 moved |= __mptcp_move_skb(sk, skb); in __mptcp_move_skbs()
2193 static bool mptcp_can_spool_backlog(struct sock *sk, struct list_head *skbs) in mptcp_can_spool_backlog() argument
2195 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_can_spool_backlog()
2200 DEBUG_NET_WARN_ON_ONCE(msk->backlog_unaccounted && sk->sk_socket && in mptcp_can_spool_backlog()
2201 mem_cgroup_from_sk(sk)); in mptcp_can_spool_backlog()
2205 sk_rmem_alloc_get(sk) > sk->sk_rcvbuf) in mptcp_can_spool_backlog()
2213 static void mptcp_backlog_spooled(struct sock *sk, u32 moved, in mptcp_backlog_spooled() argument
2216 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_backlog_spooled()
2222 static bool mptcp_move_skbs(struct sock *sk) in mptcp_move_skbs() argument
2228 mptcp_data_lock(sk); in mptcp_move_skbs()
2229 while (mptcp_can_spool_backlog(sk, &skbs)) { in mptcp_move_skbs()
2230 mptcp_data_unlock(sk); in mptcp_move_skbs()
2231 enqueued |= __mptcp_move_skbs(sk, &skbs, &moved); in mptcp_move_skbs()
2233 mptcp_data_lock(sk); in mptcp_move_skbs()
2234 mptcp_backlog_spooled(sk, moved, &skbs); in mptcp_move_skbs()
2236 mptcp_data_unlock(sk); in mptcp_move_skbs()
2240 static unsigned int mptcp_inq_hint(const struct sock *sk) in mptcp_inq_hint() argument
2242 const struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_inq_hint()
2245 skb = skb_peek(&sk->sk_receive_queue); in mptcp_inq_hint()
2255 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) in mptcp_inq_hint()
2261 static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, in mptcp_recvmsg() argument
2264 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recvmsg()
2272 return inet_recv_error(sk, msg, len, addr_len); in mptcp_recvmsg()
2274 lock_sock(sk); in mptcp_recvmsg()
2275 if (unlikely(sk->sk_state == TCP_LISTEN)) { in mptcp_recvmsg()
2282 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); in mptcp_recvmsg()
2285 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); in mptcp_recvmsg()
2293 bytes_read = __mptcp_recvmsg_mskq(sk, msg, len - copied, flags, in mptcp_recvmsg()
2303 if (!list_empty(&msk->backlog_list) && mptcp_move_skbs(sk)) in mptcp_recvmsg()
2313 if (sk->sk_err || in mptcp_recvmsg()
2314 sk->sk_state == TCP_CLOSE || in mptcp_recvmsg()
2315 (sk->sk_shutdown & RCV_SHUTDOWN) || in mptcp_recvmsg()
2320 if (sk->sk_err) { in mptcp_recvmsg()
2321 copied = sock_error(sk); in mptcp_recvmsg()
2325 if (sk->sk_shutdown & RCV_SHUTDOWN) in mptcp_recvmsg()
2328 if (sk->sk_state == TCP_CLOSE) { in mptcp_recvmsg()
2346 err = sk_wait_data(sk, &timeo, NULL); in mptcp_recvmsg()
2358 tcp_recv_timestamp(msg, sk, &tss); in mptcp_recvmsg()
2361 unsigned int inq = mptcp_inq_hint(sk); in mptcp_recvmsg()
2368 msk, skb_queue_empty(&sk->sk_receive_queue), copied); in mptcp_recvmsg()
2370 release_sock(sk); in mptcp_recvmsg()
2376 struct sock *sk = timer_container_of(sk, t, mptcp_retransmit_timer); in mptcp_retransmit_timer() local
2377 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_retransmit_timer()
2379 bh_lock_sock(sk); in mptcp_retransmit_timer()
2380 if (!sock_owned_by_user(sk)) { in mptcp_retransmit_timer()
2383 mptcp_schedule_work(sk); in mptcp_retransmit_timer()
2388 bh_unlock_sock(sk); in mptcp_retransmit_timer()
2389 sock_put(sk); in mptcp_retransmit_timer()
2396 struct sock *sk = &icsk->icsk_inet.sk; in mptcp_tout_timer() local
2398 mptcp_schedule_work(sk); in mptcp_tout_timer()
2399 sock_put(sk); in mptcp_tout_timer()
2443 bool __mptcp_retransmit_pending_data(struct sock *sk) in __mptcp_retransmit_pending_data() argument
2446 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retransmit_pending_data()
2455 mptcp_data_lock(sk); in __mptcp_retransmit_pending_data()
2456 __mptcp_clean_una_wakeup(sk); in __mptcp_retransmit_pending_data()
2457 rtx_head = mptcp_rtx_head(sk); in __mptcp_retransmit_pending_data()
2459 mptcp_data_unlock(sk); in __mptcp_retransmit_pending_data()
2465 mptcp_data_unlock(sk); in __mptcp_retransmit_pending_data()
2511 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, in __mptcp_close_ssk() argument
2515 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close_ssk()
2532 sk_forward_alloc_add(sk, fwd_remaining); in __mptcp_close_ssk()
2543 (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) { in __mptcp_close_ssk()
2545 sock_set_flag(sk, SOCK_DEAD); in __mptcp_close_ssk()
2546 mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1)); in __mptcp_close_ssk()
2558 need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); in __mptcp_close_ssk()
2584 __mptcp_subflow_error_report(sk, ssk); in __mptcp_close_ssk()
2593 __mptcp_sync_sndbuf(sk); in __mptcp_close_ssk()
2595 __mptcp_push_pending(sk, 0); in __mptcp_close_ssk()
2604 if (sk->sk_state != TCP_ESTABLISHED || in __mptcp_close_ssk()
2605 msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) { in __mptcp_close_ssk()
2606 mptcp_set_state(sk, TCP_CLOSE); in __mptcp_close_ssk()
2607 mptcp_close_wake_up(sk); in __mptcp_close_ssk()
2609 mptcp_start_tout_timer(sk); in __mptcp_close_ssk()
2614 void mptcp_close_ssk(struct sock *sk, struct sock *ssk, in mptcp_close_ssk() argument
2617 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_close_ssk()
2626 if (sk->sk_state == TCP_ESTABLISHED) in mptcp_close_ssk()
2627 mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); in mptcp_close_ssk()
2633 if (skb->sk != ssk) in mptcp_close_ssk()
2636 atomic_sub(skb->truesize, &skb->sk->sk_rmem_alloc); in mptcp_close_ssk()
2637 skb->sk = NULL; in mptcp_close_ssk()
2643 mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); in mptcp_close_ssk()
2645 __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); in mptcp_close_ssk()
2648 static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) in mptcp_sync_mss() argument
2653 static void __mptcp_close_subflow(struct sock *sk) in __mptcp_close_subflow() argument
2656 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close_subflow()
2666 inet_sk_state_load(sk) != TCP_ESTABLISHED || in __mptcp_close_subflow()
2674 mptcp_close_ssk(sk, ssk, subflow); in __mptcp_close_subflow()
2679 static bool mptcp_close_tout_expired(const struct sock *sk) in mptcp_close_tout_expired() argument
2681 if (!inet_csk(sk)->icsk_mtup.probe_timestamp || in mptcp_close_tout_expired()
2682 sk->sk_state == TCP_CLOSE) in mptcp_close_tout_expired()
2686 inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk)); in mptcp_close_tout_expired()
2692 struct sock *sk = (struct sock *)msk; in mptcp_check_fastclose() local
2712 switch (sk->sk_state) { in mptcp_check_fastclose()
2714 WRITE_ONCE(sk->sk_err, ECONNREFUSED); in mptcp_check_fastclose()
2717 WRITE_ONCE(sk->sk_err, EPIPE); in mptcp_check_fastclose()
2722 WRITE_ONCE(sk->sk_err, ECONNRESET); in mptcp_check_fastclose()
2725 mptcp_set_state(sk, TCP_CLOSE); in mptcp_check_fastclose()
2726 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); in mptcp_check_fastclose()
2731 if (sock_flag(sk, SOCK_DEAD)) in mptcp_check_fastclose()
2734 sk->sk_state_change(sk); in mptcp_check_fastclose()
2735 sk_error_report(sk); in mptcp_check_fastclose()
2738 static void __mptcp_retrans(struct sock *sk) in __mptcp_retrans() argument
2741 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_retrans()
2748 mptcp_clean_una_wakeup(sk); in __mptcp_retrans()
2752 dfrag = mptcp_rtx_head(sk); in __mptcp_retrans()
2755 struct inet_connection_sock *icsk = inet_csk(sk); in __mptcp_retrans()
2759 mptcp_set_datafin_timeout(sk); in __mptcp_retrans()
2765 if (!mptcp_send_head(sk)) in __mptcp_retrans()
2804 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); in __mptcp_retrans()
2808 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); in __mptcp_retrans()
2828 mptcp_check_and_set_pending(sk); in __mptcp_retrans()
2830 if (!mptcp_rtx_timer_pending(sk)) in __mptcp_retrans()
2831 mptcp_reset_rtx_timer(sk); in __mptcp_retrans()
2848 struct sock *sk = (struct sock *)msk; in mptcp_reset_tout_timer() local
2851 if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp) in mptcp_reset_tout_timer()
2854 close_timeout = (unsigned long)inet_csk(sk)->icsk_mtup.probe_timestamp - in mptcp_reset_tout_timer()
2855 tcp_jiffies32 + jiffies + mptcp_close_timeout(sk); in mptcp_reset_tout_timer()
2860 timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout; in mptcp_reset_tout_timer()
2862 sk_reset_timer(sk, &inet_csk(sk)->mptcp_tout_timer, timeout); in mptcp_reset_tout_timer()
2881 static void mptcp_backlog_purge(struct sock *sk) in mptcp_backlog_purge() argument
2883 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_backlog_purge()
2887 mptcp_data_lock(sk); in mptcp_backlog_purge()
2890 mptcp_data_unlock(sk); in mptcp_backlog_purge()
2893 mptcp_borrow_fwdmem(sk, skb); in mptcp_backlog_purge()
2896 sk_mem_reclaim(sk); in mptcp_backlog_purge()
2899 static void mptcp_do_fastclose(struct sock *sk) in mptcp_do_fastclose() argument
2902 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_do_fastclose()
2904 mptcp_set_state(sk, TCP_CLOSE); in mptcp_do_fastclose()
2905 mptcp_backlog_purge(sk); in mptcp_do_fastclose()
2938 struct sock *sk = (struct sock *)msk; in mptcp_worker() local
2942 lock_sock(sk); in mptcp_worker()
2943 state = sk->sk_state; in mptcp_worker()
2951 mptcp_check_send_data_fin(sk); in mptcp_worker()
2952 mptcp_check_data_fin_ack(sk); in mptcp_worker()
2953 mptcp_check_data_fin(sk); in mptcp_worker()
2956 __mptcp_close_subflow(sk); in mptcp_worker()
2958 if (mptcp_close_tout_expired(sk)) { in mptcp_worker()
2961 mptcp_do_fastclose(sk); in mptcp_worker()
2963 __mptcp_close_ssk(sk, subflow->tcp_sock, subflow, 0); in mptcp_worker()
2964 mptcp_close_wake_up(sk); in mptcp_worker()
2967 if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) { in mptcp_worker()
2968 __mptcp_destroy_sock(sk); in mptcp_worker()
2973 __mptcp_retrans(sk); in mptcp_worker()
2980 release_sock(sk); in mptcp_worker()
2981 sock_put(sk); in mptcp_worker()
2984 static void __mptcp_init_sock(struct sock *sk) in __mptcp_init_sock() argument
2986 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_init_sock()
3000 inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; in __mptcp_init_sock()
3001 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in __mptcp_init_sock()
3014 timer_setup(&sk->mptcp_retransmit_timer, mptcp_retransmit_timer, 0); in __mptcp_init_sock()
3015 timer_setup(&msk->sk.mptcp_tout_timer, mptcp_tout_timer, 0); in __mptcp_init_sock()
3018 static void mptcp_ca_reset(struct sock *sk) in mptcp_ca_reset() argument
3020 struct inet_connection_sock *icsk = inet_csk(sk); in mptcp_ca_reset()
3022 tcp_assign_congestion_control(sk); in mptcp_ca_reset()
3023 strscpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name, in mptcp_ca_reset()
3024 sizeof(mptcp_sk(sk)->ca_name)); in mptcp_ca_reset()
3027 tcp_cleanup_congestion_control(sk); in mptcp_ca_reset()
3031 static int mptcp_init_sock(struct sock *sk) in mptcp_init_sock() argument
3033 struct net *net = sock_net(sk); in mptcp_init_sock()
3036 __mptcp_init_sock(sk); in mptcp_init_sock()
3045 ret = mptcp_init_sched(mptcp_sk(sk), in mptcp_init_sock()
3051 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags); in mptcp_init_sock()
3056 mptcp_ca_reset(sk); in mptcp_init_sock()
3058 sk_sockets_allocated_inc(sk); in mptcp_init_sock()
3059 sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]); in mptcp_init_sock()
3060 sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]); in mptcp_init_sock()
3061 sk->sk_write_space = sk_stream_write_space; in mptcp_init_sock()
3066 static void __mptcp_clear_xmit(struct sock *sk) in __mptcp_clear_xmit() argument
3068 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_clear_xmit()
3073 dfrag_clear(sk, dfrag); in __mptcp_clear_xmit()
3076 void mptcp_cancel_work(struct sock *sk) in mptcp_cancel_work() argument
3078 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_cancel_work()
3081 __sock_put(sk); in mptcp_cancel_work()
3084 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) in mptcp_subflow_shutdown() argument
3097 if (__mptcp_check_fallback(mptcp_sk(sk))) { in mptcp_subflow_shutdown()
3105 WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt); in mptcp_subflow_shutdown()
3106 mptcp_schedule_work(sk); in mptcp_subflow_shutdown()
3110 if (!mptcp_rtx_timer_pending(sk)) in mptcp_subflow_shutdown()
3111 mptcp_reset_rtx_timer(sk); in mptcp_subflow_shutdown()
3119 void mptcp_set_state(struct sock *sk, int state) in mptcp_set_state() argument
3121 int oldstate = sk->sk_state; in mptcp_set_state()
3126 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); in mptcp_set_state()
3136 MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); in mptcp_set_state()
3139 inet_sk_state_store(sk, state); in mptcp_set_state()
3159 static int mptcp_close_state(struct sock *sk) in mptcp_close_state() argument
3161 int next = (int)new_state[sk->sk_state]; in mptcp_close_state()
3164 mptcp_set_state(sk, ns); in mptcp_close_state()
3169 static void mptcp_check_send_data_fin(struct sock *sk) in mptcp_check_send_data_fin() argument
3172 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_check_send_data_fin()
3175 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), in mptcp_check_send_data_fin()
3182 mptcp_send_head(sk)) in mptcp_check_send_data_fin()
3190 mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); in mptcp_check_send_data_fin()
3194 static void __mptcp_wr_shutdown(struct sock *sk) in __mptcp_wr_shutdown() argument
3196 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_wr_shutdown()
3199 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, in __mptcp_wr_shutdown()
3200 !!mptcp_send_head(sk)); in __mptcp_wr_shutdown()
3206 mptcp_check_send_data_fin(sk); in __mptcp_wr_shutdown()
3209 static void __mptcp_destroy_sock(struct sock *sk) in __mptcp_destroy_sock() argument
3211 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_destroy_sock()
3217 mptcp_stop_rtx_timer(sk); in __mptcp_destroy_sock()
3218 sk_stop_timer(sk, &inet_csk(sk)->mptcp_tout_timer); in __mptcp_destroy_sock()
3222 sk->sk_prot->destroy(sk); in __mptcp_destroy_sock()
3224 sk_stream_kill_queues(sk); in __mptcp_destroy_sock()
3225 xfrm_sk_free_policy(sk); in __mptcp_destroy_sock()
3227 sock_put(sk); in __mptcp_destroy_sock()
3230 void __mptcp_unaccepted_force_close(struct sock *sk) in __mptcp_unaccepted_force_close() argument
3232 sock_set_flag(sk, SOCK_DEAD); in __mptcp_unaccepted_force_close()
3233 mptcp_do_fastclose(sk); in __mptcp_unaccepted_force_close()
3234 __mptcp_destroy_sock(sk); in __mptcp_unaccepted_force_close()
3237 static __poll_t mptcp_check_readable(struct sock *sk) in mptcp_check_readable() argument
3239 return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0; in mptcp_check_readable()
3242 static void mptcp_check_listen_stop(struct sock *sk) in mptcp_check_listen_stop() argument
3246 if (inet_sk_state_load(sk) != TCP_LISTEN) in mptcp_check_listen_stop()
3249 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); in mptcp_check_listen_stop()
3250 ssk = mptcp_sk(sk)->first; in mptcp_check_listen_stop()
3256 mptcp_subflow_queue_clean(sk, ssk); in mptcp_check_listen_stop()
3262 bool __mptcp_close(struct sock *sk, long timeout) in __mptcp_close() argument
3265 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_close()
3269 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); in __mptcp_close()
3271 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { in __mptcp_close()
3272 mptcp_check_listen_stop(sk); in __mptcp_close()
3273 mptcp_set_state(sk, TCP_CLOSE); in __mptcp_close()
3281 mptcp_do_fastclose(sk); in __mptcp_close()
3283 } else if (mptcp_close_state(sk)) { in __mptcp_close()
3284 __mptcp_wr_shutdown(sk); in __mptcp_close()
3287 sk_stream_wait_close(sk, timeout); in __mptcp_close()
3310 sock_orphan(sk); in __mptcp_close()
3316 mptcp_set_state(sk, TCP_CLOSE); in __mptcp_close()
3318 sock_hold(sk); in __mptcp_close()
3319 pr_debug("msk=%p state=%d\n", sk, sk->sk_state); in __mptcp_close()
3322 if (sk->sk_state == TCP_CLOSE) { in __mptcp_close()
3323 __mptcp_destroy_sock(sk); in __mptcp_close()
3326 mptcp_start_tout_timer(sk); in __mptcp_close()
3332 static void mptcp_close(struct sock *sk, long timeout) in mptcp_close() argument
3336 lock_sock(sk); in mptcp_close()
3338 do_cancel_work = __mptcp_close(sk, timeout); in mptcp_close()
3339 release_sock(sk); in mptcp_close()
3341 mptcp_cancel_work(sk); in mptcp_close()
3343 sock_put(sk); in mptcp_close()
3372 struct sock *sk = (struct sock *)msk; in mptcp_destroy_common() local
3374 __mptcp_clear_xmit(sk); in mptcp_destroy_common()
3375 mptcp_backlog_purge(sk); in mptcp_destroy_common()
3379 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, 0); in mptcp_destroy_common()
3381 __skb_queue_purge(&sk->sk_receive_queue); in mptcp_destroy_common()
3391 static int mptcp_disconnect(struct sock *sk, int flags) in mptcp_disconnect() argument
3393 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_disconnect()
3402 mptcp_check_listen_stop(sk); in mptcp_disconnect()
3403 mptcp_set_state(sk, TCP_CLOSE); in mptcp_disconnect()
3405 mptcp_stop_rtx_timer(sk); in mptcp_disconnect()
3406 mptcp_stop_tout_timer(sk); in mptcp_disconnect()
3413 mptcp_do_fastclose(sk); in mptcp_disconnect()
3433 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); in mptcp_disconnect()
3435 mptcp_ca_reset(sk); in mptcp_disconnect()
3447 WRITE_ONCE(sk->sk_shutdown, 0); in mptcp_disconnect()
3448 sk_error_report(sk); in mptcp_disconnect()
3453 static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) in mptcp_inet6_sk() argument
3455 struct mptcp6_sock *msk6 = container_of(mptcp_sk(sk), struct mptcp6_sock, msk); in mptcp_inet6_sk()
3460 static void mptcp_copy_ip6_options(struct sock *newsk, const struct sock *sk) in mptcp_copy_ip6_options() argument
3462 const struct ipv6_pinfo *np = inet6_sk(sk); in mptcp_copy_ip6_options()
3480 static void mptcp_copy_ip_options(struct sock *newsk, const struct sock *sk) in mptcp_copy_ip_options() argument
3483 const struct inet_sock *inet = inet_sk(sk); in mptcp_copy_ip_options()
3500 struct sock *mptcp_sk_clone_init(const struct sock *sk, in mptcp_sk_clone_init() argument
3506 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); in mptcp_sk_clone_init()
3522 mptcp_copy_ip6_options(nsk, sk); in mptcp_sk_clone_init()
3525 mptcp_copy_ip_options(nsk, sk); in mptcp_sk_clone_init()
3539 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; in mptcp_sk_clone_init()
3540 mptcp_init_sched(msk, mptcp_sk(sk)->sched); in mptcp_sk_clone_init()
3581 static void mptcp_destroy(struct sock *sk) in mptcp_destroy() argument
3583 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_destroy()
3588 sk_sockets_allocated_dec(sk); in mptcp_destroy()
3591 void __mptcp_data_acked(struct sock *sk) in __mptcp_data_acked() argument
3593 if (!sock_owned_by_user(sk)) in __mptcp_data_acked()
3594 __mptcp_clean_una(sk); in __mptcp_data_acked()
3596 __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags); in __mptcp_data_acked()
3599 void __mptcp_check_push(struct sock *sk, struct sock *ssk) in __mptcp_check_push() argument
3601 if (!sock_owned_by_user(sk)) in __mptcp_check_push()
3602 __mptcp_subflow_push_pending(sk, ssk, false); in __mptcp_check_push()
3604 __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); in __mptcp_check_push()
3612 static void mptcp_release_cb(struct sock *sk) in mptcp_release_cb() argument
3613 __must_hold(&sk->sk_lock.slock) in mptcp_release_cb()
3615 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_release_cb()
3623 spool_bl = mptcp_can_spool_backlog(sk, &skbs); in mptcp_release_cb()
3638 spin_unlock_bh(&sk->sk_lock.slock); in mptcp_release_cb()
3641 __mptcp_flush_join_list(sk, &join_list); in mptcp_release_cb()
3643 __mptcp_push_pending(sk, 0); in mptcp_release_cb()
3645 __mptcp_retrans(sk); in mptcp_release_cb()
3646 if (spool_bl && __mptcp_move_skbs(sk, &skbs, &moved)) { in mptcp_release_cb()
3649 sk->sk_data_ready(sk); in mptcp_release_cb()
3653 spin_lock_bh(&sk->sk_lock.slock); in mptcp_release_cb()
3655 mptcp_backlog_spooled(sk, moved, &skbs); in mptcp_release_cb()
3659 __mptcp_clean_una_wakeup(sk); in mptcp_release_cb()
3666 __mptcp_sync_state(sk, msk->pending_state); in mptcp_release_cb()
3668 __mptcp_error_report(sk); in mptcp_release_cb()
3670 __mptcp_sync_sndbuf(sk); in mptcp_release_cb()
3703 struct sock *sk = subflow->conn; in mptcp_subflow_process_delegated() local
3706 mptcp_data_lock(sk); in mptcp_subflow_process_delegated()
3707 if (!sock_owned_by_user(sk)) in mptcp_subflow_process_delegated()
3708 __mptcp_subflow_push_pending(sk, ssk, true); in mptcp_subflow_process_delegated()
3710 __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); in mptcp_subflow_process_delegated()
3711 mptcp_data_unlock(sk); in mptcp_subflow_process_delegated()
3714 mptcp_data_lock(sk); in mptcp_subflow_process_delegated()
3715 if (!sock_owned_by_user(sk)) in mptcp_subflow_process_delegated()
3716 __mptcp_sync_sndbuf(sk); in mptcp_subflow_process_delegated()
3718 __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags); in mptcp_subflow_process_delegated()
3719 mptcp_data_unlock(sk); in mptcp_subflow_process_delegated()
3725 static int mptcp_hash(struct sock *sk) in mptcp_hash() argument
3734 static void mptcp_unhash(struct sock *sk) in mptcp_unhash() argument
3739 static int mptcp_get_port(struct sock *sk, unsigned short snum) in mptcp_get_port() argument
3741 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_get_port()
3754 struct sock *sk; in mptcp_finish_connect() local
3757 sk = subflow->conn; in mptcp_finish_connect()
3758 msk = mptcp_sk(sk); in mptcp_finish_connect()
3760 pr_debug("msk=%p, token=%u\n", sk, subflow->token); in mptcp_finish_connect()
3774 void mptcp_sock_graft(struct sock *sk, struct socket *parent) in mptcp_sock_graft() argument
3776 write_lock_bh(&sk->sk_callback_lock); in mptcp_sock_graft()
3777 rcu_assign_pointer(sk->sk_wq, &parent->wq); in mptcp_sock_graft()
3778 sk_set_socket(sk, parent); in mptcp_sock_graft()
3779 write_unlock_bh(&sk->sk_callback_lock); in mptcp_sock_graft()
3785 static void mptcp_sock_check_graft(struct sock *sk, struct sock *ssk) in mptcp_sock_check_graft() argument
3789 write_lock_bh(&sk->sk_callback_lock); in mptcp_sock_check_graft()
3790 sock = sk->sk_socket; in mptcp_sock_check_graft()
3791 write_unlock_bh(&sk->sk_callback_lock); in mptcp_sock_check_graft()
3794 __mptcp_inherit_cgrp_data(sk, ssk); in mptcp_sock_check_graft()
3795 __mptcp_inherit_memcg(sk, ssk, GFP_ATOMIC); in mptcp_sock_check_graft()
3866 static void mptcp_shutdown(struct sock *sk, int how) in mptcp_shutdown() argument
3868 pr_debug("sk=%p, how=%d\n", sk, how); in mptcp_shutdown()
3870 if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) in mptcp_shutdown()
3871 __mptcp_wr_shutdown(sk); in mptcp_shutdown()
3876 const struct sock *sk = (void *)msk; in mptcp_ioctl_outq() local
3879 if (sk->sk_state == TCP_LISTEN) in mptcp_ioctl_outq()
3882 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) in mptcp_ioctl_outq()
3903 static int mptcp_ioctl(struct sock *sk, int cmd, int *karg) in mptcp_ioctl() argument
3905 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_ioctl()
3910 if (sk->sk_state == TCP_LISTEN) in mptcp_ioctl()
3913 lock_sock(sk); in mptcp_ioctl()
3914 if (mptcp_move_skbs(sk)) in mptcp_ioctl()
3916 *karg = mptcp_inq_hint(sk); in mptcp_ioctl()
3917 release_sock(sk); in mptcp_ioctl()
3920 slow = lock_sock_fast(sk); in mptcp_ioctl()
3922 unlock_sock_fast(sk, slow); in mptcp_ioctl()
3925 slow = lock_sock_fast(sk); in mptcp_ioctl()
3927 unlock_sock_fast(sk, slow); in mptcp_ioctl()
3936 static int mptcp_connect(struct sock *sk, struct sockaddr_unsized *uaddr, in mptcp_connect() argument
3940 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_connect()
3948 mptcp_set_state(sk, TCP_SYN_SENT); in mptcp_connect()
3958 if (mptcp_active_should_disable(sk)) in mptcp_connect()
3970 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE); in mptcp_connect()
3994 inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk)); in mptcp_connect()
4006 mptcp_set_state(sk, TCP_CLOSE); in mptcp_connect()
4010 mptcp_copy_inaddrs(sk, ssk); in mptcp_connect()
4049 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_bind()
4050 struct sock *ssk, *sk = sock->sk; in mptcp_bind() local
4053 lock_sock(sk); in mptcp_bind()
4060 if (sk->sk_family == AF_INET) in mptcp_bind()
4063 else if (sk->sk_family == AF_INET6) in mptcp_bind()
4067 mptcp_copy_inaddrs(sk, ssk); in mptcp_bind()
4070 release_sock(sk); in mptcp_bind()
4076 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_listen()
4077 struct sock *sk = sock->sk; in mptcp_listen() local
4083 lock_sock(sk); in mptcp_listen()
4095 mptcp_set_state(sk, TCP_LISTEN); in mptcp_listen()
4096 sock_set_flag(sk, SOCK_RCU_FREE); in mptcp_listen()
4101 mptcp_set_state(sk, inet_sk_state_load(ssk)); in mptcp_listen()
4104 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); in mptcp_listen()
4105 mptcp_copy_inaddrs(sk, ssk); in mptcp_listen()
4110 release_sock(sk); in mptcp_listen()
4114 static void mptcp_graft_subflows(struct sock *sk) in mptcp_graft_subflows() argument
4117 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_graft_subflows()
4128 mptcp_data_lock(sk); in mptcp_graft_subflows()
4130 mptcp_data_unlock(sk); in mptcp_graft_subflows()
4132 __mptcp_flush_join_list(sk, &join_list); in mptcp_graft_subflows()
4144 mptcp_sock_graft(ssk, sk->sk_socket); in mptcp_graft_subflows()
4146 if (!mem_cgroup_sk_enabled(sk)) in mptcp_graft_subflows()
4149 __mptcp_inherit_cgrp_data(sk, ssk); in mptcp_graft_subflows()
4150 __mptcp_inherit_memcg(sk, ssk, GFP_KERNEL); in mptcp_graft_subflows()
4156 if (mem_cgroup_sk_enabled(sk)) { in mptcp_graft_subflows()
4163 mptcp_data_lock(sk); in mptcp_graft_subflows()
4164 amt = sk_mem_pages(sk->sk_forward_alloc + in mptcp_graft_subflows()
4166 atomic_read(&sk->sk_rmem_alloc)) - in mptcp_graft_subflows()
4167 sk_mem_pages(sk->sk_forward_alloc + in mptcp_graft_subflows()
4168 atomic_read(&sk->sk_rmem_alloc)); in mptcp_graft_subflows()
4170 mptcp_data_unlock(sk); in mptcp_graft_subflows()
4173 mem_cgroup_sk_charge(sk, amt, gfp); in mptcp_graft_subflows()
4180 struct mptcp_sock *msk = mptcp_sk(sock->sk); in mptcp_stream_accept()
4247 WRITE_ONCE(newsock->sk->sk_socket->ops, in mptcp_stream_accept()
4248 mptcp_fallback_tcp_ops(newsock->sk)); in mptcp_stream_accept()
4257 struct sock *sk = (struct sock *)msk; in mptcp_check_writeable() local
4259 if (__mptcp_stream_is_writeable(sk, 1)) in mptcp_check_writeable()
4262 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); in mptcp_check_writeable()
4264 if (__mptcp_stream_is_writeable(sk, 1)) in mptcp_check_writeable()
4273 struct sock *sk = sock->sk; in mptcp_poll() local
4279 msk = mptcp_sk(sk); in mptcp_poll()
4282 state = inet_sk_state_load(sk); in mptcp_poll()
4293 shutdown = READ_ONCE(sk->sk_shutdown); in mptcp_poll()
4300 mask |= mptcp_check_readable(sk); in mptcp_poll()
4306 inet_test_bit(DEFER_CONNECT, sk)) { in mptcp_poll()
4313 if (READ_ONCE(sk->sk_err)) in mptcp_poll()
4319 static struct sk_buff *mptcp_recv_skb(struct sock *sk, u32 *off) in mptcp_recv_skb() argument
4321 struct mptcp_sock *msk = mptcp_sk(sk); in mptcp_recv_skb()
4326 mptcp_move_skbs(sk); in mptcp_recv_skb()
4328 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { in mptcp_recv_skb()
4334 mptcp_eat_recv_skb(sk, skb); in mptcp_recv_skb()
4343 static int __mptcp_read_sock(struct sock *sk, read_descriptor_t *desc, in __mptcp_read_sock() argument
4346 struct mptcp_sock *msk = mptcp_sk(sk); in __mptcp_read_sock()
4353 if (sk->sk_state == TCP_LISTEN) in __mptcp_read_sock()
4355 while ((skb = mptcp_recv_skb(sk, &offset)) != NULL) { in __mptcp_read_sock()
4377 mptcp_eat_recv_skb(sk, skb); in __mptcp_read_sock()
4386 mptcp_recv_skb(sk, &offset); in __mptcp_read_sock()
4393 static int mptcp_read_sock(struct sock *sk, read_descriptor_t *desc, in mptcp_read_sock() argument
4396 return __mptcp_read_sock(sk, desc, recv_actor, false); in mptcp_read_sock()
4399 static int __mptcp_splice_read(struct sock *sk, struct tcp_splice_state *tss) in __mptcp_splice_read() argument
4407 return mptcp_read_sock(sk, &rd_desc, tcp_splice_data_recv); in __mptcp_splice_read()
4434 struct sock *sk = sock->sk; in mptcp_splice_read() local
4445 lock_sock(sk); in mptcp_splice_read()
4447 mptcp_rps_record_subflows(mptcp_sk(sk)); in mptcp_splice_read()
4449 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK); in mptcp_splice_read()
4451 ret = __mptcp_splice_read(sk, &tss); in mptcp_splice_read()
4457 if (sock_flag(sk, SOCK_DONE)) in mptcp_splice_read()
4459 if (sk->sk_err) { in mptcp_splice_read()
4460 ret = sock_error(sk); in mptcp_splice_read()
4463 if (sk->sk_shutdown & RCV_SHUTDOWN) in mptcp_splice_read()
4465 if (sk->sk_state == TCP_CLOSE) { in mptcp_splice_read()
4481 if (!skb_queue_empty(&sk->sk_receive_queue)) in mptcp_splice_read()
4483 ret = sk_wait_data(sk, &timeo, NULL); in mptcp_splice_read()
4497 release_sock(sk); in mptcp_splice_read()
4498 lock_sock(sk); in mptcp_splice_read()
4500 if (sk->sk_err || sk->sk_state == TCP_CLOSE || in mptcp_splice_read()
4501 (sk->sk_shutdown & RCV_SHUTDOWN) || in mptcp_splice_read()
4506 release_sock(sk); in mptcp_splice_read()