1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <linux/sched/signal.h> 13 #include <linux/atomic.h> 14 #include <net/aligned_data.h> 15 #include <net/sock.h> 16 #include <net/inet_common.h> 17 #include <net/inet_hashtables.h> 18 #include <net/protocol.h> 19 #include <net/tcp_states.h> 20 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21 #include <net/transp_v6.h> 22 #endif 23 #include <net/mptcp.h> 24 #include <net/hotdata.h> 25 #include <net/xfrm.h> 26 #include <asm/ioctls.h> 27 #include "protocol.h" 28 #include "mib.h" 29 30 #define CREATE_TRACE_POINTS 31 #include <trace/events/mptcp.h> 32 33 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 34 struct mptcp6_sock { 35 struct mptcp_sock msk; 36 struct ipv6_pinfo np; 37 }; 38 #endif 39 40 enum { 41 MPTCP_CMSG_TS = BIT(0), 42 MPTCP_CMSG_INQ = BIT(1), 43 }; 44 45 static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp; 46 47 static void __mptcp_destroy_sock(struct sock *sk); 48 static void mptcp_check_send_data_fin(struct sock *sk); 49 50 DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions) = { 51 .bh_lock = INIT_LOCAL_LOCK(bh_lock), 52 }; 53 static struct net_device *mptcp_napi_dev; 54 55 /* Returns end sequence number of the receiver's advertised window */ 56 static u64 mptcp_wnd_end(const struct mptcp_sock *msk) 57 { 58 return READ_ONCE(msk->wnd_end); 59 } 60 61 static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk) 62 { 63 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 64 if (sk->sk_prot == &tcpv6_prot) 65 return &inet6_stream_ops; 66 #endif 67 WARN_ON_ONCE(sk->sk_prot != &tcp_prot); 68 return &inet_stream_ops; 69 } 70 71 static int __mptcp_socket_create(struct mptcp_sock *msk) 72 { 73 struct mptcp_subflow_context *subflow; 74 struct sock *sk = (struct sock *)msk; 75 struct socket *ssock; 76 int err; 77 78 err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock); 79 if (err) 80 return err; 81 82 msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; 83 WRITE_ONCE(msk->first, ssock->sk); 84 subflow = mptcp_subflow_ctx(ssock->sk); 85 list_add(&subflow->node, &msk->conn_list); 86 sock_hold(ssock->sk); 87 subflow->request_mptcp = 1; 88 subflow->subflow_id = msk->subflow_id++; 89 90 /* This is the first subflow, always with id 0 */ 91 WRITE_ONCE(subflow->local_id, 0); 92 mptcp_sock_graft(msk->first, sk->sk_socket); 93 iput(SOCK_INODE(ssock)); 94 95 return 0; 96 } 97 98 /* If the MPC handshake is not started, returns the first subflow, 99 * eventually allocating it. 100 */ 101 struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) 102 { 103 struct sock *sk = (struct sock *)msk; 104 int ret; 105 106 if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 107 return ERR_PTR(-EINVAL); 108 109 if (!msk->first) { 110 ret = __mptcp_socket_create(msk); 111 if (ret) 112 return ERR_PTR(ret); 113 } 114 115 return msk->first; 116 } 117 118 static void mptcp_drop(struct sock *sk, struct sk_buff *skb) 119 { 120 sk_drops_add(sk, skb); 121 __kfree_skb(skb); 122 } 123 124 static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 125 struct sk_buff *from) 126 { 127 bool fragstolen; 128 int delta; 129 130 if (unlikely(MPTCP_SKB_CB(to)->cant_coalesce) || 131 MPTCP_SKB_CB(from)->offset || 132 ((to->len + from->len) > (sk->sk_rcvbuf >> 3)) || 133 !skb_try_coalesce(to, from, &fragstolen, &delta)) 134 return false; 135 136 pr_debug("colesced seq %llx into %llx new len %d new end seq %llx\n", 137 MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, 138 to->len, MPTCP_SKB_CB(from)->end_seq); 139 MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; 140 141 /* note the fwd memory can reach a negative value after accounting 142 * for the delta, but the later skb free will restore a non 143 * negative one 144 */ 145 atomic_add(delta, &sk->sk_rmem_alloc); 146 sk_mem_charge(sk, delta); 147 kfree_skb_partial(from, fragstolen); 148 149 return true; 150 } 151 152 static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, 153 struct sk_buff *from) 154 { 155 if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) 156 return false; 157 158 return mptcp_try_coalesce((struct sock *)msk, to, from); 159 } 160 161 /* "inspired" by tcp_data_queue_ofo(), main differences: 162 * - use mptcp seqs 163 * - don't cope with sacks 164 */ 165 static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) 166 { 167 struct sock *sk = (struct sock *)msk; 168 struct rb_node **p, *parent; 169 u64 seq, end_seq, max_seq; 170 struct sk_buff *skb1; 171 172 seq = MPTCP_SKB_CB(skb)->map_seq; 173 end_seq = MPTCP_SKB_CB(skb)->end_seq; 174 max_seq = atomic64_read(&msk->rcv_wnd_sent); 175 176 pr_debug("msk=%p seq=%llx limit=%llx empty=%d\n", msk, seq, max_seq, 177 RB_EMPTY_ROOT(&msk->out_of_order_queue)); 178 if (after64(end_seq, max_seq)) { 179 /* out of window */ 180 mptcp_drop(sk, skb); 181 pr_debug("oow by %lld, rcv_wnd_sent %llu\n", 182 (unsigned long long)end_seq - (unsigned long)max_seq, 183 (unsigned long long)atomic64_read(&msk->rcv_wnd_sent)); 184 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); 185 return; 186 } 187 188 p = &msk->out_of_order_queue.rb_node; 189 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); 190 if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { 191 rb_link_node(&skb->rbnode, NULL, p); 192 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 193 msk->ooo_last_skb = skb; 194 goto end; 195 } 196 197 /* with 2 subflows, adding at end of ooo queue is quite likely 198 * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. 199 */ 200 if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { 201 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 202 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 203 return; 204 } 205 206 /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ 207 if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { 208 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 209 parent = &msk->ooo_last_skb->rbnode; 210 p = &parent->rb_right; 211 goto insert; 212 } 213 214 /* Find place to insert this segment. Handle overlaps on the way. */ 215 parent = NULL; 216 while (*p) { 217 parent = *p; 218 skb1 = rb_to_skb(parent); 219 if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 220 p = &parent->rb_left; 221 continue; 222 } 223 if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { 224 if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { 225 /* All the bits are present. Drop. */ 226 mptcp_drop(sk, skb); 227 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 228 return; 229 } 230 if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 231 /* partial overlap: 232 * | skb | 233 * | skb1 | 234 * continue traversing 235 */ 236 } else { 237 /* skb's seq == skb1's seq and skb covers skb1. 238 * Replace skb1 with skb. 239 */ 240 rb_replace_node(&skb1->rbnode, &skb->rbnode, 241 &msk->out_of_order_queue); 242 mptcp_drop(sk, skb1); 243 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 244 goto merge_right; 245 } 246 } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { 247 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 248 return; 249 } 250 p = &parent->rb_right; 251 } 252 253 insert: 254 /* Insert segment into RB tree. */ 255 rb_link_node(&skb->rbnode, parent, p); 256 rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 257 258 merge_right: 259 /* Remove other segments covered by skb. */ 260 while ((skb1 = skb_rb_next(skb)) != NULL) { 261 if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) 262 break; 263 rb_erase(&skb1->rbnode, &msk->out_of_order_queue); 264 mptcp_drop(sk, skb1); 265 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 266 } 267 /* If there is no skb after us, we are the last_skb ! */ 268 if (!skb1) 269 msk->ooo_last_skb = skb; 270 271 end: 272 skb_condense(skb); 273 skb_set_owner_r(skb, sk); 274 } 275 276 static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 277 struct sk_buff *skb, unsigned int offset, 278 size_t copy_len) 279 { 280 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 281 struct sock *sk = (struct sock *)msk; 282 struct sk_buff *tail; 283 bool has_rxtstamp; 284 285 __skb_unlink(skb, &ssk->sk_receive_queue); 286 287 skb_ext_reset(skb); 288 skb_orphan(skb); 289 290 /* try to fetch required memory from subflow */ 291 if (!sk_rmem_schedule(sk, skb, skb->truesize)) { 292 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED); 293 goto drop; 294 } 295 296 has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; 297 298 /* the skb map_seq accounts for the skb offset: 299 * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq 300 * value 301 */ 302 MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); 303 MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; 304 MPTCP_SKB_CB(skb)->offset = offset; 305 MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; 306 MPTCP_SKB_CB(skb)->cant_coalesce = 0; 307 308 if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { 309 /* in sequence */ 310 msk->bytes_received += copy_len; 311 WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); 312 tail = skb_peek_tail(&sk->sk_receive_queue); 313 if (tail && mptcp_try_coalesce(sk, tail, skb)) 314 return true; 315 316 skb_set_owner_r(skb, sk); 317 __skb_queue_tail(&sk->sk_receive_queue, skb); 318 return true; 319 } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { 320 mptcp_data_queue_ofo(msk, skb); 321 return false; 322 } 323 324 /* old data, keep it simple and drop the whole pkt, sender 325 * will retransmit as needed, if needed. 326 */ 327 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 328 drop: 329 mptcp_drop(sk, skb); 330 return false; 331 } 332 333 static void mptcp_stop_rtx_timer(struct sock *sk) 334 { 335 struct inet_connection_sock *icsk = inet_csk(sk); 336 337 sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 338 mptcp_sk(sk)->timer_ival = 0; 339 } 340 341 static void mptcp_close_wake_up(struct sock *sk) 342 { 343 if (sock_flag(sk, SOCK_DEAD)) 344 return; 345 346 sk->sk_state_change(sk); 347 if (sk->sk_shutdown == SHUTDOWN_MASK || 348 sk->sk_state == TCP_CLOSE) 349 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 350 else 351 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 352 } 353 354 /* called under the msk socket lock */ 355 static bool mptcp_pending_data_fin_ack(struct sock *sk) 356 { 357 struct mptcp_sock *msk = mptcp_sk(sk); 358 359 return ((1 << sk->sk_state) & 360 (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 361 msk->write_seq == READ_ONCE(msk->snd_una); 362 } 363 364 static void mptcp_check_data_fin_ack(struct sock *sk) 365 { 366 struct mptcp_sock *msk = mptcp_sk(sk); 367 368 /* Look for an acknowledged DATA_FIN */ 369 if (mptcp_pending_data_fin_ack(sk)) { 370 WRITE_ONCE(msk->snd_data_fin_enable, 0); 371 372 switch (sk->sk_state) { 373 case TCP_FIN_WAIT1: 374 mptcp_set_state(sk, TCP_FIN_WAIT2); 375 break; 376 case TCP_CLOSING: 377 case TCP_LAST_ACK: 378 mptcp_set_state(sk, TCP_CLOSE); 379 break; 380 } 381 382 mptcp_close_wake_up(sk); 383 } 384 } 385 386 /* can be called with no lock acquired */ 387 static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 388 { 389 struct mptcp_sock *msk = mptcp_sk(sk); 390 391 if (READ_ONCE(msk->rcv_data_fin) && 392 ((1 << inet_sk_state_load(sk)) & 393 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 394 u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 395 396 if (READ_ONCE(msk->ack_seq) == rcv_data_fin_seq) { 397 if (seq) 398 *seq = rcv_data_fin_seq; 399 400 return true; 401 } 402 } 403 404 return false; 405 } 406 407 static void mptcp_set_datafin_timeout(struct sock *sk) 408 { 409 struct inet_connection_sock *icsk = inet_csk(sk); 410 u32 retransmits; 411 412 retransmits = min_t(u32, icsk->icsk_retransmits, 413 ilog2(TCP_RTO_MAX / TCP_RTO_MIN)); 414 415 mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits; 416 } 417 418 static void __mptcp_set_timeout(struct sock *sk, long tout) 419 { 420 mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 421 } 422 423 static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) 424 { 425 const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 426 427 return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? 428 icsk_timeout(inet_csk(ssk)) - jiffies : 0; 429 } 430 431 static void mptcp_set_timeout(struct sock *sk) 432 { 433 struct mptcp_subflow_context *subflow; 434 long tout = 0; 435 436 mptcp_for_each_subflow(mptcp_sk(sk), subflow) 437 tout = max(tout, mptcp_timeout_from_subflow(subflow)); 438 __mptcp_set_timeout(sk, tout); 439 } 440 441 static inline bool tcp_can_send_ack(const struct sock *ssk) 442 { 443 return !((1 << inet_sk_state_load(ssk)) & 444 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 445 } 446 447 void __mptcp_subflow_send_ack(struct sock *ssk) 448 { 449 if (tcp_can_send_ack(ssk)) 450 tcp_send_ack(ssk); 451 } 452 453 static void mptcp_subflow_send_ack(struct sock *ssk) 454 { 455 bool slow; 456 457 slow = lock_sock_fast(ssk); 458 __mptcp_subflow_send_ack(ssk); 459 unlock_sock_fast(ssk, slow); 460 } 461 462 static void mptcp_send_ack(struct mptcp_sock *msk) 463 { 464 struct mptcp_subflow_context *subflow; 465 466 mptcp_for_each_subflow(msk, subflow) 467 mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); 468 } 469 470 static void mptcp_subflow_cleanup_rbuf(struct sock *ssk, int copied) 471 { 472 bool slow; 473 474 slow = lock_sock_fast(ssk); 475 if (tcp_can_send_ack(ssk)) 476 tcp_cleanup_rbuf(ssk, copied); 477 unlock_sock_fast(ssk, slow); 478 } 479 480 static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) 481 { 482 const struct inet_connection_sock *icsk = inet_csk(ssk); 483 u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending); 484 const struct tcp_sock *tp = tcp_sk(ssk); 485 486 return (ack_pending & ICSK_ACK_SCHED) && 487 ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) > 488 READ_ONCE(icsk->icsk_ack.rcv_mss)) || 489 (rx_empty && ack_pending & 490 (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED))); 491 } 492 493 static void mptcp_cleanup_rbuf(struct mptcp_sock *msk, int copied) 494 { 495 int old_space = READ_ONCE(msk->old_wspace); 496 struct mptcp_subflow_context *subflow; 497 struct sock *sk = (struct sock *)msk; 498 int space = __mptcp_space(sk); 499 bool cleanup, rx_empty; 500 501 cleanup = (space > 0) && (space >= (old_space << 1)) && copied; 502 rx_empty = !sk_rmem_alloc_get(sk) && copied; 503 504 mptcp_for_each_subflow(msk, subflow) { 505 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 506 507 if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) 508 mptcp_subflow_cleanup_rbuf(ssk, copied); 509 } 510 } 511 512 static bool mptcp_check_data_fin(struct sock *sk) 513 { 514 struct mptcp_sock *msk = mptcp_sk(sk); 515 u64 rcv_data_fin_seq; 516 bool ret = false; 517 518 /* Need to ack a DATA_FIN received from a peer while this side 519 * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 520 * msk->rcv_data_fin was set when parsing the incoming options 521 * at the subflow level and the msk lock was not held, so this 522 * is the first opportunity to act on the DATA_FIN and change 523 * the msk state. 524 * 525 * If we are caught up to the sequence number of the incoming 526 * DATA_FIN, send the DATA_ACK now and do state transition. If 527 * not caught up, do nothing and let the recv code send DATA_ACK 528 * when catching up. 529 */ 530 531 if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 532 WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); 533 WRITE_ONCE(msk->rcv_data_fin, 0); 534 535 WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN); 536 smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 537 538 switch (sk->sk_state) { 539 case TCP_ESTABLISHED: 540 mptcp_set_state(sk, TCP_CLOSE_WAIT); 541 break; 542 case TCP_FIN_WAIT1: 543 mptcp_set_state(sk, TCP_CLOSING); 544 break; 545 case TCP_FIN_WAIT2: 546 mptcp_set_state(sk, TCP_CLOSE); 547 break; 548 default: 549 /* Other states not expected */ 550 WARN_ON_ONCE(1); 551 break; 552 } 553 554 ret = true; 555 if (!__mptcp_check_fallback(msk)) 556 mptcp_send_ack(msk); 557 mptcp_close_wake_up(sk); 558 } 559 return ret; 560 } 561 562 static void mptcp_dss_corruption(struct mptcp_sock *msk, struct sock *ssk) 563 { 564 if (READ_ONCE(msk->allow_infinite_fallback)) { 565 MPTCP_INC_STATS(sock_net(ssk), 566 MPTCP_MIB_DSSCORRUPTIONFALLBACK); 567 mptcp_do_fallback(ssk); 568 } else { 569 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSCORRUPTIONRESET); 570 mptcp_subflow_reset(ssk); 571 } 572 } 573 574 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 575 struct sock *ssk) 576 { 577 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 578 struct sock *sk = (struct sock *)msk; 579 bool more_data_avail; 580 struct tcp_sock *tp; 581 bool ret = false; 582 583 pr_debug("msk=%p ssk=%p\n", msk, ssk); 584 tp = tcp_sk(ssk); 585 do { 586 u32 map_remaining, offset; 587 u32 seq = tp->copied_seq; 588 struct sk_buff *skb; 589 bool fin; 590 591 if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf) 592 break; 593 594 /* try to move as much data as available */ 595 map_remaining = subflow->map_data_len - 596 mptcp_subflow_get_map_offset(subflow); 597 598 skb = skb_peek(&ssk->sk_receive_queue); 599 if (unlikely(!skb)) 600 break; 601 602 if (__mptcp_check_fallback(msk)) { 603 /* Under fallback skbs have no MPTCP extension and TCP could 604 * collapse them between the dummy map creation and the 605 * current dequeue. Be sure to adjust the map size. 606 */ 607 map_remaining = skb->len; 608 subflow->map_data_len = skb->len; 609 } 610 611 offset = seq - TCP_SKB_CB(skb)->seq; 612 fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 613 if (fin) 614 seq++; 615 616 if (offset < skb->len) { 617 size_t len = skb->len - offset; 618 619 ret = __mptcp_move_skb(msk, ssk, skb, offset, len) || ret; 620 seq += len; 621 622 if (unlikely(map_remaining < len)) { 623 DEBUG_NET_WARN_ON_ONCE(1); 624 mptcp_dss_corruption(msk, ssk); 625 } 626 } else { 627 if (unlikely(!fin)) { 628 DEBUG_NET_WARN_ON_ONCE(1); 629 mptcp_dss_corruption(msk, ssk); 630 } 631 632 sk_eat_skb(ssk, skb); 633 } 634 635 WRITE_ONCE(tp->copied_seq, seq); 636 more_data_avail = mptcp_subflow_data_available(ssk); 637 638 } while (more_data_avail); 639 640 if (ret) 641 msk->last_data_recv = tcp_jiffies32; 642 return ret; 643 } 644 645 static bool __mptcp_ofo_queue(struct mptcp_sock *msk) 646 { 647 struct sock *sk = (struct sock *)msk; 648 struct sk_buff *skb, *tail; 649 bool moved = false; 650 struct rb_node *p; 651 u64 end_seq; 652 653 p = rb_first(&msk->out_of_order_queue); 654 pr_debug("msk=%p empty=%d\n", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); 655 while (p) { 656 skb = rb_to_skb(p); 657 if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) 658 break; 659 660 p = rb_next(p); 661 rb_erase(&skb->rbnode, &msk->out_of_order_queue); 662 663 if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, 664 msk->ack_seq))) { 665 mptcp_drop(sk, skb); 666 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 667 continue; 668 } 669 670 end_seq = MPTCP_SKB_CB(skb)->end_seq; 671 tail = skb_peek_tail(&sk->sk_receive_queue); 672 if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { 673 int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 674 675 /* skip overlapping data, if any */ 676 pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d\n", 677 MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, 678 delta); 679 MPTCP_SKB_CB(skb)->offset += delta; 680 MPTCP_SKB_CB(skb)->map_seq += delta; 681 __skb_queue_tail(&sk->sk_receive_queue, skb); 682 } 683 msk->bytes_received += end_seq - msk->ack_seq; 684 WRITE_ONCE(msk->ack_seq, end_seq); 685 moved = true; 686 } 687 return moved; 688 } 689 690 static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) 691 { 692 int err = sock_error(ssk); 693 int ssk_state; 694 695 if (!err) 696 return false; 697 698 /* only propagate errors on fallen-back sockets or 699 * on MPC connect 700 */ 701 if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk))) 702 return false; 703 704 /* We need to propagate only transition to CLOSE state. 705 * Orphaned socket will see such state change via 706 * subflow_sched_work_if_closed() and that path will properly 707 * destroy the msk as needed. 708 */ 709 ssk_state = inet_sk_state_load(ssk); 710 if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD)) 711 mptcp_set_state(sk, ssk_state); 712 WRITE_ONCE(sk->sk_err, -err); 713 714 /* This barrier is coupled with smp_rmb() in mptcp_poll() */ 715 smp_wmb(); 716 sk_error_report(sk); 717 return true; 718 } 719 720 void __mptcp_error_report(struct sock *sk) 721 { 722 struct mptcp_subflow_context *subflow; 723 struct mptcp_sock *msk = mptcp_sk(sk); 724 725 mptcp_for_each_subflow(msk, subflow) 726 if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) 727 break; 728 } 729 730 /* In most cases we will be able to lock the mptcp socket. If its already 731 * owned, we need to defer to the work queue to avoid ABBA deadlock. 732 */ 733 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 734 { 735 struct sock *sk = (struct sock *)msk; 736 bool moved; 737 738 moved = __mptcp_move_skbs_from_subflow(msk, ssk); 739 __mptcp_ofo_queue(msk); 740 if (unlikely(ssk->sk_err)) { 741 if (!sock_owned_by_user(sk)) 742 __mptcp_error_report(sk); 743 else 744 __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags); 745 } 746 747 /* If the moves have caught up with the DATA_FIN sequence number 748 * it's time to ack the DATA_FIN and change socket state, but 749 * this is not a good place to change state. Let the workqueue 750 * do it. 751 */ 752 if (mptcp_pending_data_fin(sk, NULL)) 753 mptcp_schedule_work(sk); 754 return moved; 755 } 756 757 static void __mptcp_rcvbuf_update(struct sock *sk, struct sock *ssk) 758 { 759 if (unlikely(ssk->sk_rcvbuf > sk->sk_rcvbuf)) 760 WRITE_ONCE(sk->sk_rcvbuf, ssk->sk_rcvbuf); 761 } 762 763 static void __mptcp_data_ready(struct sock *sk, struct sock *ssk) 764 { 765 struct mptcp_sock *msk = mptcp_sk(sk); 766 767 __mptcp_rcvbuf_update(sk, ssk); 768 769 /* Wake-up the reader only for in-sequence data */ 770 if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk)) 771 sk->sk_data_ready(sk); 772 } 773 774 void mptcp_data_ready(struct sock *sk, struct sock *ssk) 775 { 776 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 777 778 /* The peer can send data while we are shutting down this 779 * subflow at msk destruction time, but we must avoid enqueuing 780 * more data to the msk receive queue 781 */ 782 if (unlikely(subflow->disposable)) 783 return; 784 785 mptcp_data_lock(sk); 786 if (!sock_owned_by_user(sk)) 787 __mptcp_data_ready(sk, ssk); 788 else 789 __set_bit(MPTCP_DEQUEUE, &mptcp_sk(sk)->cb_flags); 790 mptcp_data_unlock(sk); 791 } 792 793 static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk) 794 { 795 mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq); 796 WRITE_ONCE(msk->allow_infinite_fallback, false); 797 mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); 798 } 799 800 static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) 801 { 802 struct sock *sk = (struct sock *)msk; 803 804 if (sk->sk_state != TCP_ESTABLISHED) 805 return false; 806 807 /* attach to msk socket only after we are sure we will deal with it 808 * at close time 809 */ 810 if (sk->sk_socket && !ssk->sk_socket) 811 mptcp_sock_graft(ssk, sk->sk_socket); 812 813 mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++; 814 mptcp_sockopt_sync_locked(msk, ssk); 815 mptcp_subflow_joined(msk, ssk); 816 mptcp_stop_tout_timer(sk); 817 __mptcp_propagate_sndbuf(sk, ssk); 818 return true; 819 } 820 821 static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list) 822 { 823 struct mptcp_subflow_context *tmp, *subflow; 824 struct mptcp_sock *msk = mptcp_sk(sk); 825 826 list_for_each_entry_safe(subflow, tmp, join_list, node) { 827 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 828 bool slow = lock_sock_fast(ssk); 829 830 list_move_tail(&subflow->node, &msk->conn_list); 831 if (!__mptcp_finish_join(msk, ssk)) 832 mptcp_subflow_reset(ssk); 833 unlock_sock_fast(ssk, slow); 834 } 835 } 836 837 static bool mptcp_rtx_timer_pending(struct sock *sk) 838 { 839 return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 840 } 841 842 static void mptcp_reset_rtx_timer(struct sock *sk) 843 { 844 struct inet_connection_sock *icsk = inet_csk(sk); 845 unsigned long tout; 846 847 /* prevent rescheduling on close */ 848 if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 849 return; 850 851 tout = mptcp_sk(sk)->timer_ival; 852 sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 853 } 854 855 bool mptcp_schedule_work(struct sock *sk) 856 { 857 if (inet_sk_state_load(sk) != TCP_CLOSE && 858 schedule_work(&mptcp_sk(sk)->work)) { 859 /* each subflow already holds a reference to the sk, and the 860 * workqueue is invoked by a subflow, so sk can't go away here. 861 */ 862 sock_hold(sk); 863 return true; 864 } 865 return false; 866 } 867 868 static bool mptcp_skb_can_collapse_to(u64 write_seq, 869 const struct sk_buff *skb, 870 const struct mptcp_ext *mpext) 871 { 872 if (!tcp_skb_can_collapse_to(skb)) 873 return false; 874 875 /* can collapse only if MPTCP level sequence is in order and this 876 * mapping has not been xmitted yet 877 */ 878 return mpext && mpext->data_seq + mpext->data_len == write_seq && 879 !mpext->frozen; 880 } 881 882 /* we can append data to the given data frag if: 883 * - there is space available in the backing page_frag 884 * - the data frag tail matches the current page_frag free offset 885 * - the data frag end sequence number matches the current write seq 886 */ 887 static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 888 const struct page_frag *pfrag, 889 const struct mptcp_data_frag *df) 890 { 891 return df && pfrag->page == df->page && 892 pfrag->size - pfrag->offset > 0 && 893 pfrag->offset == (df->offset + df->data_len) && 894 df->data_seq + df->data_len == msk->write_seq; 895 } 896 897 static void dfrag_uncharge(struct sock *sk, int len) 898 { 899 sk_mem_uncharge(sk, len); 900 sk_wmem_queued_add(sk, -len); 901 } 902 903 static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 904 { 905 int len = dfrag->data_len + dfrag->overhead; 906 907 list_del(&dfrag->list); 908 dfrag_uncharge(sk, len); 909 put_page(dfrag->page); 910 } 911 912 /* called under both the msk socket lock and the data lock */ 913 static void __mptcp_clean_una(struct sock *sk) 914 { 915 struct mptcp_sock *msk = mptcp_sk(sk); 916 struct mptcp_data_frag *dtmp, *dfrag; 917 u64 snd_una; 918 919 snd_una = msk->snd_una; 920 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 921 if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 922 break; 923 924 if (unlikely(dfrag == msk->first_pending)) { 925 /* in recovery mode can see ack after the current snd head */ 926 if (WARN_ON_ONCE(!msk->recovery)) 927 break; 928 929 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 930 } 931 932 dfrag_clear(sk, dfrag); 933 } 934 935 dfrag = mptcp_rtx_head(sk); 936 if (dfrag && after64(snd_una, dfrag->data_seq)) { 937 u64 delta = snd_una - dfrag->data_seq; 938 939 /* prevent wrap around in recovery mode */ 940 if (unlikely(delta > dfrag->already_sent)) { 941 if (WARN_ON_ONCE(!msk->recovery)) 942 goto out; 943 if (WARN_ON_ONCE(delta > dfrag->data_len)) 944 goto out; 945 dfrag->already_sent += delta - dfrag->already_sent; 946 } 947 948 dfrag->data_seq += delta; 949 dfrag->offset += delta; 950 dfrag->data_len -= delta; 951 dfrag->already_sent -= delta; 952 953 dfrag_uncharge(sk, delta); 954 } 955 956 /* all retransmitted data acked, recovery completed */ 957 if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) 958 msk->recovery = false; 959 960 out: 961 if (snd_una == msk->snd_nxt && snd_una == msk->write_seq) { 962 if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) 963 mptcp_stop_rtx_timer(sk); 964 } else { 965 mptcp_reset_rtx_timer(sk); 966 } 967 968 if (mptcp_pending_data_fin_ack(sk)) 969 mptcp_schedule_work(sk); 970 } 971 972 static void __mptcp_clean_una_wakeup(struct sock *sk) 973 { 974 lockdep_assert_held_once(&sk->sk_lock.slock); 975 976 __mptcp_clean_una(sk); 977 mptcp_write_space(sk); 978 } 979 980 static void mptcp_clean_una_wakeup(struct sock *sk) 981 { 982 mptcp_data_lock(sk); 983 __mptcp_clean_una_wakeup(sk); 984 mptcp_data_unlock(sk); 985 } 986 987 static void mptcp_enter_memory_pressure(struct sock *sk) 988 { 989 struct mptcp_subflow_context *subflow; 990 struct mptcp_sock *msk = mptcp_sk(sk); 991 bool first = true; 992 993 mptcp_for_each_subflow(msk, subflow) { 994 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 995 996 if (first) 997 tcp_enter_memory_pressure(ssk); 998 sk_stream_moderate_sndbuf(ssk); 999 1000 first = false; 1001 } 1002 __mptcp_sync_sndbuf(sk); 1003 } 1004 1005 /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 1006 * data 1007 */ 1008 static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 1009 { 1010 if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 1011 pfrag, sk->sk_allocation))) 1012 return true; 1013 1014 mptcp_enter_memory_pressure(sk); 1015 return false; 1016 } 1017 1018 static struct mptcp_data_frag * 1019 mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 1020 int orig_offset) 1021 { 1022 int offset = ALIGN(orig_offset, sizeof(long)); 1023 struct mptcp_data_frag *dfrag; 1024 1025 dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 1026 dfrag->data_len = 0; 1027 dfrag->data_seq = msk->write_seq; 1028 dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 1029 dfrag->offset = offset + sizeof(struct mptcp_data_frag); 1030 dfrag->already_sent = 0; 1031 dfrag->page = pfrag->page; 1032 1033 return dfrag; 1034 } 1035 1036 struct mptcp_sendmsg_info { 1037 int mss_now; 1038 int size_goal; 1039 u16 limit; 1040 u16 sent; 1041 unsigned int flags; 1042 bool data_lock_held; 1043 }; 1044 1045 static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk, 1046 u64 data_seq, int avail_size) 1047 { 1048 u64 window_end = mptcp_wnd_end(msk); 1049 u64 mptcp_snd_wnd; 1050 1051 if (__mptcp_check_fallback(msk)) 1052 return avail_size; 1053 1054 mptcp_snd_wnd = window_end - data_seq; 1055 avail_size = min_t(unsigned int, mptcp_snd_wnd, avail_size); 1056 1057 if (unlikely(tcp_sk(ssk)->snd_wnd < mptcp_snd_wnd)) { 1058 tcp_sk(ssk)->snd_wnd = min_t(u64, U32_MAX, mptcp_snd_wnd); 1059 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_SNDWNDSHARED); 1060 } 1061 1062 return avail_size; 1063 } 1064 1065 static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) 1066 { 1067 struct skb_ext *mpext = __skb_ext_alloc(gfp); 1068 1069 if (!mpext) 1070 return false; 1071 __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); 1072 return true; 1073 } 1074 1075 static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) 1076 { 1077 struct sk_buff *skb; 1078 1079 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 1080 if (likely(skb)) { 1081 if (likely(__mptcp_add_ext(skb, gfp))) { 1082 skb_reserve(skb, MAX_TCP_HEADER); 1083 skb->ip_summed = CHECKSUM_PARTIAL; 1084 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 1085 return skb; 1086 } 1087 __kfree_skb(skb); 1088 } else { 1089 mptcp_enter_memory_pressure(sk); 1090 } 1091 return NULL; 1092 } 1093 1094 static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) 1095 { 1096 struct sk_buff *skb; 1097 1098 skb = __mptcp_do_alloc_tx_skb(sk, gfp); 1099 if (!skb) 1100 return NULL; 1101 1102 if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1103 tcp_skb_entail(ssk, skb); 1104 return skb; 1105 } 1106 tcp_skb_tsorted_anchor_cleanup(skb); 1107 kfree_skb(skb); 1108 return NULL; 1109 } 1110 1111 static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) 1112 { 1113 gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation; 1114 1115 return __mptcp_alloc_tx_skb(sk, ssk, gfp); 1116 } 1117 1118 /* note: this always recompute the csum on the whole skb, even 1119 * if we just appended a single frag. More status info needed 1120 */ 1121 static void mptcp_update_data_checksum(struct sk_buff *skb, int added) 1122 { 1123 struct mptcp_ext *mpext = mptcp_get_ext(skb); 1124 __wsum csum = ~csum_unfold(mpext->csum); 1125 int offset = skb->len - added; 1126 1127 mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset)); 1128 } 1129 1130 static void mptcp_update_infinite_map(struct mptcp_sock *msk, 1131 struct sock *ssk, 1132 struct mptcp_ext *mpext) 1133 { 1134 if (!mpext) 1135 return; 1136 1137 mpext->infinite_map = 1; 1138 mpext->data_len = 0; 1139 1140 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPTX); 1141 mptcp_subflow_ctx(ssk)->send_infinite_map = 0; 1142 pr_fallback(msk); 1143 mptcp_do_fallback(ssk); 1144 } 1145 1146 #define MPTCP_MAX_GSO_SIZE (GSO_LEGACY_MAX_SIZE - (MAX_TCP_HEADER + 1)) 1147 1148 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 1149 struct mptcp_data_frag *dfrag, 1150 struct mptcp_sendmsg_info *info) 1151 { 1152 u64 data_seq = dfrag->data_seq + info->sent; 1153 int offset = dfrag->offset + info->sent; 1154 struct mptcp_sock *msk = mptcp_sk(sk); 1155 bool zero_window_probe = false; 1156 struct mptcp_ext *mpext = NULL; 1157 bool can_coalesce = false; 1158 bool reuse_skb = true; 1159 struct sk_buff *skb; 1160 size_t copy; 1161 int i; 1162 1163 pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u\n", 1164 msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); 1165 1166 if (WARN_ON_ONCE(info->sent > info->limit || 1167 info->limit > dfrag->data_len)) 1168 return 0; 1169 1170 if (unlikely(!__tcp_can_send(ssk))) 1171 return -EAGAIN; 1172 1173 /* compute send limit */ 1174 if (unlikely(ssk->sk_gso_max_size > MPTCP_MAX_GSO_SIZE)) 1175 ssk->sk_gso_max_size = MPTCP_MAX_GSO_SIZE; 1176 info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); 1177 copy = info->size_goal; 1178 1179 skb = tcp_write_queue_tail(ssk); 1180 if (skb && copy > skb->len) { 1181 /* Limit the write to the size available in the 1182 * current skb, if any, so that we create at most a new skb. 1183 * Explicitly tells TCP internals to avoid collapsing on later 1184 * queue management operation, to avoid breaking the ext <-> 1185 * SSN association set here 1186 */ 1187 mpext = mptcp_get_ext(skb); 1188 if (!mptcp_skb_can_collapse_to(data_seq, skb, mpext)) { 1189 TCP_SKB_CB(skb)->eor = 1; 1190 tcp_mark_push(tcp_sk(ssk), skb); 1191 goto alloc_skb; 1192 } 1193 1194 i = skb_shinfo(skb)->nr_frags; 1195 can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset); 1196 if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) { 1197 tcp_mark_push(tcp_sk(ssk), skb); 1198 goto alloc_skb; 1199 } 1200 1201 copy -= skb->len; 1202 } else { 1203 alloc_skb: 1204 skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); 1205 if (!skb) 1206 return -ENOMEM; 1207 1208 i = skb_shinfo(skb)->nr_frags; 1209 reuse_skb = false; 1210 mpext = mptcp_get_ext(skb); 1211 } 1212 1213 /* Zero window and all data acked? Probe. */ 1214 copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy); 1215 if (copy == 0) { 1216 u64 snd_una = READ_ONCE(msk->snd_una); 1217 1218 if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) { 1219 tcp_remove_empty_skb(ssk); 1220 return 0; 1221 } 1222 1223 zero_window_probe = true; 1224 data_seq = snd_una - 1; 1225 copy = 1; 1226 } 1227 1228 copy = min_t(size_t, copy, info->limit - info->sent); 1229 if (!sk_wmem_schedule(ssk, copy)) { 1230 tcp_remove_empty_skb(ssk); 1231 return -ENOMEM; 1232 } 1233 1234 if (can_coalesce) { 1235 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1236 } else { 1237 get_page(dfrag->page); 1238 skb_fill_page_desc(skb, i, dfrag->page, offset, copy); 1239 } 1240 1241 skb->len += copy; 1242 skb->data_len += copy; 1243 skb->truesize += copy; 1244 sk_wmem_queued_add(ssk, copy); 1245 sk_mem_charge(ssk, copy); 1246 WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy); 1247 TCP_SKB_CB(skb)->end_seq += copy; 1248 tcp_skb_pcount_set(skb, 0); 1249 1250 /* on skb reuse we just need to update the DSS len */ 1251 if (reuse_skb) { 1252 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1253 mpext->data_len += copy; 1254 goto out; 1255 } 1256 1257 memset(mpext, 0, sizeof(*mpext)); 1258 mpext->data_seq = data_seq; 1259 mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 1260 mpext->data_len = copy; 1261 mpext->use_map = 1; 1262 mpext->dsn64 = 1; 1263 1264 pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d\n", 1265 mpext->data_seq, mpext->subflow_seq, mpext->data_len, 1266 mpext->dsn64); 1267 1268 if (zero_window_probe) { 1269 mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 1270 mpext->frozen = 1; 1271 if (READ_ONCE(msk->csum_enabled)) 1272 mptcp_update_data_checksum(skb, copy); 1273 tcp_push_pending_frames(ssk); 1274 return 0; 1275 } 1276 out: 1277 if (READ_ONCE(msk->csum_enabled)) 1278 mptcp_update_data_checksum(skb, copy); 1279 if (mptcp_subflow_ctx(ssk)->send_infinite_map) 1280 mptcp_update_infinite_map(msk, ssk, mpext); 1281 trace_mptcp_sendmsg_frag(mpext); 1282 mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 1283 return copy; 1284 } 1285 1286 #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ 1287 sizeof(struct tcphdr) - \ 1288 MAX_TCP_OPTION_SPACE - \ 1289 sizeof(struct ipv6hdr) - \ 1290 sizeof(struct frag_hdr)) 1291 1292 struct subflow_send_info { 1293 struct sock *ssk; 1294 u64 linger_time; 1295 }; 1296 1297 void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) 1298 { 1299 if (!subflow->stale) 1300 return; 1301 1302 subflow->stale = 0; 1303 MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); 1304 } 1305 1306 bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) 1307 { 1308 if (unlikely(subflow->stale)) { 1309 u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); 1310 1311 if (subflow->stale_rcv_tstamp == rcv_tstamp) 1312 return false; 1313 1314 mptcp_subflow_set_active(subflow); 1315 } 1316 return __mptcp_subflow_active(subflow); 1317 } 1318 1319 #define SSK_MODE_ACTIVE 0 1320 #define SSK_MODE_BACKUP 1 1321 #define SSK_MODE_MAX 2 1322 1323 /* implement the mptcp packet scheduler; 1324 * returns the subflow that will transmit the next DSS 1325 * additionally updates the rtx timeout 1326 */ 1327 struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 1328 { 1329 struct subflow_send_info send_info[SSK_MODE_MAX]; 1330 struct mptcp_subflow_context *subflow; 1331 struct sock *sk = (struct sock *)msk; 1332 u32 pace, burst, wmem; 1333 int i, nr_active = 0; 1334 struct sock *ssk; 1335 u64 linger_time; 1336 long tout = 0; 1337 1338 /* pick the subflow with the lower wmem/wspace ratio */ 1339 for (i = 0; i < SSK_MODE_MAX; ++i) { 1340 send_info[i].ssk = NULL; 1341 send_info[i].linger_time = -1; 1342 } 1343 1344 mptcp_for_each_subflow(msk, subflow) { 1345 bool backup = subflow->backup || subflow->request_bkup; 1346 1347 trace_mptcp_subflow_get_send(subflow); 1348 ssk = mptcp_subflow_tcp_sock(subflow); 1349 if (!mptcp_subflow_active(subflow)) 1350 continue; 1351 1352 tout = max(tout, mptcp_timeout_from_subflow(subflow)); 1353 nr_active += !backup; 1354 pace = subflow->avg_pacing_rate; 1355 if (unlikely(!pace)) { 1356 /* init pacing rate from socket */ 1357 subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); 1358 pace = subflow->avg_pacing_rate; 1359 if (!pace) 1360 continue; 1361 } 1362 1363 linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace); 1364 if (linger_time < send_info[backup].linger_time) { 1365 send_info[backup].ssk = ssk; 1366 send_info[backup].linger_time = linger_time; 1367 } 1368 } 1369 __mptcp_set_timeout(sk, tout); 1370 1371 /* pick the best backup if no other subflow is active */ 1372 if (!nr_active) 1373 send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk; 1374 1375 /* According to the blest algorithm, to avoid HoL blocking for the 1376 * faster flow, we need to: 1377 * - estimate the faster flow linger time 1378 * - use the above to estimate the amount of byte transferred 1379 * by the faster flow 1380 * - check that the amount of queued data is greter than the above, 1381 * otherwise do not use the picked, slower, subflow 1382 * We select the subflow with the shorter estimated time to flush 1383 * the queued mem, which basically ensure the above. We just need 1384 * to check that subflow has a non empty cwin. 1385 */ 1386 ssk = send_info[SSK_MODE_ACTIVE].ssk; 1387 if (!ssk || !sk_stream_memory_free(ssk)) 1388 return NULL; 1389 1390 burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt); 1391 wmem = READ_ONCE(ssk->sk_wmem_queued); 1392 if (!burst) 1393 return ssk; 1394 1395 subflow = mptcp_subflow_ctx(ssk); 1396 subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + 1397 READ_ONCE(ssk->sk_pacing_rate) * burst, 1398 burst + wmem); 1399 msk->snd_burst = burst; 1400 return ssk; 1401 } 1402 1403 static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) 1404 { 1405 tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); 1406 release_sock(ssk); 1407 } 1408 1409 static void mptcp_update_post_push(struct mptcp_sock *msk, 1410 struct mptcp_data_frag *dfrag, 1411 u32 sent) 1412 { 1413 u64 snd_nxt_new = dfrag->data_seq; 1414 1415 dfrag->already_sent += sent; 1416 1417 msk->snd_burst -= sent; 1418 1419 snd_nxt_new += dfrag->already_sent; 1420 1421 /* snd_nxt_new can be smaller than snd_nxt in case mptcp 1422 * is recovering after a failover. In that event, this re-sends 1423 * old segments. 1424 * 1425 * Thus compute snd_nxt_new candidate based on 1426 * the dfrag->data_seq that was sent and the data 1427 * that has been handed to the subflow for transmission 1428 * and skip update in case it was old dfrag. 1429 */ 1430 if (likely(after64(snd_nxt_new, msk->snd_nxt))) { 1431 msk->bytes_sent += snd_nxt_new - msk->snd_nxt; 1432 WRITE_ONCE(msk->snd_nxt, snd_nxt_new); 1433 } 1434 } 1435 1436 void mptcp_check_and_set_pending(struct sock *sk) 1437 { 1438 if (mptcp_send_head(sk)) { 1439 mptcp_data_lock(sk); 1440 mptcp_sk(sk)->cb_flags |= BIT(MPTCP_PUSH_PENDING); 1441 mptcp_data_unlock(sk); 1442 } 1443 } 1444 1445 static int __subflow_push_pending(struct sock *sk, struct sock *ssk, 1446 struct mptcp_sendmsg_info *info) 1447 { 1448 struct mptcp_sock *msk = mptcp_sk(sk); 1449 struct mptcp_data_frag *dfrag; 1450 int len, copied = 0, err = 0; 1451 1452 while ((dfrag = mptcp_send_head(sk))) { 1453 info->sent = dfrag->already_sent; 1454 info->limit = dfrag->data_len; 1455 len = dfrag->data_len - dfrag->already_sent; 1456 while (len > 0) { 1457 int ret = 0; 1458 1459 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info); 1460 if (ret <= 0) { 1461 err = copied ? : ret; 1462 goto out; 1463 } 1464 1465 info->sent += ret; 1466 copied += ret; 1467 len -= ret; 1468 1469 mptcp_update_post_push(msk, dfrag, ret); 1470 } 1471 WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1472 1473 if (msk->snd_burst <= 0 || 1474 !sk_stream_memory_free(ssk) || 1475 !mptcp_subflow_active(mptcp_subflow_ctx(ssk))) { 1476 err = copied; 1477 goto out; 1478 } 1479 mptcp_set_timeout(sk); 1480 } 1481 err = copied; 1482 1483 out: 1484 if (err > 0) 1485 msk->last_data_sent = tcp_jiffies32; 1486 return err; 1487 } 1488 1489 void __mptcp_push_pending(struct sock *sk, unsigned int flags) 1490 { 1491 struct sock *prev_ssk = NULL, *ssk = NULL; 1492 struct mptcp_sock *msk = mptcp_sk(sk); 1493 struct mptcp_sendmsg_info info = { 1494 .flags = flags, 1495 }; 1496 bool do_check_data_fin = false; 1497 int push_count = 1; 1498 1499 while (mptcp_send_head(sk) && (push_count > 0)) { 1500 struct mptcp_subflow_context *subflow; 1501 int ret = 0; 1502 1503 if (mptcp_sched_get_send(msk)) 1504 break; 1505 1506 push_count = 0; 1507 1508 mptcp_for_each_subflow(msk, subflow) { 1509 if (READ_ONCE(subflow->scheduled)) { 1510 mptcp_subflow_set_scheduled(subflow, false); 1511 1512 prev_ssk = ssk; 1513 ssk = mptcp_subflow_tcp_sock(subflow); 1514 if (ssk != prev_ssk) { 1515 /* First check. If the ssk has changed since 1516 * the last round, release prev_ssk 1517 */ 1518 if (prev_ssk) 1519 mptcp_push_release(prev_ssk, &info); 1520 1521 /* Need to lock the new subflow only if different 1522 * from the previous one, otherwise we are still 1523 * helding the relevant lock 1524 */ 1525 lock_sock(ssk); 1526 } 1527 1528 push_count++; 1529 1530 ret = __subflow_push_pending(sk, ssk, &info); 1531 if (ret <= 0) { 1532 if (ret != -EAGAIN || 1533 (1 << ssk->sk_state) & 1534 (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSE)) 1535 push_count--; 1536 continue; 1537 } 1538 do_check_data_fin = true; 1539 } 1540 } 1541 } 1542 1543 /* at this point we held the socket lock for the last subflow we used */ 1544 if (ssk) 1545 mptcp_push_release(ssk, &info); 1546 1547 /* ensure the rtx timer is running */ 1548 if (!mptcp_rtx_timer_pending(sk)) 1549 mptcp_reset_rtx_timer(sk); 1550 if (do_check_data_fin) 1551 mptcp_check_send_data_fin(sk); 1552 } 1553 1554 static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first) 1555 { 1556 struct mptcp_sock *msk = mptcp_sk(sk); 1557 struct mptcp_sendmsg_info info = { 1558 .data_lock_held = true, 1559 }; 1560 bool keep_pushing = true; 1561 struct sock *xmit_ssk; 1562 int copied = 0; 1563 1564 info.flags = 0; 1565 while (mptcp_send_head(sk) && keep_pushing) { 1566 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1567 int ret = 0; 1568 1569 /* check for a different subflow usage only after 1570 * spooling the first chunk of data 1571 */ 1572 if (first) { 1573 mptcp_subflow_set_scheduled(subflow, false); 1574 ret = __subflow_push_pending(sk, ssk, &info); 1575 first = false; 1576 if (ret <= 0) 1577 break; 1578 copied += ret; 1579 continue; 1580 } 1581 1582 if (mptcp_sched_get_send(msk)) 1583 goto out; 1584 1585 if (READ_ONCE(subflow->scheduled)) { 1586 mptcp_subflow_set_scheduled(subflow, false); 1587 ret = __subflow_push_pending(sk, ssk, &info); 1588 if (ret <= 0) 1589 keep_pushing = false; 1590 copied += ret; 1591 } 1592 1593 mptcp_for_each_subflow(msk, subflow) { 1594 if (READ_ONCE(subflow->scheduled)) { 1595 xmit_ssk = mptcp_subflow_tcp_sock(subflow); 1596 if (xmit_ssk != ssk) { 1597 mptcp_subflow_delegate(subflow, 1598 MPTCP_DELEGATE_SEND); 1599 keep_pushing = false; 1600 } 1601 } 1602 } 1603 } 1604 1605 out: 1606 /* __mptcp_alloc_tx_skb could have released some wmem and we are 1607 * not going to flush it via release_sock() 1608 */ 1609 if (copied) { 1610 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 1611 info.size_goal); 1612 if (!mptcp_rtx_timer_pending(sk)) 1613 mptcp_reset_rtx_timer(sk); 1614 1615 if (msk->snd_data_fin_enable && 1616 msk->snd_nxt + 1 == msk->write_seq) 1617 mptcp_schedule_work(sk); 1618 } 1619 } 1620 1621 static int mptcp_disconnect(struct sock *sk, int flags); 1622 1623 static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, 1624 size_t len, int *copied_syn) 1625 { 1626 unsigned int saved_flags = msg->msg_flags; 1627 struct mptcp_sock *msk = mptcp_sk(sk); 1628 struct sock *ssk; 1629 int ret; 1630 1631 /* on flags based fastopen the mptcp is supposed to create the 1632 * first subflow right now. Otherwise we are in the defer_connect 1633 * path, and the first subflow must be already present. 1634 * Since the defer_connect flag is cleared after the first succsful 1635 * fastopen attempt, no need to check for additional subflow status. 1636 */ 1637 if (msg->msg_flags & MSG_FASTOPEN) { 1638 ssk = __mptcp_nmpc_sk(msk); 1639 if (IS_ERR(ssk)) 1640 return PTR_ERR(ssk); 1641 } 1642 if (!msk->first) 1643 return -EINVAL; 1644 1645 ssk = msk->first; 1646 1647 lock_sock(ssk); 1648 msg->msg_flags |= MSG_DONTWAIT; 1649 msk->fastopening = 1; 1650 ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL); 1651 msk->fastopening = 0; 1652 msg->msg_flags = saved_flags; 1653 release_sock(ssk); 1654 1655 /* do the blocking bits of inet_stream_connect outside the ssk socket lock */ 1656 if (ret == -EINPROGRESS && !(msg->msg_flags & MSG_DONTWAIT)) { 1657 ret = __inet_stream_connect(sk->sk_socket, msg->msg_name, 1658 msg->msg_namelen, msg->msg_flags, 1); 1659 1660 /* Keep the same behaviour of plain TCP: zero the copied bytes in 1661 * case of any error, except timeout or signal 1662 */ 1663 if (ret && ret != -EINPROGRESS && ret != -ERESTARTSYS && ret != -EINTR) 1664 *copied_syn = 0; 1665 } else if (ret && ret != -EINPROGRESS) { 1666 /* The disconnect() op called by tcp_sendmsg_fastopen()/ 1667 * __inet_stream_connect() can fail, due to looking check, 1668 * see mptcp_disconnect(). 1669 * Attempt it again outside the problematic scope. 1670 */ 1671 if (!mptcp_disconnect(sk, 0)) { 1672 sk->sk_disconnects++; 1673 sk->sk_socket->state = SS_UNCONNECTED; 1674 } 1675 } 1676 inet_clear_bit(DEFER_CONNECT, sk); 1677 1678 return ret; 1679 } 1680 1681 static int do_copy_data_nocache(struct sock *sk, int copy, 1682 struct iov_iter *from, char *to) 1683 { 1684 if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) { 1685 if (!copy_from_iter_full_nocache(to, copy, from)) 1686 return -EFAULT; 1687 } else if (!copy_from_iter_full(to, copy, from)) { 1688 return -EFAULT; 1689 } 1690 return 0; 1691 } 1692 1693 /* open-code sk_stream_memory_free() plus sent limit computation to 1694 * avoid indirect calls in fast-path. 1695 * Called under the msk socket lock, so we can avoid a bunch of ONCE 1696 * annotations. 1697 */ 1698 static u32 mptcp_send_limit(const struct sock *sk) 1699 { 1700 const struct mptcp_sock *msk = mptcp_sk(sk); 1701 u32 limit, not_sent; 1702 1703 if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf)) 1704 return 0; 1705 1706 limit = mptcp_notsent_lowat(sk); 1707 if (limit == UINT_MAX) 1708 return UINT_MAX; 1709 1710 not_sent = msk->write_seq - msk->snd_nxt; 1711 if (not_sent >= limit) 1712 return 0; 1713 1714 return limit - not_sent; 1715 } 1716 1717 static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1718 { 1719 struct mptcp_sock *msk = mptcp_sk(sk); 1720 struct page_frag *pfrag; 1721 size_t copied = 0; 1722 int ret = 0; 1723 long timeo; 1724 1725 /* silently ignore everything else */ 1726 msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_FASTOPEN; 1727 1728 lock_sock(sk); 1729 1730 if (unlikely(inet_test_bit(DEFER_CONNECT, sk) || 1731 msg->msg_flags & MSG_FASTOPEN)) { 1732 int copied_syn = 0; 1733 1734 ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn); 1735 copied += copied_syn; 1736 if (ret == -EINPROGRESS && copied_syn > 0) 1737 goto out; 1738 else if (ret) 1739 goto do_error; 1740 } 1741 1742 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 1743 1744 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 1745 ret = sk_stream_wait_connect(sk, &timeo); 1746 if (ret) 1747 goto do_error; 1748 } 1749 1750 ret = -EPIPE; 1751 if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))) 1752 goto do_error; 1753 1754 pfrag = sk_page_frag(sk); 1755 1756 while (msg_data_left(msg)) { 1757 int total_ts, frag_truesize = 0; 1758 struct mptcp_data_frag *dfrag; 1759 bool dfrag_collapsed; 1760 size_t psize, offset; 1761 u32 copy_limit; 1762 1763 /* ensure fitting the notsent_lowat() constraint */ 1764 copy_limit = mptcp_send_limit(sk); 1765 if (!copy_limit) 1766 goto wait_for_memory; 1767 1768 /* reuse tail pfrag, if possible, or carve a new one from the 1769 * page allocator 1770 */ 1771 dfrag = mptcp_pending_tail(sk); 1772 dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 1773 if (!dfrag_collapsed) { 1774 if (!mptcp_page_frag_refill(sk, pfrag)) 1775 goto wait_for_memory; 1776 1777 dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); 1778 frag_truesize = dfrag->overhead; 1779 } 1780 1781 /* we do not bound vs wspace, to allow a single packet. 1782 * memory accounting will prevent execessive memory usage 1783 * anyway 1784 */ 1785 offset = dfrag->offset + dfrag->data_len; 1786 psize = pfrag->size - offset; 1787 psize = min_t(size_t, psize, msg_data_left(msg)); 1788 psize = min_t(size_t, psize, copy_limit); 1789 total_ts = psize + frag_truesize; 1790 1791 if (!sk_wmem_schedule(sk, total_ts)) 1792 goto wait_for_memory; 1793 1794 ret = do_copy_data_nocache(sk, psize, &msg->msg_iter, 1795 page_address(dfrag->page) + offset); 1796 if (ret) 1797 goto do_error; 1798 1799 /* data successfully copied into the write queue */ 1800 sk_forward_alloc_add(sk, -total_ts); 1801 copied += psize; 1802 dfrag->data_len += psize; 1803 frag_truesize += psize; 1804 pfrag->offset += frag_truesize; 1805 WRITE_ONCE(msk->write_seq, msk->write_seq + psize); 1806 1807 /* charge data on mptcp pending queue to the msk socket 1808 * Note: we charge such data both to sk and ssk 1809 */ 1810 sk_wmem_queued_add(sk, frag_truesize); 1811 if (!dfrag_collapsed) { 1812 get_page(dfrag->page); 1813 list_add_tail(&dfrag->list, &msk->rtx_queue); 1814 if (!msk->first_pending) 1815 WRITE_ONCE(msk->first_pending, dfrag); 1816 } 1817 pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d\n", msk, 1818 dfrag->data_seq, dfrag->data_len, dfrag->already_sent, 1819 !dfrag_collapsed); 1820 1821 continue; 1822 1823 wait_for_memory: 1824 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 1825 __mptcp_push_pending(sk, msg->msg_flags); 1826 ret = sk_stream_wait_memory(sk, &timeo); 1827 if (ret) 1828 goto do_error; 1829 } 1830 1831 if (copied) 1832 __mptcp_push_pending(sk, msg->msg_flags); 1833 1834 out: 1835 release_sock(sk); 1836 return copied; 1837 1838 do_error: 1839 if (copied) 1840 goto out; 1841 1842 copied = sk_stream_error(sk, msg->msg_flags, ret); 1843 goto out; 1844 } 1845 1846 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied); 1847 1848 static int __mptcp_recvmsg_mskq(struct sock *sk, 1849 struct msghdr *msg, 1850 size_t len, int flags, 1851 struct scm_timestamping_internal *tss, 1852 int *cmsg_flags) 1853 { 1854 struct mptcp_sock *msk = mptcp_sk(sk); 1855 struct sk_buff *skb, *tmp; 1856 int copied = 0; 1857 1858 skb_queue_walk_safe(&sk->sk_receive_queue, skb, tmp) { 1859 u32 offset = MPTCP_SKB_CB(skb)->offset; 1860 u32 data_len = skb->len - offset; 1861 u32 count = min_t(size_t, len - copied, data_len); 1862 int err; 1863 1864 if (!(flags & MSG_TRUNC)) { 1865 err = skb_copy_datagram_msg(skb, offset, msg, count); 1866 if (unlikely(err < 0)) { 1867 if (!copied) 1868 return err; 1869 break; 1870 } 1871 } 1872 1873 if (MPTCP_SKB_CB(skb)->has_rxtstamp) { 1874 tcp_update_recv_tstamps(skb, tss); 1875 *cmsg_flags |= MPTCP_CMSG_TS; 1876 } 1877 1878 copied += count; 1879 1880 if (count < data_len) { 1881 if (!(flags & MSG_PEEK)) { 1882 MPTCP_SKB_CB(skb)->offset += count; 1883 MPTCP_SKB_CB(skb)->map_seq += count; 1884 msk->bytes_consumed += count; 1885 } 1886 break; 1887 } 1888 1889 if (!(flags & MSG_PEEK)) { 1890 /* avoid the indirect call, we know the destructor is sock_wfree */ 1891 skb->destructor = NULL; 1892 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 1893 sk_mem_uncharge(sk, skb->truesize); 1894 __skb_unlink(skb, &sk->sk_receive_queue); 1895 __kfree_skb(skb); 1896 msk->bytes_consumed += count; 1897 } 1898 1899 if (copied >= len) 1900 break; 1901 } 1902 1903 mptcp_rcv_space_adjust(msk, copied); 1904 return copied; 1905 } 1906 1907 /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1908 * 1909 * Only difference: Use highest rtt estimate of the subflows in use. 1910 */ 1911 static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1912 { 1913 struct mptcp_subflow_context *subflow; 1914 struct sock *sk = (struct sock *)msk; 1915 u8 scaling_ratio = U8_MAX; 1916 u32 time, advmss = 1; 1917 u64 rtt_us, mstamp; 1918 1919 msk_owned_by_me(msk); 1920 1921 if (copied <= 0) 1922 return; 1923 1924 if (!msk->rcvspace_init) 1925 mptcp_rcv_space_init(msk, msk->first); 1926 1927 msk->rcvq_space.copied += copied; 1928 1929 mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1930 time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1931 1932 rtt_us = msk->rcvq_space.rtt_us; 1933 if (rtt_us && time < (rtt_us >> 3)) 1934 return; 1935 1936 rtt_us = 0; 1937 mptcp_for_each_subflow(msk, subflow) { 1938 const struct tcp_sock *tp; 1939 u64 sf_rtt_us; 1940 u32 sf_advmss; 1941 1942 tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1943 1944 sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1945 sf_advmss = READ_ONCE(tp->advmss); 1946 1947 rtt_us = max(sf_rtt_us, rtt_us); 1948 advmss = max(sf_advmss, advmss); 1949 scaling_ratio = min(tp->scaling_ratio, scaling_ratio); 1950 } 1951 1952 msk->rcvq_space.rtt_us = rtt_us; 1953 msk->scaling_ratio = scaling_ratio; 1954 if (time < (rtt_us >> 3) || rtt_us == 0) 1955 return; 1956 1957 if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1958 goto new_measure; 1959 1960 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) && 1961 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1962 u64 rcvwin, grow; 1963 int rcvbuf; 1964 1965 rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 1966 1967 grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 1968 1969 do_div(grow, msk->rcvq_space.space); 1970 rcvwin += (grow << 1); 1971 1972 rcvbuf = min_t(u64, mptcp_space_from_win(sk, rcvwin), 1973 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); 1974 1975 if (rcvbuf > sk->sk_rcvbuf) { 1976 u32 window_clamp; 1977 1978 window_clamp = mptcp_win_from_space(sk, rcvbuf); 1979 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 1980 1981 /* Make subflows follow along. If we do not do this, we 1982 * get drops at subflow level if skbs can't be moved to 1983 * the mptcp rx queue fast enough (announced rcv_win can 1984 * exceed ssk->sk_rcvbuf). 1985 */ 1986 mptcp_for_each_subflow(msk, subflow) { 1987 struct sock *ssk; 1988 bool slow; 1989 1990 ssk = mptcp_subflow_tcp_sock(subflow); 1991 slow = lock_sock_fast(ssk); 1992 WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 1993 WRITE_ONCE(tcp_sk(ssk)->window_clamp, window_clamp); 1994 if (tcp_can_send_ack(ssk)) 1995 tcp_cleanup_rbuf(ssk, 1); 1996 unlock_sock_fast(ssk, slow); 1997 } 1998 } 1999 } 2000 2001 msk->rcvq_space.space = msk->rcvq_space.copied; 2002 new_measure: 2003 msk->rcvq_space.copied = 0; 2004 msk->rcvq_space.time = mstamp; 2005 } 2006 2007 static struct mptcp_subflow_context * 2008 __mptcp_first_ready_from(struct mptcp_sock *msk, 2009 struct mptcp_subflow_context *subflow) 2010 { 2011 struct mptcp_subflow_context *start_subflow = subflow; 2012 2013 while (!READ_ONCE(subflow->data_avail)) { 2014 subflow = mptcp_next_subflow(msk, subflow); 2015 if (subflow == start_subflow) 2016 return NULL; 2017 } 2018 return subflow; 2019 } 2020 2021 static bool __mptcp_move_skbs(struct sock *sk) 2022 { 2023 struct mptcp_subflow_context *subflow; 2024 struct mptcp_sock *msk = mptcp_sk(sk); 2025 bool ret = false; 2026 2027 if (list_empty(&msk->conn_list)) 2028 return false; 2029 2030 /* verify we can move any data from the subflow, eventually updating */ 2031 if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) 2032 mptcp_for_each_subflow(msk, subflow) 2033 __mptcp_rcvbuf_update(sk, subflow->tcp_sock); 2034 2035 subflow = list_first_entry(&msk->conn_list, 2036 struct mptcp_subflow_context, node); 2037 for (;;) { 2038 struct sock *ssk; 2039 bool slowpath; 2040 2041 /* 2042 * As an optimization avoid traversing the subflows list 2043 * and ev. acquiring the subflow socket lock before baling out 2044 */ 2045 if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf) 2046 break; 2047 2048 subflow = __mptcp_first_ready_from(msk, subflow); 2049 if (!subflow) 2050 break; 2051 2052 ssk = mptcp_subflow_tcp_sock(subflow); 2053 slowpath = lock_sock_fast(ssk); 2054 ret = __mptcp_move_skbs_from_subflow(msk, ssk) || ret; 2055 if (unlikely(ssk->sk_err)) 2056 __mptcp_error_report(sk); 2057 unlock_sock_fast(ssk, slowpath); 2058 2059 subflow = mptcp_next_subflow(msk, subflow); 2060 } 2061 2062 __mptcp_ofo_queue(msk); 2063 if (ret) 2064 mptcp_check_data_fin((struct sock *)msk); 2065 return ret; 2066 } 2067 2068 static unsigned int mptcp_inq_hint(const struct sock *sk) 2069 { 2070 const struct mptcp_sock *msk = mptcp_sk(sk); 2071 const struct sk_buff *skb; 2072 2073 skb = skb_peek(&sk->sk_receive_queue); 2074 if (skb) { 2075 u64 hint_val = READ_ONCE(msk->ack_seq) - MPTCP_SKB_CB(skb)->map_seq; 2076 2077 if (hint_val >= INT_MAX) 2078 return INT_MAX; 2079 2080 return (unsigned int)hint_val; 2081 } 2082 2083 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 2084 return 1; 2085 2086 return 0; 2087 } 2088 2089 static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 2090 int flags, int *addr_len) 2091 { 2092 struct mptcp_sock *msk = mptcp_sk(sk); 2093 struct scm_timestamping_internal tss; 2094 int copied = 0, cmsg_flags = 0; 2095 int target; 2096 long timeo; 2097 2098 /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */ 2099 if (unlikely(flags & MSG_ERRQUEUE)) 2100 return inet_recv_error(sk, msg, len, addr_len); 2101 2102 lock_sock(sk); 2103 if (unlikely(sk->sk_state == TCP_LISTEN)) { 2104 copied = -ENOTCONN; 2105 goto out_err; 2106 } 2107 2108 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 2109 2110 len = min_t(size_t, len, INT_MAX); 2111 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 2112 2113 if (unlikely(msk->recvmsg_inq)) 2114 cmsg_flags = MPTCP_CMSG_INQ; 2115 2116 while (copied < len) { 2117 int err, bytes_read; 2118 2119 bytes_read = __mptcp_recvmsg_mskq(sk, msg, len - copied, flags, &tss, &cmsg_flags); 2120 if (unlikely(bytes_read < 0)) { 2121 if (!copied) 2122 copied = bytes_read; 2123 goto out_err; 2124 } 2125 2126 copied += bytes_read; 2127 2128 if (skb_queue_empty(&sk->sk_receive_queue) && __mptcp_move_skbs(sk)) 2129 continue; 2130 2131 /* only the MPTCP socket status is relevant here. The exit 2132 * conditions mirror closely tcp_recvmsg() 2133 */ 2134 if (copied >= target) 2135 break; 2136 2137 if (copied) { 2138 if (sk->sk_err || 2139 sk->sk_state == TCP_CLOSE || 2140 (sk->sk_shutdown & RCV_SHUTDOWN) || 2141 !timeo || 2142 signal_pending(current)) 2143 break; 2144 } else { 2145 if (sk->sk_err) { 2146 copied = sock_error(sk); 2147 break; 2148 } 2149 2150 if (sk->sk_shutdown & RCV_SHUTDOWN) { 2151 /* race breaker: the shutdown could be after the 2152 * previous receive queue check 2153 */ 2154 if (__mptcp_move_skbs(sk)) 2155 continue; 2156 break; 2157 } 2158 2159 if (sk->sk_state == TCP_CLOSE) { 2160 copied = -ENOTCONN; 2161 break; 2162 } 2163 2164 if (!timeo) { 2165 copied = -EAGAIN; 2166 break; 2167 } 2168 2169 if (signal_pending(current)) { 2170 copied = sock_intr_errno(timeo); 2171 break; 2172 } 2173 } 2174 2175 pr_debug("block timeout %ld\n", timeo); 2176 mptcp_cleanup_rbuf(msk, copied); 2177 err = sk_wait_data(sk, &timeo, NULL); 2178 if (err < 0) { 2179 err = copied ? : err; 2180 goto out_err; 2181 } 2182 } 2183 2184 mptcp_cleanup_rbuf(msk, copied); 2185 2186 out_err: 2187 if (cmsg_flags && copied >= 0) { 2188 if (cmsg_flags & MPTCP_CMSG_TS) 2189 tcp_recv_timestamp(msg, sk, &tss); 2190 2191 if (cmsg_flags & MPTCP_CMSG_INQ) { 2192 unsigned int inq = mptcp_inq_hint(sk); 2193 2194 put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 2195 } 2196 } 2197 2198 pr_debug("msk=%p rx queue empty=%d copied=%d\n", 2199 msk, skb_queue_empty(&sk->sk_receive_queue), copied); 2200 2201 release_sock(sk); 2202 return copied; 2203 } 2204 2205 static void mptcp_retransmit_timer(struct timer_list *t) 2206 { 2207 struct inet_connection_sock *icsk = timer_container_of(icsk, t, 2208 icsk_retransmit_timer); 2209 struct sock *sk = &icsk->icsk_inet.sk; 2210 struct mptcp_sock *msk = mptcp_sk(sk); 2211 2212 bh_lock_sock(sk); 2213 if (!sock_owned_by_user(sk)) { 2214 /* we need a process context to retransmit */ 2215 if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) 2216 mptcp_schedule_work(sk); 2217 } else { 2218 /* delegate our work to tcp_release_cb() */ 2219 __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); 2220 } 2221 bh_unlock_sock(sk); 2222 sock_put(sk); 2223 } 2224 2225 static void mptcp_tout_timer(struct timer_list *t) 2226 { 2227 struct sock *sk = timer_container_of(sk, t, sk_timer); 2228 2229 mptcp_schedule_work(sk); 2230 sock_put(sk); 2231 } 2232 2233 /* Find an idle subflow. Return NULL if there is unacked data at tcp 2234 * level. 2235 * 2236 * A backup subflow is returned only if that is the only kind available. 2237 */ 2238 struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) 2239 { 2240 struct sock *backup = NULL, *pick = NULL; 2241 struct mptcp_subflow_context *subflow; 2242 int min_stale_count = INT_MAX; 2243 2244 mptcp_for_each_subflow(msk, subflow) { 2245 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2246 2247 if (!__mptcp_subflow_active(subflow)) 2248 continue; 2249 2250 /* still data outstanding at TCP level? skip this */ 2251 if (!tcp_rtx_and_write_queues_empty(ssk)) { 2252 mptcp_pm_subflow_chk_stale(msk, ssk); 2253 min_stale_count = min_t(int, min_stale_count, subflow->stale_count); 2254 continue; 2255 } 2256 2257 if (subflow->backup || subflow->request_bkup) { 2258 if (!backup) 2259 backup = ssk; 2260 continue; 2261 } 2262 2263 if (!pick) 2264 pick = ssk; 2265 } 2266 2267 if (pick) 2268 return pick; 2269 2270 /* use backup only if there are no progresses anywhere */ 2271 return min_stale_count > 1 ? backup : NULL; 2272 } 2273 2274 bool __mptcp_retransmit_pending_data(struct sock *sk) 2275 { 2276 struct mptcp_data_frag *cur, *rtx_head; 2277 struct mptcp_sock *msk = mptcp_sk(sk); 2278 2279 if (__mptcp_check_fallback(msk)) 2280 return false; 2281 2282 /* the closing socket has some data untransmitted and/or unacked: 2283 * some data in the mptcp rtx queue has not really xmitted yet. 2284 * keep it simple and re-inject the whole mptcp level rtx queue 2285 */ 2286 mptcp_data_lock(sk); 2287 __mptcp_clean_una_wakeup(sk); 2288 rtx_head = mptcp_rtx_head(sk); 2289 if (!rtx_head) { 2290 mptcp_data_unlock(sk); 2291 return false; 2292 } 2293 2294 msk->recovery_snd_nxt = msk->snd_nxt; 2295 msk->recovery = true; 2296 mptcp_data_unlock(sk); 2297 2298 msk->first_pending = rtx_head; 2299 msk->snd_burst = 0; 2300 2301 /* be sure to clear the "sent status" on all re-injected fragments */ 2302 list_for_each_entry(cur, &msk->rtx_queue, list) { 2303 if (!cur->already_sent) 2304 break; 2305 cur->already_sent = 0; 2306 } 2307 2308 return true; 2309 } 2310 2311 /* flags for __mptcp_close_ssk() */ 2312 #define MPTCP_CF_PUSH BIT(1) 2313 #define MPTCP_CF_FASTCLOSE BIT(2) 2314 2315 /* be sure to send a reset only if the caller asked for it, also 2316 * clean completely the subflow status when the subflow reaches 2317 * TCP_CLOSE state 2318 */ 2319 static void __mptcp_subflow_disconnect(struct sock *ssk, 2320 struct mptcp_subflow_context *subflow, 2321 unsigned int flags) 2322 { 2323 if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) || 2324 (flags & MPTCP_CF_FASTCLOSE)) { 2325 /* The MPTCP code never wait on the subflow sockets, TCP-level 2326 * disconnect should never fail 2327 */ 2328 WARN_ON_ONCE(tcp_disconnect(ssk, 0)); 2329 mptcp_subflow_ctx_reset(subflow); 2330 } else { 2331 tcp_shutdown(ssk, SEND_SHUTDOWN); 2332 } 2333 } 2334 2335 /* subflow sockets can be either outgoing (connect) or incoming 2336 * (accept). 2337 * 2338 * Outgoing subflows use in-kernel sockets. 2339 * Incoming subflows do not have their own 'struct socket' allocated, 2340 * so we need to use tcp_close() after detaching them from the mptcp 2341 * parent socket. 2342 */ 2343 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2344 struct mptcp_subflow_context *subflow, 2345 unsigned int flags) 2346 { 2347 struct mptcp_sock *msk = mptcp_sk(sk); 2348 bool dispose_it, need_push = false; 2349 2350 /* If the first subflow moved to a close state before accept, e.g. due 2351 * to an incoming reset or listener shutdown, the subflow socket is 2352 * already deleted by inet_child_forget() and the mptcp socket can't 2353 * survive too. 2354 */ 2355 if (msk->in_accept_queue && msk->first == ssk && 2356 (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) { 2357 /* ensure later check in mptcp_worker() will dispose the msk */ 2358 sock_set_flag(sk, SOCK_DEAD); 2359 mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1)); 2360 lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2361 mptcp_subflow_drop_ctx(ssk); 2362 goto out_release; 2363 } 2364 2365 dispose_it = msk->free_first || ssk != msk->first; 2366 if (dispose_it) 2367 list_del(&subflow->node); 2368 2369 lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2370 2371 if ((flags & MPTCP_CF_FASTCLOSE) && !__mptcp_check_fallback(msk)) { 2372 /* be sure to force the tcp_close path 2373 * to generate the egress reset 2374 */ 2375 ssk->sk_lingertime = 0; 2376 sock_set_flag(ssk, SOCK_LINGER); 2377 subflow->send_fastclose = 1; 2378 } 2379 2380 need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); 2381 if (!dispose_it) { 2382 __mptcp_subflow_disconnect(ssk, subflow, flags); 2383 release_sock(ssk); 2384 2385 goto out; 2386 } 2387 2388 subflow->disposable = 1; 2389 2390 /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops 2391 * the ssk has been already destroyed, we just need to release the 2392 * reference owned by msk; 2393 */ 2394 if (!inet_csk(ssk)->icsk_ulp_ops) { 2395 WARN_ON_ONCE(!sock_flag(ssk, SOCK_DEAD)); 2396 kfree_rcu(subflow, rcu); 2397 } else { 2398 /* otherwise tcp will dispose of the ssk and subflow ctx */ 2399 __tcp_close(ssk, 0); 2400 2401 /* close acquired an extra ref */ 2402 __sock_put(ssk); 2403 } 2404 2405 out_release: 2406 __mptcp_subflow_error_report(sk, ssk); 2407 release_sock(ssk); 2408 2409 sock_put(ssk); 2410 2411 if (ssk == msk->first) 2412 WRITE_ONCE(msk->first, NULL); 2413 2414 out: 2415 __mptcp_sync_sndbuf(sk); 2416 if (need_push) 2417 __mptcp_push_pending(sk, 0); 2418 2419 /* Catch every 'all subflows closed' scenario, including peers silently 2420 * closing them, e.g. due to timeout. 2421 * For established sockets, allow an additional timeout before closing, 2422 * as the protocol can still create more subflows. 2423 */ 2424 if (list_is_singular(&msk->conn_list) && msk->first && 2425 inet_sk_state_load(msk->first) == TCP_CLOSE) { 2426 if (sk->sk_state != TCP_ESTABLISHED || 2427 msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) { 2428 mptcp_set_state(sk, TCP_CLOSE); 2429 mptcp_close_wake_up(sk); 2430 } else { 2431 mptcp_start_tout_timer(sk); 2432 } 2433 } 2434 } 2435 2436 void mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2437 struct mptcp_subflow_context *subflow) 2438 { 2439 /* The first subflow can already be closed and still in the list */ 2440 if (subflow->close_event_done) 2441 return; 2442 2443 subflow->close_event_done = true; 2444 2445 if (sk->sk_state == TCP_ESTABLISHED) 2446 mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); 2447 2448 /* subflow aborted before reaching the fully_established status 2449 * attempt the creation of the next subflow 2450 */ 2451 mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); 2452 2453 __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); 2454 } 2455 2456 static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 2457 { 2458 return 0; 2459 } 2460 2461 static void __mptcp_close_subflow(struct sock *sk) 2462 { 2463 struct mptcp_subflow_context *subflow, *tmp; 2464 struct mptcp_sock *msk = mptcp_sk(sk); 2465 2466 might_sleep(); 2467 2468 mptcp_for_each_subflow_safe(msk, subflow, tmp) { 2469 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2470 int ssk_state = inet_sk_state_load(ssk); 2471 2472 if (ssk_state != TCP_CLOSE && 2473 (ssk_state != TCP_CLOSE_WAIT || 2474 inet_sk_state_load(sk) != TCP_ESTABLISHED)) 2475 continue; 2476 2477 /* 'subflow_data_ready' will re-sched once rx queue is empty */ 2478 if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) 2479 continue; 2480 2481 mptcp_close_ssk(sk, ssk, subflow); 2482 } 2483 2484 } 2485 2486 static bool mptcp_close_tout_expired(const struct sock *sk) 2487 { 2488 if (!inet_csk(sk)->icsk_mtup.probe_timestamp || 2489 sk->sk_state == TCP_CLOSE) 2490 return false; 2491 2492 return time_after32(tcp_jiffies32, 2493 inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk)); 2494 } 2495 2496 static void mptcp_check_fastclose(struct mptcp_sock *msk) 2497 { 2498 struct mptcp_subflow_context *subflow, *tmp; 2499 struct sock *sk = (struct sock *)msk; 2500 2501 if (likely(!READ_ONCE(msk->rcv_fastclose))) 2502 return; 2503 2504 mptcp_token_destroy(msk); 2505 2506 mptcp_for_each_subflow_safe(msk, subflow, tmp) { 2507 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2508 bool slow; 2509 2510 slow = lock_sock_fast(tcp_sk); 2511 if (tcp_sk->sk_state != TCP_CLOSE) { 2512 mptcp_send_active_reset_reason(tcp_sk); 2513 tcp_set_state(tcp_sk, TCP_CLOSE); 2514 } 2515 unlock_sock_fast(tcp_sk, slow); 2516 } 2517 2518 /* Mirror the tcp_reset() error propagation */ 2519 switch (sk->sk_state) { 2520 case TCP_SYN_SENT: 2521 WRITE_ONCE(sk->sk_err, ECONNREFUSED); 2522 break; 2523 case TCP_CLOSE_WAIT: 2524 WRITE_ONCE(sk->sk_err, EPIPE); 2525 break; 2526 case TCP_CLOSE: 2527 return; 2528 default: 2529 WRITE_ONCE(sk->sk_err, ECONNRESET); 2530 } 2531 2532 mptcp_set_state(sk, TCP_CLOSE); 2533 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 2534 smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 2535 set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); 2536 2537 /* the calling mptcp_worker will properly destroy the socket */ 2538 if (sock_flag(sk, SOCK_DEAD)) 2539 return; 2540 2541 sk->sk_state_change(sk); 2542 sk_error_report(sk); 2543 } 2544 2545 static void __mptcp_retrans(struct sock *sk) 2546 { 2547 struct mptcp_sock *msk = mptcp_sk(sk); 2548 struct mptcp_subflow_context *subflow; 2549 struct mptcp_sendmsg_info info = {}; 2550 struct mptcp_data_frag *dfrag; 2551 struct sock *ssk; 2552 int ret, err; 2553 u16 len = 0; 2554 2555 mptcp_clean_una_wakeup(sk); 2556 2557 /* first check ssk: need to kick "stale" logic */ 2558 err = mptcp_sched_get_retrans(msk); 2559 dfrag = mptcp_rtx_head(sk); 2560 if (!dfrag) { 2561 if (mptcp_data_fin_enabled(msk)) { 2562 struct inet_connection_sock *icsk = inet_csk(sk); 2563 2564 icsk->icsk_retransmits++; 2565 mptcp_set_datafin_timeout(sk); 2566 mptcp_send_ack(msk); 2567 2568 goto reset_timer; 2569 } 2570 2571 if (!mptcp_send_head(sk)) 2572 return; 2573 2574 goto reset_timer; 2575 } 2576 2577 if (err) 2578 goto reset_timer; 2579 2580 mptcp_for_each_subflow(msk, subflow) { 2581 if (READ_ONCE(subflow->scheduled)) { 2582 u16 copied = 0; 2583 2584 mptcp_subflow_set_scheduled(subflow, false); 2585 2586 ssk = mptcp_subflow_tcp_sock(subflow); 2587 2588 lock_sock(ssk); 2589 2590 /* limit retransmission to the bytes already sent on some subflows */ 2591 info.sent = 0; 2592 info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : 2593 dfrag->already_sent; 2594 while (info.sent < info.limit) { 2595 ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 2596 if (ret <= 0) 2597 break; 2598 2599 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 2600 copied += ret; 2601 info.sent += ret; 2602 } 2603 if (copied) { 2604 len = max(copied, len); 2605 tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 2606 info.size_goal); 2607 WRITE_ONCE(msk->allow_infinite_fallback, false); 2608 } 2609 2610 release_sock(ssk); 2611 } 2612 } 2613 2614 msk->bytes_retrans += len; 2615 dfrag->already_sent = max(dfrag->already_sent, len); 2616 2617 reset_timer: 2618 mptcp_check_and_set_pending(sk); 2619 2620 if (!mptcp_rtx_timer_pending(sk)) 2621 mptcp_reset_rtx_timer(sk); 2622 } 2623 2624 /* schedule the timeout timer for the relevant event: either close timeout 2625 * or mp_fail timeout. The close timeout takes precedence on the mp_fail one 2626 */ 2627 void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) 2628 { 2629 struct sock *sk = (struct sock *)msk; 2630 unsigned long timeout, close_timeout; 2631 2632 if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp) 2633 return; 2634 2635 close_timeout = (unsigned long)inet_csk(sk)->icsk_mtup.probe_timestamp - 2636 tcp_jiffies32 + jiffies + mptcp_close_timeout(sk); 2637 2638 /* the close timeout takes precedence on the fail one, and here at least one of 2639 * them is active 2640 */ 2641 timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout; 2642 2643 sk_reset_timer(sk, &sk->sk_timer, timeout); 2644 } 2645 2646 static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) 2647 { 2648 struct sock *ssk = msk->first; 2649 bool slow; 2650 2651 if (!ssk) 2652 return; 2653 2654 pr_debug("MP_FAIL doesn't respond, reset the subflow\n"); 2655 2656 slow = lock_sock_fast(ssk); 2657 mptcp_subflow_reset(ssk); 2658 WRITE_ONCE(mptcp_subflow_ctx(ssk)->fail_tout, 0); 2659 unlock_sock_fast(ssk, slow); 2660 } 2661 2662 static void mptcp_do_fastclose(struct sock *sk) 2663 { 2664 struct mptcp_subflow_context *subflow, *tmp; 2665 struct mptcp_sock *msk = mptcp_sk(sk); 2666 2667 mptcp_set_state(sk, TCP_CLOSE); 2668 mptcp_for_each_subflow_safe(msk, subflow, tmp) 2669 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), 2670 subflow, MPTCP_CF_FASTCLOSE); 2671 } 2672 2673 static void mptcp_worker(struct work_struct *work) 2674 { 2675 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 2676 struct sock *sk = (struct sock *)msk; 2677 unsigned long fail_tout; 2678 int state; 2679 2680 lock_sock(sk); 2681 state = sk->sk_state; 2682 if (unlikely((1 << state) & (TCPF_CLOSE | TCPF_LISTEN))) 2683 goto unlock; 2684 2685 mptcp_check_fastclose(msk); 2686 2687 mptcp_pm_worker(msk); 2688 2689 mptcp_check_send_data_fin(sk); 2690 mptcp_check_data_fin_ack(sk); 2691 mptcp_check_data_fin(sk); 2692 2693 if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 2694 __mptcp_close_subflow(sk); 2695 2696 if (mptcp_close_tout_expired(sk)) { 2697 mptcp_do_fastclose(sk); 2698 mptcp_close_wake_up(sk); 2699 } 2700 2701 if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) { 2702 __mptcp_destroy_sock(sk); 2703 goto unlock; 2704 } 2705 2706 if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 2707 __mptcp_retrans(sk); 2708 2709 fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0; 2710 if (fail_tout && time_after(jiffies, fail_tout)) 2711 mptcp_mp_fail_no_response(msk); 2712 2713 unlock: 2714 release_sock(sk); 2715 sock_put(sk); 2716 } 2717 2718 static void __mptcp_init_sock(struct sock *sk) 2719 { 2720 struct mptcp_sock *msk = mptcp_sk(sk); 2721 2722 INIT_LIST_HEAD(&msk->conn_list); 2723 INIT_LIST_HEAD(&msk->join_list); 2724 INIT_LIST_HEAD(&msk->rtx_queue); 2725 INIT_WORK(&msk->work, mptcp_worker); 2726 msk->out_of_order_queue = RB_ROOT; 2727 msk->first_pending = NULL; 2728 msk->timer_ival = TCP_RTO_MIN; 2729 msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; 2730 2731 WRITE_ONCE(msk->first, NULL); 2732 inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 2733 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 2734 WRITE_ONCE(msk->allow_infinite_fallback, true); 2735 msk->recovery = false; 2736 msk->subflow_id = 1; 2737 msk->last_data_sent = tcp_jiffies32; 2738 msk->last_data_recv = tcp_jiffies32; 2739 msk->last_ack_recv = tcp_jiffies32; 2740 2741 mptcp_pm_data_init(msk); 2742 2743 /* re-use the csk retrans timer for MPTCP-level retrans */ 2744 timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 2745 timer_setup(&sk->sk_timer, mptcp_tout_timer, 0); 2746 } 2747 2748 static void mptcp_ca_reset(struct sock *sk) 2749 { 2750 struct inet_connection_sock *icsk = inet_csk(sk); 2751 2752 tcp_assign_congestion_control(sk); 2753 strscpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name, 2754 sizeof(mptcp_sk(sk)->ca_name)); 2755 2756 /* no need to keep a reference to the ops, the name will suffice */ 2757 tcp_cleanup_congestion_control(sk); 2758 icsk->icsk_ca_ops = NULL; 2759 } 2760 2761 static int mptcp_init_sock(struct sock *sk) 2762 { 2763 struct net *net = sock_net(sk); 2764 int ret; 2765 2766 __mptcp_init_sock(sk); 2767 2768 if (!mptcp_is_enabled(net)) 2769 return -ENOPROTOOPT; 2770 2771 if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 2772 return -ENOMEM; 2773 2774 rcu_read_lock(); 2775 ret = mptcp_init_sched(mptcp_sk(sk), 2776 mptcp_sched_find(mptcp_get_scheduler(net))); 2777 rcu_read_unlock(); 2778 if (ret) 2779 return ret; 2780 2781 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags); 2782 2783 /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will 2784 * propagate the correct value 2785 */ 2786 mptcp_ca_reset(sk); 2787 2788 sk_sockets_allocated_inc(sk); 2789 sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]); 2790 sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]); 2791 2792 return 0; 2793 } 2794 2795 static void __mptcp_clear_xmit(struct sock *sk) 2796 { 2797 struct mptcp_sock *msk = mptcp_sk(sk); 2798 struct mptcp_data_frag *dtmp, *dfrag; 2799 2800 WRITE_ONCE(msk->first_pending, NULL); 2801 list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 2802 dfrag_clear(sk, dfrag); 2803 } 2804 2805 void mptcp_cancel_work(struct sock *sk) 2806 { 2807 struct mptcp_sock *msk = mptcp_sk(sk); 2808 2809 if (cancel_work_sync(&msk->work)) 2810 __sock_put(sk); 2811 } 2812 2813 void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 2814 { 2815 lock_sock(ssk); 2816 2817 switch (ssk->sk_state) { 2818 case TCP_LISTEN: 2819 if (!(how & RCV_SHUTDOWN)) 2820 break; 2821 fallthrough; 2822 case TCP_SYN_SENT: 2823 WARN_ON_ONCE(tcp_disconnect(ssk, O_NONBLOCK)); 2824 break; 2825 default: 2826 if (__mptcp_check_fallback(mptcp_sk(sk))) { 2827 pr_debug("Fallback\n"); 2828 ssk->sk_shutdown |= how; 2829 tcp_shutdown(ssk, how); 2830 2831 /* simulate the data_fin ack reception to let the state 2832 * machine move forward 2833 */ 2834 WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt); 2835 mptcp_schedule_work(sk); 2836 } else { 2837 pr_debug("Sending DATA_FIN on subflow %p\n", ssk); 2838 tcp_send_ack(ssk); 2839 if (!mptcp_rtx_timer_pending(sk)) 2840 mptcp_reset_rtx_timer(sk); 2841 } 2842 break; 2843 } 2844 2845 release_sock(ssk); 2846 } 2847 2848 void mptcp_set_state(struct sock *sk, int state) 2849 { 2850 int oldstate = sk->sk_state; 2851 2852 switch (state) { 2853 case TCP_ESTABLISHED: 2854 if (oldstate != TCP_ESTABLISHED) 2855 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); 2856 break; 2857 case TCP_CLOSE_WAIT: 2858 /* Unlike TCP, MPTCP sk would not have the TCP_SYN_RECV state: 2859 * MPTCP "accepted" sockets will be created later on. So no 2860 * transition from TCP_SYN_RECV to TCP_CLOSE_WAIT. 2861 */ 2862 break; 2863 default: 2864 if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT) 2865 MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); 2866 } 2867 2868 inet_sk_state_store(sk, state); 2869 } 2870 2871 static const unsigned char new_state[16] = { 2872 /* current state: new state: action: */ 2873 [0 /* (Invalid) */] = TCP_CLOSE, 2874 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2875 [TCP_SYN_SENT] = TCP_CLOSE, 2876 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 2877 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 2878 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 2879 [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 2880 [TCP_CLOSE] = TCP_CLOSE, 2881 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 2882 [TCP_LAST_ACK] = TCP_LAST_ACK, 2883 [TCP_LISTEN] = TCP_CLOSE, 2884 [TCP_CLOSING] = TCP_CLOSING, 2885 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 2886 }; 2887 2888 static int mptcp_close_state(struct sock *sk) 2889 { 2890 int next = (int)new_state[sk->sk_state]; 2891 int ns = next & TCP_STATE_MASK; 2892 2893 mptcp_set_state(sk, ns); 2894 2895 return next & TCP_ACTION_FIN; 2896 } 2897 2898 static void mptcp_check_send_data_fin(struct sock *sk) 2899 { 2900 struct mptcp_subflow_context *subflow; 2901 struct mptcp_sock *msk = mptcp_sk(sk); 2902 2903 pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu\n", 2904 msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), 2905 msk->snd_nxt, msk->write_seq); 2906 2907 /* we still need to enqueue subflows or not really shutting down, 2908 * skip this 2909 */ 2910 if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || 2911 mptcp_send_head(sk)) 2912 return; 2913 2914 WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2915 2916 mptcp_for_each_subflow(msk, subflow) { 2917 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2918 2919 mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); 2920 } 2921 } 2922 2923 static void __mptcp_wr_shutdown(struct sock *sk) 2924 { 2925 struct mptcp_sock *msk = mptcp_sk(sk); 2926 2927 pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d\n", 2928 msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, 2929 !!mptcp_send_head(sk)); 2930 2931 /* will be ignored by fallback sockets */ 2932 WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 2933 WRITE_ONCE(msk->snd_data_fin_enable, 1); 2934 2935 mptcp_check_send_data_fin(sk); 2936 } 2937 2938 static void __mptcp_destroy_sock(struct sock *sk) 2939 { 2940 struct mptcp_sock *msk = mptcp_sk(sk); 2941 2942 pr_debug("msk=%p\n", msk); 2943 2944 might_sleep(); 2945 2946 mptcp_stop_rtx_timer(sk); 2947 sk_stop_timer(sk, &sk->sk_timer); 2948 msk->pm.status = 0; 2949 mptcp_release_sched(msk); 2950 2951 sk->sk_prot->destroy(sk); 2952 2953 sk_stream_kill_queues(sk); 2954 xfrm_sk_free_policy(sk); 2955 2956 sock_put(sk); 2957 } 2958 2959 void __mptcp_unaccepted_force_close(struct sock *sk) 2960 { 2961 sock_set_flag(sk, SOCK_DEAD); 2962 mptcp_do_fastclose(sk); 2963 __mptcp_destroy_sock(sk); 2964 } 2965 2966 static __poll_t mptcp_check_readable(struct sock *sk) 2967 { 2968 return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0; 2969 } 2970 2971 static void mptcp_check_listen_stop(struct sock *sk) 2972 { 2973 struct sock *ssk; 2974 2975 if (inet_sk_state_load(sk) != TCP_LISTEN) 2976 return; 2977 2978 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 2979 ssk = mptcp_sk(sk)->first; 2980 if (WARN_ON_ONCE(!ssk || inet_sk_state_load(ssk) != TCP_LISTEN)) 2981 return; 2982 2983 lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2984 tcp_set_state(ssk, TCP_CLOSE); 2985 mptcp_subflow_queue_clean(sk, ssk); 2986 inet_csk_listen_stop(ssk); 2987 mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED); 2988 release_sock(ssk); 2989 } 2990 2991 bool __mptcp_close(struct sock *sk, long timeout) 2992 { 2993 struct mptcp_subflow_context *subflow; 2994 struct mptcp_sock *msk = mptcp_sk(sk); 2995 bool do_cancel_work = false; 2996 int subflows_alive = 0; 2997 2998 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 2999 3000 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { 3001 mptcp_check_listen_stop(sk); 3002 mptcp_set_state(sk, TCP_CLOSE); 3003 goto cleanup; 3004 } 3005 3006 if (mptcp_data_avail(msk) || timeout < 0) { 3007 /* If the msk has read data, or the caller explicitly ask it, 3008 * do the MPTCP equivalent of TCP reset, aka MPTCP fastclose 3009 */ 3010 mptcp_do_fastclose(sk); 3011 timeout = 0; 3012 } else if (mptcp_close_state(sk)) { 3013 __mptcp_wr_shutdown(sk); 3014 } 3015 3016 sk_stream_wait_close(sk, timeout); 3017 3018 cleanup: 3019 /* orphan all the subflows */ 3020 mptcp_for_each_subflow(msk, subflow) { 3021 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3022 bool slow = lock_sock_fast_nested(ssk); 3023 3024 subflows_alive += ssk->sk_state != TCP_CLOSE; 3025 3026 /* since the close timeout takes precedence on the fail one, 3027 * cancel the latter 3028 */ 3029 if (ssk == msk->first) 3030 subflow->fail_tout = 0; 3031 3032 /* detach from the parent socket, but allow data_ready to 3033 * push incoming data into the mptcp stack, to properly ack it 3034 */ 3035 ssk->sk_socket = NULL; 3036 ssk->sk_wq = NULL; 3037 unlock_sock_fast(ssk, slow); 3038 } 3039 sock_orphan(sk); 3040 3041 /* all the subflows are closed, only timeout can change the msk 3042 * state, let's not keep resources busy for no reasons 3043 */ 3044 if (subflows_alive == 0) 3045 mptcp_set_state(sk, TCP_CLOSE); 3046 3047 sock_hold(sk); 3048 pr_debug("msk=%p state=%d\n", sk, sk->sk_state); 3049 mptcp_pm_connection_closed(msk); 3050 3051 if (sk->sk_state == TCP_CLOSE) { 3052 __mptcp_destroy_sock(sk); 3053 do_cancel_work = true; 3054 } else { 3055 mptcp_start_tout_timer(sk); 3056 } 3057 3058 return do_cancel_work; 3059 } 3060 3061 static void mptcp_close(struct sock *sk, long timeout) 3062 { 3063 bool do_cancel_work; 3064 3065 lock_sock(sk); 3066 3067 do_cancel_work = __mptcp_close(sk, timeout); 3068 release_sock(sk); 3069 if (do_cancel_work) 3070 mptcp_cancel_work(sk); 3071 3072 sock_put(sk); 3073 } 3074 3075 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 3076 { 3077 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3078 const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 3079 struct ipv6_pinfo *msk6 = inet6_sk(msk); 3080 3081 msk->sk_v6_daddr = ssk->sk_v6_daddr; 3082 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 3083 3084 if (msk6 && ssk6) { 3085 msk6->saddr = ssk6->saddr; 3086 msk6->flow_label = ssk6->flow_label; 3087 } 3088 #endif 3089 3090 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 3091 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 3092 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 3093 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 3094 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 3095 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 3096 } 3097 3098 static int mptcp_disconnect(struct sock *sk, int flags) 3099 { 3100 struct mptcp_sock *msk = mptcp_sk(sk); 3101 3102 /* We are on the fastopen error path. We can't call straight into the 3103 * subflows cleanup code due to lock nesting (we are already under 3104 * msk->firstsocket lock). 3105 */ 3106 if (msk->fastopening) 3107 return -EBUSY; 3108 3109 mptcp_check_listen_stop(sk); 3110 mptcp_set_state(sk, TCP_CLOSE); 3111 3112 mptcp_stop_rtx_timer(sk); 3113 mptcp_stop_tout_timer(sk); 3114 3115 mptcp_pm_connection_closed(msk); 3116 3117 /* msk->subflow is still intact, the following will not free the first 3118 * subflow 3119 */ 3120 mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE); 3121 WRITE_ONCE(msk->flags, 0); 3122 msk->cb_flags = 0; 3123 msk->recovery = false; 3124 WRITE_ONCE(msk->can_ack, false); 3125 WRITE_ONCE(msk->fully_established, false); 3126 WRITE_ONCE(msk->rcv_data_fin, false); 3127 WRITE_ONCE(msk->snd_data_fin_enable, false); 3128 WRITE_ONCE(msk->rcv_fastclose, false); 3129 WRITE_ONCE(msk->use_64bit_ack, false); 3130 WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 3131 mptcp_pm_data_reset(msk); 3132 mptcp_ca_reset(sk); 3133 msk->bytes_consumed = 0; 3134 msk->bytes_acked = 0; 3135 msk->bytes_received = 0; 3136 msk->bytes_sent = 0; 3137 msk->bytes_retrans = 0; 3138 msk->rcvspace_init = 0; 3139 3140 WRITE_ONCE(sk->sk_shutdown, 0); 3141 sk_error_report(sk); 3142 return 0; 3143 } 3144 3145 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3146 static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 3147 { 3148 struct mptcp6_sock *msk6 = container_of(mptcp_sk(sk), struct mptcp6_sock, msk); 3149 3150 return &msk6->np; 3151 } 3152 3153 static void mptcp_copy_ip6_options(struct sock *newsk, const struct sock *sk) 3154 { 3155 const struct ipv6_pinfo *np = inet6_sk(sk); 3156 struct ipv6_txoptions *opt; 3157 struct ipv6_pinfo *newnp; 3158 3159 newnp = inet6_sk(newsk); 3160 3161 rcu_read_lock(); 3162 opt = rcu_dereference(np->opt); 3163 if (opt) { 3164 opt = ipv6_dup_options(newsk, opt); 3165 if (!opt) 3166 net_warn_ratelimited("%s: Failed to copy ip6 options\n", __func__); 3167 } 3168 RCU_INIT_POINTER(newnp->opt, opt); 3169 rcu_read_unlock(); 3170 } 3171 #endif 3172 3173 static void mptcp_copy_ip_options(struct sock *newsk, const struct sock *sk) 3174 { 3175 struct ip_options_rcu *inet_opt, *newopt = NULL; 3176 const struct inet_sock *inet = inet_sk(sk); 3177 struct inet_sock *newinet; 3178 3179 newinet = inet_sk(newsk); 3180 3181 rcu_read_lock(); 3182 inet_opt = rcu_dereference(inet->inet_opt); 3183 if (inet_opt) { 3184 newopt = sock_kmemdup(newsk, inet_opt, sizeof(*inet_opt) + 3185 inet_opt->opt.optlen, GFP_ATOMIC); 3186 if (!newopt) 3187 net_warn_ratelimited("%s: Failed to copy ip options\n", __func__); 3188 } 3189 RCU_INIT_POINTER(newinet->inet_opt, newopt); 3190 rcu_read_unlock(); 3191 } 3192 3193 struct sock *mptcp_sk_clone_init(const struct sock *sk, 3194 const struct mptcp_options_received *mp_opt, 3195 struct sock *ssk, 3196 struct request_sock *req) 3197 { 3198 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 3199 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 3200 struct mptcp_subflow_context *subflow; 3201 struct mptcp_sock *msk; 3202 3203 if (!nsk) 3204 return NULL; 3205 3206 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3207 if (nsk->sk_family == AF_INET6) 3208 inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 3209 #endif 3210 3211 __mptcp_init_sock(nsk); 3212 3213 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3214 if (nsk->sk_family == AF_INET6) 3215 mptcp_copy_ip6_options(nsk, sk); 3216 else 3217 #endif 3218 mptcp_copy_ip_options(nsk, sk); 3219 3220 msk = mptcp_sk(nsk); 3221 WRITE_ONCE(msk->local_key, subflow_req->local_key); 3222 WRITE_ONCE(msk->token, subflow_req->token); 3223 msk->in_accept_queue = 1; 3224 WRITE_ONCE(msk->fully_established, false); 3225 if (mp_opt->suboptions & OPTION_MPTCP_CSUMREQD) 3226 WRITE_ONCE(msk->csum_enabled, true); 3227 3228 WRITE_ONCE(msk->write_seq, subflow_req->idsn + 1); 3229 WRITE_ONCE(msk->snd_nxt, msk->write_seq); 3230 WRITE_ONCE(msk->snd_una, msk->write_seq); 3231 WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); 3232 msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; 3233 mptcp_init_sched(msk, mptcp_sk(sk)->sched); 3234 3235 /* passive msk is created after the first/MPC subflow */ 3236 msk->subflow_id = 2; 3237 3238 sock_reset_flag(nsk, SOCK_RCU_FREE); 3239 security_inet_csk_clone(nsk, req); 3240 3241 /* this can't race with mptcp_close(), as the msk is 3242 * not yet exposted to user-space 3243 */ 3244 mptcp_set_state(nsk, TCP_ESTABLISHED); 3245 3246 /* The msk maintain a ref to each subflow in the connections list */ 3247 WRITE_ONCE(msk->first, ssk); 3248 subflow = mptcp_subflow_ctx(ssk); 3249 list_add(&subflow->node, &msk->conn_list); 3250 sock_hold(ssk); 3251 3252 /* new mpc subflow takes ownership of the newly 3253 * created mptcp socket 3254 */ 3255 mptcp_token_accept(subflow_req, msk); 3256 3257 /* set msk addresses early to ensure mptcp_pm_get_local_id() 3258 * uses the correct data 3259 */ 3260 mptcp_copy_inaddrs(nsk, ssk); 3261 __mptcp_propagate_sndbuf(nsk, ssk); 3262 3263 mptcp_rcv_space_init(msk, ssk); 3264 3265 if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK) 3266 __mptcp_subflow_fully_established(msk, subflow, mp_opt); 3267 bh_unlock_sock(nsk); 3268 3269 /* note: the newly allocated socket refcount is 2 now */ 3270 return nsk; 3271 } 3272 3273 void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 3274 { 3275 const struct tcp_sock *tp = tcp_sk(ssk); 3276 3277 msk->rcvspace_init = 1; 3278 msk->rcvq_space.copied = 0; 3279 msk->rcvq_space.rtt_us = 0; 3280 3281 msk->rcvq_space.time = tp->tcp_mstamp; 3282 3283 /* initial rcv_space offering made to peer */ 3284 msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 3285 TCP_INIT_CWND * tp->advmss); 3286 if (msk->rcvq_space.space == 0) 3287 msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 3288 } 3289 3290 void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) 3291 { 3292 struct mptcp_subflow_context *subflow, *tmp; 3293 struct sock *sk = (struct sock *)msk; 3294 3295 __mptcp_clear_xmit(sk); 3296 3297 /* join list will be eventually flushed (with rst) at sock lock release time */ 3298 mptcp_for_each_subflow_safe(msk, subflow, tmp) 3299 __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); 3300 3301 __skb_queue_purge(&sk->sk_receive_queue); 3302 skb_rbtree_purge(&msk->out_of_order_queue); 3303 3304 /* move all the rx fwd alloc into the sk_mem_reclaim_final in 3305 * inet_sock_destruct() will dispose it 3306 */ 3307 mptcp_token_destroy(msk); 3308 mptcp_pm_destroy(msk); 3309 } 3310 3311 static void mptcp_destroy(struct sock *sk) 3312 { 3313 struct mptcp_sock *msk = mptcp_sk(sk); 3314 3315 /* allow the following to close even the initial subflow */ 3316 msk->free_first = 1; 3317 mptcp_destroy_common(msk, 0); 3318 sk_sockets_allocated_dec(sk); 3319 } 3320 3321 void __mptcp_data_acked(struct sock *sk) 3322 { 3323 if (!sock_owned_by_user(sk)) 3324 __mptcp_clean_una(sk); 3325 else 3326 __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags); 3327 } 3328 3329 void __mptcp_check_push(struct sock *sk, struct sock *ssk) 3330 { 3331 if (!mptcp_send_head(sk)) 3332 return; 3333 3334 if (!sock_owned_by_user(sk)) 3335 __mptcp_subflow_push_pending(sk, ssk, false); 3336 else 3337 __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 3338 } 3339 3340 #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \ 3341 BIT(MPTCP_RETRANSMIT) | \ 3342 BIT(MPTCP_FLUSH_JOIN_LIST) | \ 3343 BIT(MPTCP_DEQUEUE)) 3344 3345 /* processes deferred events and flush wmem */ 3346 static void mptcp_release_cb(struct sock *sk) 3347 __must_hold(&sk->sk_lock.slock) 3348 { 3349 struct mptcp_sock *msk = mptcp_sk(sk); 3350 3351 for (;;) { 3352 unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED); 3353 struct list_head join_list; 3354 3355 if (!flags) 3356 break; 3357 3358 INIT_LIST_HEAD(&join_list); 3359 list_splice_init(&msk->join_list, &join_list); 3360 3361 /* the following actions acquire the subflow socket lock 3362 * 3363 * 1) can't be invoked in atomic scope 3364 * 2) must avoid ABBA deadlock with msk socket spinlock: the RX 3365 * datapath acquires the msk socket spinlock while helding 3366 * the subflow socket lock 3367 */ 3368 msk->cb_flags &= ~flags; 3369 spin_unlock_bh(&sk->sk_lock.slock); 3370 3371 if (flags & BIT(MPTCP_FLUSH_JOIN_LIST)) 3372 __mptcp_flush_join_list(sk, &join_list); 3373 if (flags & BIT(MPTCP_PUSH_PENDING)) 3374 __mptcp_push_pending(sk, 0); 3375 if (flags & BIT(MPTCP_RETRANSMIT)) 3376 __mptcp_retrans(sk); 3377 if ((flags & BIT(MPTCP_DEQUEUE)) && __mptcp_move_skbs(sk)) { 3378 /* notify ack seq update */ 3379 mptcp_cleanup_rbuf(msk, 0); 3380 sk->sk_data_ready(sk); 3381 } 3382 3383 cond_resched(); 3384 spin_lock_bh(&sk->sk_lock.slock); 3385 } 3386 3387 if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) 3388 __mptcp_clean_una_wakeup(sk); 3389 if (unlikely(msk->cb_flags)) { 3390 /* be sure to sync the msk state before taking actions 3391 * depending on sk_state (MPTCP_ERROR_REPORT) 3392 * On sk release avoid actions depending on the first subflow 3393 */ 3394 if (__test_and_clear_bit(MPTCP_SYNC_STATE, &msk->cb_flags) && msk->first) 3395 __mptcp_sync_state(sk, msk->pending_state); 3396 if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) 3397 __mptcp_error_report(sk); 3398 if (__test_and_clear_bit(MPTCP_SYNC_SNDBUF, &msk->cb_flags)) 3399 __mptcp_sync_sndbuf(sk); 3400 } 3401 } 3402 3403 /* MP_JOIN client subflow must wait for 4th ack before sending any data: 3404 * TCP can't schedule delack timer before the subflow is fully established. 3405 * MPTCP uses the delack timer to do 3rd ack retransmissions 3406 */ 3407 static void schedule_3rdack_retransmission(struct sock *ssk) 3408 { 3409 struct inet_connection_sock *icsk = inet_csk(ssk); 3410 struct tcp_sock *tp = tcp_sk(ssk); 3411 unsigned long timeout; 3412 3413 if (READ_ONCE(mptcp_subflow_ctx(ssk)->fully_established)) 3414 return; 3415 3416 /* reschedule with a timeout above RTT, as we must look only for drop */ 3417 if (tp->srtt_us) 3418 timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1)); 3419 else 3420 timeout = TCP_TIMEOUT_INIT; 3421 timeout += jiffies; 3422 3423 WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER); 3424 smp_store_release(&icsk->icsk_ack.pending, 3425 icsk->icsk_ack.pending | ICSK_ACK_SCHED | ICSK_ACK_TIMER); 3426 sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout); 3427 } 3428 3429 void mptcp_subflow_process_delegated(struct sock *ssk, long status) 3430 { 3431 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3432 struct sock *sk = subflow->conn; 3433 3434 if (status & BIT(MPTCP_DELEGATE_SEND)) { 3435 mptcp_data_lock(sk); 3436 if (!sock_owned_by_user(sk)) 3437 __mptcp_subflow_push_pending(sk, ssk, true); 3438 else 3439 __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 3440 mptcp_data_unlock(sk); 3441 } 3442 if (status & BIT(MPTCP_DELEGATE_SNDBUF)) { 3443 mptcp_data_lock(sk); 3444 if (!sock_owned_by_user(sk)) 3445 __mptcp_sync_sndbuf(sk); 3446 else 3447 __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags); 3448 mptcp_data_unlock(sk); 3449 } 3450 if (status & BIT(MPTCP_DELEGATE_ACK)) 3451 schedule_3rdack_retransmission(ssk); 3452 } 3453 3454 static int mptcp_hash(struct sock *sk) 3455 { 3456 /* should never be called, 3457 * we hash the TCP subflows not the MPTCP socket 3458 */ 3459 WARN_ON_ONCE(1); 3460 return 0; 3461 } 3462 3463 static void mptcp_unhash(struct sock *sk) 3464 { 3465 /* called from sk_common_release(), but nothing to do here */ 3466 } 3467 3468 static int mptcp_get_port(struct sock *sk, unsigned short snum) 3469 { 3470 struct mptcp_sock *msk = mptcp_sk(sk); 3471 3472 pr_debug("msk=%p, ssk=%p\n", msk, msk->first); 3473 if (WARN_ON_ONCE(!msk->first)) 3474 return -EINVAL; 3475 3476 return inet_csk_get_port(msk->first, snum); 3477 } 3478 3479 void mptcp_finish_connect(struct sock *ssk) 3480 { 3481 struct mptcp_subflow_context *subflow; 3482 struct mptcp_sock *msk; 3483 struct sock *sk; 3484 3485 subflow = mptcp_subflow_ctx(ssk); 3486 sk = subflow->conn; 3487 msk = mptcp_sk(sk); 3488 3489 pr_debug("msk=%p, token=%u\n", sk, subflow->token); 3490 3491 subflow->map_seq = subflow->iasn; 3492 subflow->map_subflow_seq = 1; 3493 3494 /* the socket is not connected yet, no msk/subflow ops can access/race 3495 * accessing the field below 3496 */ 3497 WRITE_ONCE(msk->local_key, subflow->local_key); 3498 3499 mptcp_pm_new_connection(msk, ssk, 0); 3500 } 3501 3502 void mptcp_sock_graft(struct sock *sk, struct socket *parent) 3503 { 3504 write_lock_bh(&sk->sk_callback_lock); 3505 rcu_assign_pointer(sk->sk_wq, &parent->wq); 3506 sk_set_socket(sk, parent); 3507 WRITE_ONCE(sk->sk_uid, SOCK_INODE(parent)->i_uid); 3508 write_unlock_bh(&sk->sk_callback_lock); 3509 } 3510 3511 bool mptcp_finish_join(struct sock *ssk) 3512 { 3513 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3514 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 3515 struct sock *parent = (void *)msk; 3516 bool ret = true; 3517 3518 pr_debug("msk=%p, subflow=%p\n", msk, subflow); 3519 3520 /* mptcp socket already closing? */ 3521 if (!mptcp_is_fully_established(parent)) { 3522 subflow->reset_reason = MPTCP_RST_EMPTCP; 3523 return false; 3524 } 3525 3526 /* active subflow, already present inside the conn_list */ 3527 if (!list_empty(&subflow->node)) { 3528 mptcp_subflow_joined(msk, ssk); 3529 mptcp_propagate_sndbuf(parent, ssk); 3530 return true; 3531 } 3532 3533 if (!mptcp_pm_allow_new_subflow(msk)) { 3534 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_JOINREJECTED); 3535 goto err_prohibited; 3536 } 3537 3538 /* If we can't acquire msk socket lock here, let the release callback 3539 * handle it 3540 */ 3541 mptcp_data_lock(parent); 3542 if (!sock_owned_by_user(parent)) { 3543 ret = __mptcp_finish_join(msk, ssk); 3544 if (ret) { 3545 sock_hold(ssk); 3546 list_add_tail(&subflow->node, &msk->conn_list); 3547 } 3548 } else { 3549 sock_hold(ssk); 3550 list_add_tail(&subflow->node, &msk->join_list); 3551 __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); 3552 } 3553 mptcp_data_unlock(parent); 3554 3555 if (!ret) { 3556 err_prohibited: 3557 subflow->reset_reason = MPTCP_RST_EPROHIBIT; 3558 return false; 3559 } 3560 3561 return true; 3562 } 3563 3564 static void mptcp_shutdown(struct sock *sk, int how) 3565 { 3566 pr_debug("sk=%p, how=%d\n", sk, how); 3567 3568 if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) 3569 __mptcp_wr_shutdown(sk); 3570 } 3571 3572 static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) 3573 { 3574 const struct sock *sk = (void *)msk; 3575 u64 delta; 3576 3577 if (sk->sk_state == TCP_LISTEN) 3578 return -EINVAL; 3579 3580 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 3581 return 0; 3582 3583 delta = msk->write_seq - v; 3584 if (__mptcp_check_fallback(msk) && msk->first) { 3585 struct tcp_sock *tp = tcp_sk(msk->first); 3586 3587 /* the first subflow is disconnected after close - see 3588 * __mptcp_close_ssk(). tcp_disconnect() moves the write_seq 3589 * so ignore that status, too. 3590 */ 3591 if (!((1 << msk->first->sk_state) & 3592 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))) 3593 delta += READ_ONCE(tp->write_seq) - tp->snd_una; 3594 } 3595 if (delta > INT_MAX) 3596 delta = INT_MAX; 3597 3598 return (int)delta; 3599 } 3600 3601 static int mptcp_ioctl(struct sock *sk, int cmd, int *karg) 3602 { 3603 struct mptcp_sock *msk = mptcp_sk(sk); 3604 bool slow; 3605 3606 switch (cmd) { 3607 case SIOCINQ: 3608 if (sk->sk_state == TCP_LISTEN) 3609 return -EINVAL; 3610 3611 lock_sock(sk); 3612 if (__mptcp_move_skbs(sk)) 3613 mptcp_cleanup_rbuf(msk, 0); 3614 *karg = mptcp_inq_hint(sk); 3615 release_sock(sk); 3616 break; 3617 case SIOCOUTQ: 3618 slow = lock_sock_fast(sk); 3619 *karg = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); 3620 unlock_sock_fast(sk, slow); 3621 break; 3622 case SIOCOUTQNSD: 3623 slow = lock_sock_fast(sk); 3624 *karg = mptcp_ioctl_outq(msk, msk->snd_nxt); 3625 unlock_sock_fast(sk, slow); 3626 break; 3627 default: 3628 return -ENOIOCTLCMD; 3629 } 3630 3631 return 0; 3632 } 3633 3634 static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) 3635 { 3636 struct mptcp_subflow_context *subflow; 3637 struct mptcp_sock *msk = mptcp_sk(sk); 3638 int err = -EINVAL; 3639 struct sock *ssk; 3640 3641 ssk = __mptcp_nmpc_sk(msk); 3642 if (IS_ERR(ssk)) 3643 return PTR_ERR(ssk); 3644 3645 mptcp_set_state(sk, TCP_SYN_SENT); 3646 subflow = mptcp_subflow_ctx(ssk); 3647 #ifdef CONFIG_TCP_MD5SIG 3648 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 3649 * TCP option space. 3650 */ 3651 if (rcu_access_pointer(tcp_sk(ssk)->md5sig_info)) 3652 mptcp_subflow_early_fallback(msk, subflow); 3653 #endif 3654 if (subflow->request_mptcp) { 3655 if (mptcp_active_should_disable(sk)) { 3656 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEACTIVEDISABLED); 3657 mptcp_subflow_early_fallback(msk, subflow); 3658 } else if (mptcp_token_new_connect(ssk) < 0) { 3659 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_TOKENFALLBACKINIT); 3660 mptcp_subflow_early_fallback(msk, subflow); 3661 } 3662 } 3663 3664 WRITE_ONCE(msk->write_seq, subflow->idsn); 3665 WRITE_ONCE(msk->snd_nxt, subflow->idsn); 3666 WRITE_ONCE(msk->snd_una, subflow->idsn); 3667 if (likely(!__mptcp_check_fallback(msk))) 3668 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE); 3669 3670 /* if reaching here via the fastopen/sendmsg path, the caller already 3671 * acquired the subflow socket lock, too. 3672 */ 3673 if (!msk->fastopening) 3674 lock_sock(ssk); 3675 3676 /* the following mirrors closely a very small chunk of code from 3677 * __inet_stream_connect() 3678 */ 3679 if (ssk->sk_state != TCP_CLOSE) 3680 goto out; 3681 3682 if (BPF_CGROUP_PRE_CONNECT_ENABLED(ssk)) { 3683 err = ssk->sk_prot->pre_connect(ssk, uaddr, addr_len); 3684 if (err) 3685 goto out; 3686 } 3687 3688 err = ssk->sk_prot->connect(ssk, uaddr, addr_len); 3689 if (err < 0) 3690 goto out; 3691 3692 inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk)); 3693 3694 out: 3695 if (!msk->fastopening) 3696 release_sock(ssk); 3697 3698 /* on successful connect, the msk state will be moved to established by 3699 * subflow_finish_connect() 3700 */ 3701 if (unlikely(err)) { 3702 /* avoid leaving a dangling token in an unconnected socket */ 3703 mptcp_token_destroy(msk); 3704 mptcp_set_state(sk, TCP_CLOSE); 3705 return err; 3706 } 3707 3708 mptcp_copy_inaddrs(sk, ssk); 3709 return 0; 3710 } 3711 3712 static struct proto mptcp_prot = { 3713 .name = "MPTCP", 3714 .owner = THIS_MODULE, 3715 .init = mptcp_init_sock, 3716 .connect = mptcp_connect, 3717 .disconnect = mptcp_disconnect, 3718 .close = mptcp_close, 3719 .setsockopt = mptcp_setsockopt, 3720 .getsockopt = mptcp_getsockopt, 3721 .shutdown = mptcp_shutdown, 3722 .destroy = mptcp_destroy, 3723 .sendmsg = mptcp_sendmsg, 3724 .ioctl = mptcp_ioctl, 3725 .recvmsg = mptcp_recvmsg, 3726 .release_cb = mptcp_release_cb, 3727 .hash = mptcp_hash, 3728 .unhash = mptcp_unhash, 3729 .get_port = mptcp_get_port, 3730 .stream_memory_free = mptcp_stream_memory_free, 3731 .sockets_allocated = &mptcp_sockets_allocated, 3732 3733 .memory_allocated = &net_aligned_data.tcp_memory_allocated, 3734 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 3735 3736 .memory_pressure = &tcp_memory_pressure, 3737 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3738 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3739 .sysctl_mem = sysctl_tcp_mem, 3740 .obj_size = sizeof(struct mptcp_sock), 3741 .slab_flags = SLAB_TYPESAFE_BY_RCU, 3742 .no_autobind = true, 3743 }; 3744 3745 static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 3746 { 3747 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3748 struct sock *ssk, *sk = sock->sk; 3749 int err = -EINVAL; 3750 3751 lock_sock(sk); 3752 ssk = __mptcp_nmpc_sk(msk); 3753 if (IS_ERR(ssk)) { 3754 err = PTR_ERR(ssk); 3755 goto unlock; 3756 } 3757 3758 if (sk->sk_family == AF_INET) 3759 err = inet_bind_sk(ssk, uaddr, addr_len); 3760 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3761 else if (sk->sk_family == AF_INET6) 3762 err = inet6_bind_sk(ssk, uaddr, addr_len); 3763 #endif 3764 if (!err) 3765 mptcp_copy_inaddrs(sk, ssk); 3766 3767 unlock: 3768 release_sock(sk); 3769 return err; 3770 } 3771 3772 static int mptcp_listen(struct socket *sock, int backlog) 3773 { 3774 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3775 struct sock *sk = sock->sk; 3776 struct sock *ssk; 3777 int err; 3778 3779 pr_debug("msk=%p\n", msk); 3780 3781 lock_sock(sk); 3782 3783 err = -EINVAL; 3784 if (sock->state != SS_UNCONNECTED || sock->type != SOCK_STREAM) 3785 goto unlock; 3786 3787 ssk = __mptcp_nmpc_sk(msk); 3788 if (IS_ERR(ssk)) { 3789 err = PTR_ERR(ssk); 3790 goto unlock; 3791 } 3792 3793 mptcp_set_state(sk, TCP_LISTEN); 3794 sock_set_flag(sk, SOCK_RCU_FREE); 3795 3796 lock_sock(ssk); 3797 err = __inet_listen_sk(ssk, backlog); 3798 release_sock(ssk); 3799 mptcp_set_state(sk, inet_sk_state_load(ssk)); 3800 3801 if (!err) { 3802 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 3803 mptcp_copy_inaddrs(sk, ssk); 3804 mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED); 3805 } 3806 3807 unlock: 3808 release_sock(sk); 3809 return err; 3810 } 3811 3812 static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 3813 struct proto_accept_arg *arg) 3814 { 3815 struct mptcp_sock *msk = mptcp_sk(sock->sk); 3816 struct sock *ssk, *newsk; 3817 3818 pr_debug("msk=%p\n", msk); 3819 3820 /* Buggy applications can call accept on socket states other then LISTEN 3821 * but no need to allocate the first subflow just to error out. 3822 */ 3823 ssk = READ_ONCE(msk->first); 3824 if (!ssk) 3825 return -EINVAL; 3826 3827 pr_debug("ssk=%p, listener=%p\n", ssk, mptcp_subflow_ctx(ssk)); 3828 newsk = inet_csk_accept(ssk, arg); 3829 if (!newsk) 3830 return arg->err; 3831 3832 pr_debug("newsk=%p, subflow is mptcp=%d\n", newsk, sk_is_mptcp(newsk)); 3833 if (sk_is_mptcp(newsk)) { 3834 struct mptcp_subflow_context *subflow; 3835 struct sock *new_mptcp_sock; 3836 3837 subflow = mptcp_subflow_ctx(newsk); 3838 new_mptcp_sock = subflow->conn; 3839 3840 /* is_mptcp should be false if subflow->conn is missing, see 3841 * subflow_syn_recv_sock() 3842 */ 3843 if (WARN_ON_ONCE(!new_mptcp_sock)) { 3844 tcp_sk(newsk)->is_mptcp = 0; 3845 goto tcpfallback; 3846 } 3847 3848 newsk = new_mptcp_sock; 3849 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 3850 3851 newsk->sk_kern_sock = arg->kern; 3852 lock_sock(newsk); 3853 __inet_accept(sock, newsock, newsk); 3854 3855 set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags); 3856 msk = mptcp_sk(newsk); 3857 msk->in_accept_queue = 0; 3858 3859 /* set ssk->sk_socket of accept()ed flows to mptcp socket. 3860 * This is needed so NOSPACE flag can be set from tcp stack. 3861 */ 3862 mptcp_for_each_subflow(msk, subflow) { 3863 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3864 3865 if (!ssk->sk_socket) 3866 mptcp_sock_graft(ssk, newsock); 3867 } 3868 3869 /* Do late cleanup for the first subflow as necessary. Also 3870 * deal with bad peers not doing a complete shutdown. 3871 */ 3872 if (unlikely(inet_sk_state_load(msk->first) == TCP_CLOSE)) { 3873 __mptcp_close_ssk(newsk, msk->first, 3874 mptcp_subflow_ctx(msk->first), 0); 3875 if (unlikely(list_is_singular(&msk->conn_list))) 3876 mptcp_set_state(newsk, TCP_CLOSE); 3877 } 3878 } else { 3879 tcpfallback: 3880 newsk->sk_kern_sock = arg->kern; 3881 lock_sock(newsk); 3882 __inet_accept(sock, newsock, newsk); 3883 /* we are being invoked after accepting a non-mp-capable 3884 * flow: sk is a tcp_sk, not an mptcp one. 3885 * 3886 * Hand the socket over to tcp so all further socket ops 3887 * bypass mptcp. 3888 */ 3889 WRITE_ONCE(newsock->sk->sk_socket->ops, 3890 mptcp_fallback_tcp_ops(newsock->sk)); 3891 } 3892 release_sock(newsk); 3893 3894 return 0; 3895 } 3896 3897 static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) 3898 { 3899 struct sock *sk = (struct sock *)msk; 3900 3901 if (__mptcp_stream_is_writeable(sk, 1)) 3902 return EPOLLOUT | EPOLLWRNORM; 3903 3904 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 3905 smp_mb__after_atomic(); /* NOSPACE is changed by mptcp_write_space() */ 3906 if (__mptcp_stream_is_writeable(sk, 1)) 3907 return EPOLLOUT | EPOLLWRNORM; 3908 3909 return 0; 3910 } 3911 3912 static __poll_t mptcp_poll(struct file *file, struct socket *sock, 3913 struct poll_table_struct *wait) 3914 { 3915 struct sock *sk = sock->sk; 3916 struct mptcp_sock *msk; 3917 __poll_t mask = 0; 3918 u8 shutdown; 3919 int state; 3920 3921 msk = mptcp_sk(sk); 3922 sock_poll_wait(file, sock, wait); 3923 3924 state = inet_sk_state_load(sk); 3925 pr_debug("msk=%p state=%d flags=%lx\n", msk, state, msk->flags); 3926 if (state == TCP_LISTEN) { 3927 struct sock *ssk = READ_ONCE(msk->first); 3928 3929 if (WARN_ON_ONCE(!ssk)) 3930 return 0; 3931 3932 return inet_csk_listen_poll(ssk); 3933 } 3934 3935 shutdown = READ_ONCE(sk->sk_shutdown); 3936 if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 3937 mask |= EPOLLHUP; 3938 if (shutdown & RCV_SHUTDOWN) 3939 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 3940 3941 if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 3942 mask |= mptcp_check_readable(sk); 3943 if (shutdown & SEND_SHUTDOWN) 3944 mask |= EPOLLOUT | EPOLLWRNORM; 3945 else 3946 mask |= mptcp_check_writeable(msk); 3947 } else if (state == TCP_SYN_SENT && 3948 inet_test_bit(DEFER_CONNECT, sk)) { 3949 /* cf tcp_poll() note about TFO */ 3950 mask |= EPOLLOUT | EPOLLWRNORM; 3951 } 3952 3953 /* This barrier is coupled with smp_wmb() in __mptcp_error_report() */ 3954 smp_rmb(); 3955 if (READ_ONCE(sk->sk_err)) 3956 mask |= EPOLLERR; 3957 3958 return mask; 3959 } 3960 3961 static const struct proto_ops mptcp_stream_ops = { 3962 .family = PF_INET, 3963 .owner = THIS_MODULE, 3964 .release = inet_release, 3965 .bind = mptcp_bind, 3966 .connect = inet_stream_connect, 3967 .socketpair = sock_no_socketpair, 3968 .accept = mptcp_stream_accept, 3969 .getname = inet_getname, 3970 .poll = mptcp_poll, 3971 .ioctl = inet_ioctl, 3972 .gettstamp = sock_gettstamp, 3973 .listen = mptcp_listen, 3974 .shutdown = inet_shutdown, 3975 .setsockopt = sock_common_setsockopt, 3976 .getsockopt = sock_common_getsockopt, 3977 .sendmsg = inet_sendmsg, 3978 .recvmsg = inet_recvmsg, 3979 .mmap = sock_no_mmap, 3980 .set_rcvlowat = mptcp_set_rcvlowat, 3981 }; 3982 3983 static struct inet_protosw mptcp_protosw = { 3984 .type = SOCK_STREAM, 3985 .protocol = IPPROTO_MPTCP, 3986 .prot = &mptcp_prot, 3987 .ops = &mptcp_stream_ops, 3988 .flags = INET_PROTOSW_ICSK, 3989 }; 3990 3991 static int mptcp_napi_poll(struct napi_struct *napi, int budget) 3992 { 3993 struct mptcp_delegated_action *delegated; 3994 struct mptcp_subflow_context *subflow; 3995 int work_done = 0; 3996 3997 delegated = container_of(napi, struct mptcp_delegated_action, napi); 3998 while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { 3999 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 4000 4001 bh_lock_sock_nested(ssk); 4002 if (!sock_owned_by_user(ssk)) { 4003 mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0)); 4004 } else { 4005 /* tcp_release_cb_override already processed 4006 * the action or will do at next release_sock(). 4007 * In both case must dequeue the subflow here - on the same 4008 * CPU that scheduled it. 4009 */ 4010 smp_wmb(); 4011 clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status); 4012 } 4013 bh_unlock_sock(ssk); 4014 sock_put(ssk); 4015 4016 if (++work_done == budget) 4017 return budget; 4018 } 4019 4020 /* always provide a 0 'work_done' argument, so that napi_complete_done 4021 * will not try accessing the NULL napi->dev ptr 4022 */ 4023 napi_complete_done(napi, 0); 4024 return work_done; 4025 } 4026 4027 void __init mptcp_proto_init(void) 4028 { 4029 struct mptcp_delegated_action *delegated; 4030 int cpu; 4031 4032 mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 4033 4034 if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 4035 panic("Failed to allocate MPTCP pcpu counter\n"); 4036 4037 mptcp_napi_dev = alloc_netdev_dummy(0); 4038 if (!mptcp_napi_dev) 4039 panic("Failed to allocate MPTCP dummy netdev\n"); 4040 for_each_possible_cpu(cpu) { 4041 delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); 4042 INIT_LIST_HEAD(&delegated->head); 4043 netif_napi_add_tx(mptcp_napi_dev, &delegated->napi, 4044 mptcp_napi_poll); 4045 napi_enable(&delegated->napi); 4046 } 4047 4048 mptcp_subflow_init(); 4049 mptcp_pm_init(); 4050 mptcp_sched_init(); 4051 mptcp_token_init(); 4052 4053 if (proto_register(&mptcp_prot, 1) != 0) 4054 panic("Failed to register MPTCP proto.\n"); 4055 4056 inet_register_protosw(&mptcp_protosw); 4057 4058 BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 4059 } 4060 4061 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 4062 static const struct proto_ops mptcp_v6_stream_ops = { 4063 .family = PF_INET6, 4064 .owner = THIS_MODULE, 4065 .release = inet6_release, 4066 .bind = mptcp_bind, 4067 .connect = inet_stream_connect, 4068 .socketpair = sock_no_socketpair, 4069 .accept = mptcp_stream_accept, 4070 .getname = inet6_getname, 4071 .poll = mptcp_poll, 4072 .ioctl = inet6_ioctl, 4073 .gettstamp = sock_gettstamp, 4074 .listen = mptcp_listen, 4075 .shutdown = inet_shutdown, 4076 .setsockopt = sock_common_setsockopt, 4077 .getsockopt = sock_common_getsockopt, 4078 .sendmsg = inet6_sendmsg, 4079 .recvmsg = inet6_recvmsg, 4080 .mmap = sock_no_mmap, 4081 #ifdef CONFIG_COMPAT 4082 .compat_ioctl = inet6_compat_ioctl, 4083 #endif 4084 .set_rcvlowat = mptcp_set_rcvlowat, 4085 }; 4086 4087 static struct proto mptcp_v6_prot; 4088 4089 static struct inet_protosw mptcp_v6_protosw = { 4090 .type = SOCK_STREAM, 4091 .protocol = IPPROTO_MPTCP, 4092 .prot = &mptcp_v6_prot, 4093 .ops = &mptcp_v6_stream_ops, 4094 .flags = INET_PROTOSW_ICSK, 4095 }; 4096 4097 int __init mptcp_proto_v6_init(void) 4098 { 4099 int err; 4100 4101 mptcp_v6_prot = mptcp_prot; 4102 strscpy(mptcp_v6_prot.name, "MPTCPv6", sizeof(mptcp_v6_prot.name)); 4103 mptcp_v6_prot.slab = NULL; 4104 mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 4105 mptcp_v6_prot.ipv6_pinfo_offset = offsetof(struct mptcp6_sock, np); 4106 4107 err = proto_register(&mptcp_v6_prot, 1); 4108 if (err) 4109 return err; 4110 4111 err = inet6_register_protosw(&mptcp_v6_protosw); 4112 if (err) 4113 proto_unregister(&mptcp_v6_prot); 4114 4115 return err; 4116 } 4117 #endif 4118