1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <linux/sched/signal.h> 13 #include <linux/atomic.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20 #include <net/transp_v6.h> 21 #endif 22 #include <net/mptcp.h> 23 #include "protocol.h" 24 25 #define MPTCP_SAME_STATE TCP_MAX_STATES 26 27 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 28 struct mptcp6_sock { 29 struct mptcp_sock msk; 30 struct ipv6_pinfo np; 31 }; 32 #endif 33 34 /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 35 * completed yet or has failed, return the subflow socket. 36 * Otherwise return NULL. 37 */ 38 static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 39 { 40 if (!msk->subflow || READ_ONCE(msk->can_ack)) 41 return NULL; 42 43 return msk->subflow; 44 } 45 46 static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk) 47 { 48 return msk->first && !sk_is_mptcp(msk->first); 49 } 50 51 static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk) 52 { 53 sock_owned_by_me((const struct sock *)msk); 54 55 if (likely(!__mptcp_needs_tcp_fallback(msk))) 56 return NULL; 57 58 if (msk->subflow) { 59 release_sock((struct sock *)msk); 60 return msk->subflow; 61 } 62 63 return NULL; 64 } 65 66 static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk) 67 { 68 return !msk->first; 69 } 70 71 static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state) 72 { 73 struct mptcp_subflow_context *subflow; 74 struct sock *sk = (struct sock *)msk; 75 struct socket *ssock; 76 int err; 77 78 ssock = __mptcp_nmpc_socket(msk); 79 if (ssock) 80 goto set_state; 81 82 if (!__mptcp_can_create_subflow(msk)) 83 return ERR_PTR(-EINVAL); 84 85 err = mptcp_subflow_create_socket(sk, &ssock); 86 if (err) 87 return ERR_PTR(err); 88 89 msk->first = ssock->sk; 90 msk->subflow = ssock; 91 subflow = mptcp_subflow_ctx(ssock->sk); 92 list_add(&subflow->node, &msk->conn_list); 93 subflow->request_mptcp = 1; 94 95 set_state: 96 if (state != MPTCP_SAME_STATE) 97 inet_sk_state_store(sk, state); 98 return ssock; 99 } 100 101 static struct sock *mptcp_subflow_get(const struct mptcp_sock *msk) 102 { 103 struct mptcp_subflow_context *subflow; 104 105 sock_owned_by_me((const struct sock *)msk); 106 107 mptcp_for_each_subflow(msk, subflow) { 108 return mptcp_subflow_tcp_sock(subflow); 109 } 110 111 return NULL; 112 } 113 114 static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 115 { 116 if (!msk->cached_ext) 117 msk->cached_ext = __skb_ext_alloc(); 118 119 return !!msk->cached_ext; 120 } 121 122 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 123 { 124 struct mptcp_subflow_context *subflow; 125 struct sock *sk = (struct sock *)msk; 126 127 sock_owned_by_me(sk); 128 129 mptcp_for_each_subflow(msk, subflow) { 130 if (subflow->data_avail) 131 return mptcp_subflow_tcp_sock(subflow); 132 } 133 134 return NULL; 135 } 136 137 static inline bool mptcp_skb_can_collapse_to(const struct mptcp_sock *msk, 138 const struct sk_buff *skb, 139 const struct mptcp_ext *mpext) 140 { 141 if (!tcp_skb_can_collapse_to(skb)) 142 return false; 143 144 /* can collapse only if MPTCP level sequence is in order */ 145 return mpext && mpext->data_seq + mpext->data_len == msk->write_seq; 146 } 147 148 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 149 struct msghdr *msg, long *timeo, int *pmss_now, 150 int *ps_goal) 151 { 152 int mss_now, avail_size, size_goal, ret; 153 struct mptcp_sock *msk = mptcp_sk(sk); 154 struct mptcp_ext *mpext = NULL; 155 struct sk_buff *skb, *tail; 156 bool can_collapse = false; 157 struct page_frag *pfrag; 158 size_t psize; 159 160 /* use the mptcp page cache so that we can easily move the data 161 * from one substream to another, but do per subflow memory accounting 162 */ 163 pfrag = sk_page_frag(sk); 164 while (!sk_page_frag_refill(ssk, pfrag) || 165 !mptcp_ext_cache_refill(msk)) { 166 ret = sk_stream_wait_memory(ssk, timeo); 167 if (ret) 168 return ret; 169 if (unlikely(__mptcp_needs_tcp_fallback(msk))) 170 return 0; 171 } 172 173 /* compute copy limit */ 174 mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 175 *pmss_now = mss_now; 176 *ps_goal = size_goal; 177 avail_size = size_goal; 178 skb = tcp_write_queue_tail(ssk); 179 if (skb) { 180 mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 181 182 /* Limit the write to the size available in the 183 * current skb, if any, so that we create at most a new skb. 184 * Explicitly tells TCP internals to avoid collapsing on later 185 * queue management operation, to avoid breaking the ext <-> 186 * SSN association set here 187 */ 188 can_collapse = (size_goal - skb->len > 0) && 189 mptcp_skb_can_collapse_to(msk, skb, mpext); 190 if (!can_collapse) 191 TCP_SKB_CB(skb)->eor = 1; 192 else 193 avail_size = size_goal - skb->len; 194 } 195 psize = min_t(size_t, pfrag->size - pfrag->offset, avail_size); 196 197 /* Copy to page */ 198 pr_debug("left=%zu", msg_data_left(msg)); 199 psize = copy_page_from_iter(pfrag->page, pfrag->offset, 200 min_t(size_t, msg_data_left(msg), psize), 201 &msg->msg_iter); 202 pr_debug("left=%zu", msg_data_left(msg)); 203 if (!psize) 204 return -EINVAL; 205 206 /* tell the TCP stack to delay the push so that we can safely 207 * access the skb after the sendpages call 208 */ 209 ret = do_tcp_sendpages(ssk, pfrag->page, pfrag->offset, psize, 210 msg->msg_flags | MSG_SENDPAGE_NOTLAST); 211 if (ret <= 0) 212 return ret; 213 if (unlikely(ret < psize)) 214 iov_iter_revert(&msg->msg_iter, psize - ret); 215 216 /* if the tail skb extension is still the cached one, collapsing 217 * really happened. Note: we can't check for 'same skb' as the sk_buff 218 * hdr on tail can be transmitted, freed and re-allocated by the 219 * do_tcp_sendpages() call 220 */ 221 tail = tcp_write_queue_tail(ssk); 222 if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 223 WARN_ON_ONCE(!can_collapse); 224 mpext->data_len += ret; 225 goto out; 226 } 227 228 skb = tcp_write_queue_tail(ssk); 229 mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 230 msk->cached_ext = NULL; 231 232 memset(mpext, 0, sizeof(*mpext)); 233 mpext->data_seq = msk->write_seq; 234 mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 235 mpext->data_len = ret; 236 mpext->use_map = 1; 237 mpext->dsn64 = 1; 238 239 pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 240 mpext->data_seq, mpext->subflow_seq, mpext->data_len, 241 mpext->dsn64); 242 243 out: 244 pfrag->offset += ret; 245 msk->write_seq += ret; 246 mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 247 248 return ret; 249 } 250 251 static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk) 252 { 253 struct socket *sock; 254 255 if (likely(sk_stream_is_writeable(ssk))) 256 return; 257 258 sock = READ_ONCE(ssk->sk_socket); 259 260 if (sock) { 261 clear_bit(MPTCP_SEND_SPACE, &msk->flags); 262 smp_mb__after_atomic(); 263 /* set NOSPACE only after clearing SEND_SPACE flag */ 264 set_bit(SOCK_NOSPACE, &sock->flags); 265 } 266 } 267 268 static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 269 { 270 int mss_now = 0, size_goal = 0, ret = 0; 271 struct mptcp_sock *msk = mptcp_sk(sk); 272 struct socket *ssock; 273 size_t copied = 0; 274 struct sock *ssk; 275 long timeo; 276 277 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 278 return -EOPNOTSUPP; 279 280 lock_sock(sk); 281 ssock = __mptcp_tcp_fallback(msk); 282 if (unlikely(ssock)) { 283 fallback: 284 pr_debug("fallback passthrough"); 285 ret = sock_sendmsg(ssock, msg); 286 return ret >= 0 ? ret + copied : (copied ? copied : ret); 287 } 288 289 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 290 291 ssk = mptcp_subflow_get(msk); 292 if (!ssk) { 293 release_sock(sk); 294 return -ENOTCONN; 295 } 296 297 pr_debug("conn_list->subflow=%p", ssk); 298 299 lock_sock(ssk); 300 while (msg_data_left(msg)) { 301 ret = mptcp_sendmsg_frag(sk, ssk, msg, &timeo, &mss_now, 302 &size_goal); 303 if (ret < 0) 304 break; 305 if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) { 306 release_sock(ssk); 307 ssock = __mptcp_tcp_fallback(msk); 308 goto fallback; 309 } 310 311 copied += ret; 312 } 313 314 if (copied) { 315 ret = copied; 316 tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 317 size_goal); 318 } 319 320 ssk_check_wmem(msk, ssk); 321 release_sock(ssk); 322 release_sock(sk); 323 return ret; 324 } 325 326 int mptcp_read_actor(read_descriptor_t *desc, struct sk_buff *skb, 327 unsigned int offset, size_t len) 328 { 329 struct mptcp_read_arg *arg = desc->arg.data; 330 size_t copy_len; 331 332 copy_len = min(desc->count, len); 333 334 if (likely(arg->msg)) { 335 int err; 336 337 err = skb_copy_datagram_msg(skb, offset, arg->msg, copy_len); 338 if (err) { 339 pr_debug("error path"); 340 desc->error = err; 341 return err; 342 } 343 } else { 344 pr_debug("Flushing skb payload"); 345 } 346 347 desc->count -= copy_len; 348 349 pr_debug("consumed %zu bytes, %zu left", copy_len, desc->count); 350 return copy_len; 351 } 352 353 static void mptcp_wait_data(struct sock *sk, long *timeo) 354 { 355 DEFINE_WAIT_FUNC(wait, woken_wake_function); 356 struct mptcp_sock *msk = mptcp_sk(sk); 357 358 add_wait_queue(sk_sleep(sk), &wait); 359 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 360 361 sk_wait_event(sk, timeo, 362 test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 363 364 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 365 remove_wait_queue(sk_sleep(sk), &wait); 366 } 367 368 static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 369 int nonblock, int flags, int *addr_len) 370 { 371 struct mptcp_sock *msk = mptcp_sk(sk); 372 struct mptcp_subflow_context *subflow; 373 bool more_data_avail = false; 374 struct mptcp_read_arg arg; 375 read_descriptor_t desc; 376 bool wait_data = false; 377 struct socket *ssock; 378 struct tcp_sock *tp; 379 bool done = false; 380 struct sock *ssk; 381 int copied = 0; 382 int target; 383 long timeo; 384 385 if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 386 return -EOPNOTSUPP; 387 388 lock_sock(sk); 389 ssock = __mptcp_tcp_fallback(msk); 390 if (unlikely(ssock)) { 391 fallback: 392 pr_debug("fallback-read subflow=%p", 393 mptcp_subflow_ctx(ssock->sk)); 394 copied = sock_recvmsg(ssock, msg, flags); 395 return copied; 396 } 397 398 arg.msg = msg; 399 desc.arg.data = &arg; 400 desc.error = 0; 401 402 timeo = sock_rcvtimeo(sk, nonblock); 403 404 len = min_t(size_t, len, INT_MAX); 405 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 406 407 while (!done) { 408 u32 map_remaining; 409 int bytes_read; 410 411 ssk = mptcp_subflow_recv_lookup(msk); 412 pr_debug("msk=%p ssk=%p", msk, ssk); 413 if (!ssk) 414 goto wait_for_data; 415 416 subflow = mptcp_subflow_ctx(ssk); 417 tp = tcp_sk(ssk); 418 419 lock_sock(ssk); 420 do { 421 /* try to read as much data as available */ 422 map_remaining = subflow->map_data_len - 423 mptcp_subflow_get_map_offset(subflow); 424 desc.count = min_t(size_t, len - copied, map_remaining); 425 pr_debug("reading %zu bytes, copied %d", desc.count, 426 copied); 427 bytes_read = tcp_read_sock(ssk, &desc, 428 mptcp_read_actor); 429 if (bytes_read < 0) { 430 if (!copied) 431 copied = bytes_read; 432 done = true; 433 goto next; 434 } 435 436 pr_debug("msk ack_seq=%llx -> %llx", msk->ack_seq, 437 msk->ack_seq + bytes_read); 438 msk->ack_seq += bytes_read; 439 copied += bytes_read; 440 if (copied >= len) { 441 done = true; 442 goto next; 443 } 444 if (tp->urg_data && tp->urg_seq == tp->copied_seq) { 445 pr_err("Urgent data present, cannot proceed"); 446 done = true; 447 goto next; 448 } 449 next: 450 more_data_avail = mptcp_subflow_data_available(ssk); 451 } while (more_data_avail && !done); 452 release_sock(ssk); 453 continue; 454 455 wait_for_data: 456 more_data_avail = false; 457 458 /* only the master socket status is relevant here. The exit 459 * conditions mirror closely tcp_recvmsg() 460 */ 461 if (copied >= target) 462 break; 463 464 if (copied) { 465 if (sk->sk_err || 466 sk->sk_state == TCP_CLOSE || 467 (sk->sk_shutdown & RCV_SHUTDOWN) || 468 !timeo || 469 signal_pending(current)) 470 break; 471 } else { 472 if (sk->sk_err) { 473 copied = sock_error(sk); 474 break; 475 } 476 477 if (sk->sk_shutdown & RCV_SHUTDOWN) 478 break; 479 480 if (sk->sk_state == TCP_CLOSE) { 481 copied = -ENOTCONN; 482 break; 483 } 484 485 if (!timeo) { 486 copied = -EAGAIN; 487 break; 488 } 489 490 if (signal_pending(current)) { 491 copied = sock_intr_errno(timeo); 492 break; 493 } 494 } 495 496 pr_debug("block timeout %ld", timeo); 497 wait_data = true; 498 mptcp_wait_data(sk, &timeo); 499 if (unlikely(__mptcp_tcp_fallback(msk))) 500 goto fallback; 501 } 502 503 if (more_data_avail) { 504 if (!test_bit(MPTCP_DATA_READY, &msk->flags)) 505 set_bit(MPTCP_DATA_READY, &msk->flags); 506 } else if (!wait_data) { 507 clear_bit(MPTCP_DATA_READY, &msk->flags); 508 509 /* .. race-breaker: ssk might get new data after last 510 * data_available() returns false. 511 */ 512 ssk = mptcp_subflow_recv_lookup(msk); 513 if (unlikely(ssk)) 514 set_bit(MPTCP_DATA_READY, &msk->flags); 515 } 516 517 release_sock(sk); 518 return copied; 519 } 520 521 /* subflow sockets can be either outgoing (connect) or incoming 522 * (accept). 523 * 524 * Outgoing subflows use in-kernel sockets. 525 * Incoming subflows do not have their own 'struct socket' allocated, 526 * so we need to use tcp_close() after detaching them from the mptcp 527 * parent socket. 528 */ 529 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 530 struct mptcp_subflow_context *subflow, 531 long timeout) 532 { 533 struct socket *sock = READ_ONCE(ssk->sk_socket); 534 535 list_del(&subflow->node); 536 537 if (sock && sock != sk->sk_socket) { 538 /* outgoing subflow */ 539 sock_release(sock); 540 } else { 541 /* incoming subflow */ 542 tcp_close(ssk, timeout); 543 } 544 } 545 546 static int __mptcp_init_sock(struct sock *sk) 547 { 548 struct mptcp_sock *msk = mptcp_sk(sk); 549 550 INIT_LIST_HEAD(&msk->conn_list); 551 __set_bit(MPTCP_SEND_SPACE, &msk->flags); 552 553 msk->first = NULL; 554 555 return 0; 556 } 557 558 static int mptcp_init_sock(struct sock *sk) 559 { 560 if (!mptcp_is_enabled(sock_net(sk))) 561 return -ENOPROTOOPT; 562 563 return __mptcp_init_sock(sk); 564 } 565 566 static void mptcp_subflow_shutdown(struct sock *ssk, int how) 567 { 568 lock_sock(ssk); 569 570 switch (ssk->sk_state) { 571 case TCP_LISTEN: 572 if (!(how & RCV_SHUTDOWN)) 573 break; 574 /* fall through */ 575 case TCP_SYN_SENT: 576 tcp_disconnect(ssk, O_NONBLOCK); 577 break; 578 default: 579 ssk->sk_shutdown |= how; 580 tcp_shutdown(ssk, how); 581 break; 582 } 583 584 /* Wake up anyone sleeping in poll. */ 585 ssk->sk_state_change(ssk); 586 release_sock(ssk); 587 } 588 589 /* Called with msk lock held, releases such lock before returning */ 590 static void mptcp_close(struct sock *sk, long timeout) 591 { 592 struct mptcp_subflow_context *subflow, *tmp; 593 struct mptcp_sock *msk = mptcp_sk(sk); 594 LIST_HEAD(conn_list); 595 596 lock_sock(sk); 597 598 mptcp_token_destroy(msk->token); 599 inet_sk_state_store(sk, TCP_CLOSE); 600 601 list_splice_init(&msk->conn_list, &conn_list); 602 603 release_sock(sk); 604 605 list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 606 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 607 608 __mptcp_close_ssk(sk, ssk, subflow, timeout); 609 } 610 611 sk_common_release(sk); 612 } 613 614 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 615 { 616 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 617 const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 618 struct ipv6_pinfo *msk6 = inet6_sk(msk); 619 620 msk->sk_v6_daddr = ssk->sk_v6_daddr; 621 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 622 623 if (msk6 && ssk6) { 624 msk6->saddr = ssk6->saddr; 625 msk6->flow_label = ssk6->flow_label; 626 } 627 #endif 628 629 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 630 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 631 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 632 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 633 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 634 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 635 } 636 637 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 638 static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 639 { 640 unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 641 642 return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 643 } 644 #endif 645 646 static struct sock *mptcp_sk_clone_lock(const struct sock *sk) 647 { 648 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 649 650 if (!nsk) 651 return NULL; 652 653 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 654 if (nsk->sk_family == AF_INET6) 655 inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 656 #endif 657 658 return nsk; 659 } 660 661 static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 662 bool kern) 663 { 664 struct mptcp_sock *msk = mptcp_sk(sk); 665 struct socket *listener; 666 struct sock *newsk; 667 668 listener = __mptcp_nmpc_socket(msk); 669 if (WARN_ON_ONCE(!listener)) { 670 *err = -EINVAL; 671 return NULL; 672 } 673 674 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 675 newsk = inet_csk_accept(listener->sk, flags, err, kern); 676 if (!newsk) 677 return NULL; 678 679 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 680 681 if (sk_is_mptcp(newsk)) { 682 struct mptcp_subflow_context *subflow; 683 struct sock *new_mptcp_sock; 684 struct sock *ssk = newsk; 685 u64 ack_seq; 686 687 subflow = mptcp_subflow_ctx(newsk); 688 lock_sock(sk); 689 690 local_bh_disable(); 691 new_mptcp_sock = mptcp_sk_clone_lock(sk); 692 if (!new_mptcp_sock) { 693 *err = -ENOBUFS; 694 local_bh_enable(); 695 release_sock(sk); 696 mptcp_subflow_shutdown(newsk, SHUT_RDWR + 1); 697 tcp_close(newsk, 0); 698 return NULL; 699 } 700 701 __mptcp_init_sock(new_mptcp_sock); 702 703 msk = mptcp_sk(new_mptcp_sock); 704 msk->local_key = subflow->local_key; 705 msk->token = subflow->token; 706 msk->subflow = NULL; 707 msk->first = newsk; 708 709 mptcp_token_update_accept(newsk, new_mptcp_sock); 710 711 msk->write_seq = subflow->idsn + 1; 712 if (subflow->can_ack) { 713 msk->can_ack = true; 714 msk->remote_key = subflow->remote_key; 715 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 716 ack_seq++; 717 msk->ack_seq = ack_seq; 718 } 719 newsk = new_mptcp_sock; 720 mptcp_copy_inaddrs(newsk, ssk); 721 list_add(&subflow->node, &msk->conn_list); 722 723 /* will be fully established at mptcp_stream_accept() 724 * completion. 725 */ 726 inet_sk_state_store(new_mptcp_sock, TCP_SYN_RECV); 727 bh_unlock_sock(new_mptcp_sock); 728 local_bh_enable(); 729 release_sock(sk); 730 731 /* the subflow can already receive packet, avoid racing with 732 * the receive path and process the pending ones 733 */ 734 lock_sock(ssk); 735 subflow->rel_write_seq = 1; 736 subflow->tcp_sock = ssk; 737 subflow->conn = new_mptcp_sock; 738 if (unlikely(!skb_queue_empty(&ssk->sk_receive_queue))) 739 mptcp_subflow_data_available(ssk); 740 release_sock(ssk); 741 } 742 743 return newsk; 744 } 745 746 static void mptcp_destroy(struct sock *sk) 747 { 748 struct mptcp_sock *msk = mptcp_sk(sk); 749 750 if (msk->cached_ext) 751 __skb_ext_put(msk->cached_ext); 752 } 753 754 static int mptcp_setsockopt(struct sock *sk, int level, int optname, 755 char __user *optval, unsigned int optlen) 756 { 757 struct mptcp_sock *msk = mptcp_sk(sk); 758 struct socket *ssock; 759 760 pr_debug("msk=%p", msk); 761 762 /* @@ the meaning of setsockopt() when the socket is connected and 763 * there are multiple subflows is not yet defined. It is up to the 764 * MPTCP-level socket to configure the subflows until the subflow 765 * is in TCP fallback, when TCP socket options are passed through 766 * to the one remaining subflow. 767 */ 768 lock_sock(sk); 769 ssock = __mptcp_tcp_fallback(msk); 770 if (ssock) 771 return tcp_setsockopt(ssock->sk, level, optname, optval, 772 optlen); 773 774 release_sock(sk); 775 776 return -EOPNOTSUPP; 777 } 778 779 static int mptcp_getsockopt(struct sock *sk, int level, int optname, 780 char __user *optval, int __user *option) 781 { 782 struct mptcp_sock *msk = mptcp_sk(sk); 783 struct socket *ssock; 784 785 pr_debug("msk=%p", msk); 786 787 /* @@ the meaning of setsockopt() when the socket is connected and 788 * there are multiple subflows is not yet defined. It is up to the 789 * MPTCP-level socket to configure the subflows until the subflow 790 * is in TCP fallback, when socket options are passed through 791 * to the one remaining subflow. 792 */ 793 lock_sock(sk); 794 ssock = __mptcp_tcp_fallback(msk); 795 if (ssock) 796 return tcp_getsockopt(ssock->sk, level, optname, optval, 797 option); 798 799 release_sock(sk); 800 801 return -EOPNOTSUPP; 802 } 803 804 static int mptcp_get_port(struct sock *sk, unsigned short snum) 805 { 806 struct mptcp_sock *msk = mptcp_sk(sk); 807 struct socket *ssock; 808 809 ssock = __mptcp_nmpc_socket(msk); 810 pr_debug("msk=%p, subflow=%p", msk, ssock); 811 if (WARN_ON_ONCE(!ssock)) 812 return -EINVAL; 813 814 return inet_csk_get_port(ssock->sk, snum); 815 } 816 817 void mptcp_finish_connect(struct sock *ssk) 818 { 819 struct mptcp_subflow_context *subflow; 820 struct mptcp_sock *msk; 821 struct sock *sk; 822 u64 ack_seq; 823 824 subflow = mptcp_subflow_ctx(ssk); 825 826 if (!subflow->mp_capable) 827 return; 828 829 sk = subflow->conn; 830 msk = mptcp_sk(sk); 831 832 pr_debug("msk=%p, token=%u", sk, subflow->token); 833 834 mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 835 ack_seq++; 836 subflow->map_seq = ack_seq; 837 subflow->map_subflow_seq = 1; 838 subflow->rel_write_seq = 1; 839 840 /* the socket is not connected yet, no msk/subflow ops can access/race 841 * accessing the field below 842 */ 843 WRITE_ONCE(msk->remote_key, subflow->remote_key); 844 WRITE_ONCE(msk->local_key, subflow->local_key); 845 WRITE_ONCE(msk->token, subflow->token); 846 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 847 WRITE_ONCE(msk->ack_seq, ack_seq); 848 WRITE_ONCE(msk->can_ack, 1); 849 } 850 851 static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 852 { 853 write_lock_bh(&sk->sk_callback_lock); 854 rcu_assign_pointer(sk->sk_wq, &parent->wq); 855 sk_set_socket(sk, parent); 856 sk->sk_uid = SOCK_INODE(parent)->i_uid; 857 write_unlock_bh(&sk->sk_callback_lock); 858 } 859 860 static bool mptcp_memory_free(const struct sock *sk, int wake) 861 { 862 struct mptcp_sock *msk = mptcp_sk(sk); 863 864 return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 865 } 866 867 static struct proto mptcp_prot = { 868 .name = "MPTCP", 869 .owner = THIS_MODULE, 870 .init = mptcp_init_sock, 871 .close = mptcp_close, 872 .accept = mptcp_accept, 873 .setsockopt = mptcp_setsockopt, 874 .getsockopt = mptcp_getsockopt, 875 .shutdown = tcp_shutdown, 876 .destroy = mptcp_destroy, 877 .sendmsg = mptcp_sendmsg, 878 .recvmsg = mptcp_recvmsg, 879 .hash = inet_hash, 880 .unhash = inet_unhash, 881 .get_port = mptcp_get_port, 882 .stream_memory_free = mptcp_memory_free, 883 .obj_size = sizeof(struct mptcp_sock), 884 .no_autobind = true, 885 }; 886 887 static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 888 { 889 struct mptcp_sock *msk = mptcp_sk(sock->sk); 890 struct socket *ssock; 891 int err; 892 893 lock_sock(sock->sk); 894 ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE); 895 if (IS_ERR(ssock)) { 896 err = PTR_ERR(ssock); 897 goto unlock; 898 } 899 900 err = ssock->ops->bind(ssock, uaddr, addr_len); 901 if (!err) 902 mptcp_copy_inaddrs(sock->sk, ssock->sk); 903 904 unlock: 905 release_sock(sock->sk); 906 return err; 907 } 908 909 static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 910 int addr_len, int flags) 911 { 912 struct mptcp_sock *msk = mptcp_sk(sock->sk); 913 struct socket *ssock; 914 int err; 915 916 lock_sock(sock->sk); 917 ssock = __mptcp_socket_create(msk, TCP_SYN_SENT); 918 if (IS_ERR(ssock)) { 919 err = PTR_ERR(ssock); 920 goto unlock; 921 } 922 923 #ifdef CONFIG_TCP_MD5SIG 924 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 925 * TCP option space. 926 */ 927 if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 928 mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0; 929 #endif 930 931 err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 932 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 933 mptcp_copy_inaddrs(sock->sk, ssock->sk); 934 935 unlock: 936 release_sock(sock->sk); 937 return err; 938 } 939 940 static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr, 941 int peer) 942 { 943 if (sock->sk->sk_prot == &tcp_prot) { 944 /* we are being invoked from __sys_accept4, after 945 * mptcp_accept() has just accepted a non-mp-capable 946 * flow: sk is a tcp_sk, not an mptcp one. 947 * 948 * Hand the socket over to tcp so all further socket ops 949 * bypass mptcp. 950 */ 951 sock->ops = &inet_stream_ops; 952 } 953 954 return inet_getname(sock, uaddr, peer); 955 } 956 957 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 958 static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr, 959 int peer) 960 { 961 if (sock->sk->sk_prot == &tcpv6_prot) { 962 /* we are being invoked from __sys_accept4 after 963 * mptcp_accept() has accepted a non-mp-capable 964 * subflow: sk is a tcp_sk, not mptcp. 965 * 966 * Hand the socket over to tcp so all further 967 * socket ops bypass mptcp. 968 */ 969 sock->ops = &inet6_stream_ops; 970 } 971 972 return inet6_getname(sock, uaddr, peer); 973 } 974 #endif 975 976 static int mptcp_listen(struct socket *sock, int backlog) 977 { 978 struct mptcp_sock *msk = mptcp_sk(sock->sk); 979 struct socket *ssock; 980 int err; 981 982 pr_debug("msk=%p", msk); 983 984 lock_sock(sock->sk); 985 ssock = __mptcp_socket_create(msk, TCP_LISTEN); 986 if (IS_ERR(ssock)) { 987 err = PTR_ERR(ssock); 988 goto unlock; 989 } 990 991 err = ssock->ops->listen(ssock, backlog); 992 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 993 if (!err) 994 mptcp_copy_inaddrs(sock->sk, ssock->sk); 995 996 unlock: 997 release_sock(sock->sk); 998 return err; 999 } 1000 1001 static bool is_tcp_proto(const struct proto *p) 1002 { 1003 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1004 return p == &tcp_prot || p == &tcpv6_prot; 1005 #else 1006 return p == &tcp_prot; 1007 #endif 1008 } 1009 1010 static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 1011 int flags, bool kern) 1012 { 1013 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1014 struct socket *ssock; 1015 int err; 1016 1017 pr_debug("msk=%p", msk); 1018 1019 lock_sock(sock->sk); 1020 if (sock->sk->sk_state != TCP_LISTEN) 1021 goto unlock_fail; 1022 1023 ssock = __mptcp_nmpc_socket(msk); 1024 if (!ssock) 1025 goto unlock_fail; 1026 1027 sock_hold(ssock->sk); 1028 release_sock(sock->sk); 1029 1030 err = ssock->ops->accept(sock, newsock, flags, kern); 1031 if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) { 1032 struct mptcp_sock *msk = mptcp_sk(newsock->sk); 1033 struct mptcp_subflow_context *subflow; 1034 1035 /* set ssk->sk_socket of accept()ed flows to mptcp socket. 1036 * This is needed so NOSPACE flag can be set from tcp stack. 1037 */ 1038 list_for_each_entry(subflow, &msk->conn_list, node) { 1039 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1040 1041 if (!ssk->sk_socket) 1042 mptcp_sock_graft(ssk, newsock); 1043 } 1044 1045 inet_sk_state_store(newsock->sk, TCP_ESTABLISHED); 1046 } 1047 1048 sock_put(ssock->sk); 1049 return err; 1050 1051 unlock_fail: 1052 release_sock(sock->sk); 1053 return -EINVAL; 1054 } 1055 1056 static __poll_t mptcp_poll(struct file *file, struct socket *sock, 1057 struct poll_table_struct *wait) 1058 { 1059 struct sock *sk = sock->sk; 1060 struct mptcp_sock *msk; 1061 struct socket *ssock; 1062 __poll_t mask = 0; 1063 1064 msk = mptcp_sk(sk); 1065 lock_sock(sk); 1066 ssock = __mptcp_nmpc_socket(msk); 1067 if (ssock) { 1068 mask = ssock->ops->poll(file, ssock, wait); 1069 release_sock(sk); 1070 return mask; 1071 } 1072 1073 release_sock(sk); 1074 sock_poll_wait(file, sock, wait); 1075 lock_sock(sk); 1076 ssock = __mptcp_tcp_fallback(msk); 1077 if (unlikely(ssock)) 1078 return ssock->ops->poll(file, ssock, NULL); 1079 1080 if (test_bit(MPTCP_DATA_READY, &msk->flags)) 1081 mask = EPOLLIN | EPOLLRDNORM; 1082 if (sk_stream_is_writeable(sk) && 1083 test_bit(MPTCP_SEND_SPACE, &msk->flags)) 1084 mask |= EPOLLOUT | EPOLLWRNORM; 1085 if (sk->sk_shutdown & RCV_SHUTDOWN) 1086 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 1087 1088 release_sock(sk); 1089 1090 return mask; 1091 } 1092 1093 static int mptcp_shutdown(struct socket *sock, int how) 1094 { 1095 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1096 struct mptcp_subflow_context *subflow; 1097 int ret = 0; 1098 1099 pr_debug("sk=%p, how=%d", msk, how); 1100 1101 lock_sock(sock->sk); 1102 1103 if (how == SHUT_WR || how == SHUT_RDWR) 1104 inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 1105 1106 how++; 1107 1108 if ((how & ~SHUTDOWN_MASK) || !how) { 1109 ret = -EINVAL; 1110 goto out_unlock; 1111 } 1112 1113 if (sock->state == SS_CONNECTING) { 1114 if ((1 << sock->sk->sk_state) & 1115 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 1116 sock->state = SS_DISCONNECTING; 1117 else 1118 sock->state = SS_CONNECTED; 1119 } 1120 1121 mptcp_for_each_subflow(msk, subflow) { 1122 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 1123 1124 mptcp_subflow_shutdown(tcp_sk, how); 1125 } 1126 1127 out_unlock: 1128 release_sock(sock->sk); 1129 1130 return ret; 1131 } 1132 1133 static const struct proto_ops mptcp_stream_ops = { 1134 .family = PF_INET, 1135 .owner = THIS_MODULE, 1136 .release = inet_release, 1137 .bind = mptcp_bind, 1138 .connect = mptcp_stream_connect, 1139 .socketpair = sock_no_socketpair, 1140 .accept = mptcp_stream_accept, 1141 .getname = mptcp_v4_getname, 1142 .poll = mptcp_poll, 1143 .ioctl = inet_ioctl, 1144 .gettstamp = sock_gettstamp, 1145 .listen = mptcp_listen, 1146 .shutdown = mptcp_shutdown, 1147 .setsockopt = sock_common_setsockopt, 1148 .getsockopt = sock_common_getsockopt, 1149 .sendmsg = inet_sendmsg, 1150 .recvmsg = inet_recvmsg, 1151 .mmap = sock_no_mmap, 1152 .sendpage = inet_sendpage, 1153 #ifdef CONFIG_COMPAT 1154 .compat_setsockopt = compat_sock_common_setsockopt, 1155 .compat_getsockopt = compat_sock_common_getsockopt, 1156 #endif 1157 }; 1158 1159 static struct inet_protosw mptcp_protosw = { 1160 .type = SOCK_STREAM, 1161 .protocol = IPPROTO_MPTCP, 1162 .prot = &mptcp_prot, 1163 .ops = &mptcp_stream_ops, 1164 .flags = INET_PROTOSW_ICSK, 1165 }; 1166 1167 void mptcp_proto_init(void) 1168 { 1169 mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 1170 1171 mptcp_subflow_init(); 1172 1173 if (proto_register(&mptcp_prot, 1) != 0) 1174 panic("Failed to register MPTCP proto.\n"); 1175 1176 inet_register_protosw(&mptcp_protosw); 1177 } 1178 1179 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1180 static const struct proto_ops mptcp_v6_stream_ops = { 1181 .family = PF_INET6, 1182 .owner = THIS_MODULE, 1183 .release = inet6_release, 1184 .bind = mptcp_bind, 1185 .connect = mptcp_stream_connect, 1186 .socketpair = sock_no_socketpair, 1187 .accept = mptcp_stream_accept, 1188 .getname = mptcp_v6_getname, 1189 .poll = mptcp_poll, 1190 .ioctl = inet6_ioctl, 1191 .gettstamp = sock_gettstamp, 1192 .listen = mptcp_listen, 1193 .shutdown = mptcp_shutdown, 1194 .setsockopt = sock_common_setsockopt, 1195 .getsockopt = sock_common_getsockopt, 1196 .sendmsg = inet6_sendmsg, 1197 .recvmsg = inet6_recvmsg, 1198 .mmap = sock_no_mmap, 1199 .sendpage = inet_sendpage, 1200 #ifdef CONFIG_COMPAT 1201 .compat_setsockopt = compat_sock_common_setsockopt, 1202 .compat_getsockopt = compat_sock_common_getsockopt, 1203 #endif 1204 }; 1205 1206 static struct proto mptcp_v6_prot; 1207 1208 static void mptcp_v6_destroy(struct sock *sk) 1209 { 1210 mptcp_destroy(sk); 1211 inet6_destroy_sock(sk); 1212 } 1213 1214 static struct inet_protosw mptcp_v6_protosw = { 1215 .type = SOCK_STREAM, 1216 .protocol = IPPROTO_MPTCP, 1217 .prot = &mptcp_v6_prot, 1218 .ops = &mptcp_v6_stream_ops, 1219 .flags = INET_PROTOSW_ICSK, 1220 }; 1221 1222 int mptcp_proto_v6_init(void) 1223 { 1224 int err; 1225 1226 mptcp_v6_prot = mptcp_prot; 1227 strcpy(mptcp_v6_prot.name, "MPTCPv6"); 1228 mptcp_v6_prot.slab = NULL; 1229 mptcp_v6_prot.destroy = mptcp_v6_destroy; 1230 mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 1231 1232 err = proto_register(&mptcp_v6_prot, 1); 1233 if (err) 1234 return err; 1235 1236 err = inet6_register_protosw(&mptcp_v6_protosw); 1237 if (err) 1238 proto_unregister(&mptcp_v6_prot); 1239 1240 return err; 1241 } 1242 #endif 1243