1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <linux/sched/signal.h> 13 #include <linux/atomic.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20 #include <net/transp_v6.h> 21 #endif 22 #include <net/mptcp.h> 23 #include "protocol.h" 24 25 #define MPTCP_SAME_STATE TCP_MAX_STATES 26 27 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 28 struct mptcp6_sock { 29 struct mptcp_sock msk; 30 struct ipv6_pinfo np; 31 }; 32 #endif 33 34 struct mptcp_skb_cb { 35 u32 offset; 36 }; 37 38 #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 39 40 /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 41 * completed yet or has failed, return the subflow socket. 42 * Otherwise return NULL. 43 */ 44 static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 45 { 46 if (!msk->subflow || READ_ONCE(msk->can_ack)) 47 return NULL; 48 49 return msk->subflow; 50 } 51 52 static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk) 53 { 54 return msk->first && !sk_is_mptcp(msk->first); 55 } 56 57 static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk) 58 { 59 sock_owned_by_me((const struct sock *)msk); 60 61 if (likely(!__mptcp_needs_tcp_fallback(msk))) 62 return NULL; 63 64 if (msk->subflow) { 65 release_sock((struct sock *)msk); 66 return msk->subflow; 67 } 68 69 return NULL; 70 } 71 72 static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk) 73 { 74 return !msk->first; 75 } 76 77 static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state) 78 { 79 struct mptcp_subflow_context *subflow; 80 struct sock *sk = (struct sock *)msk; 81 struct socket *ssock; 82 int err; 83 84 ssock = __mptcp_nmpc_socket(msk); 85 if (ssock) 86 goto set_state; 87 88 if (!__mptcp_can_create_subflow(msk)) 89 return ERR_PTR(-EINVAL); 90 91 err = mptcp_subflow_create_socket(sk, &ssock); 92 if (err) 93 return ERR_PTR(err); 94 95 msk->first = ssock->sk; 96 msk->subflow = ssock; 97 subflow = mptcp_subflow_ctx(ssock->sk); 98 list_add(&subflow->node, &msk->conn_list); 99 subflow->request_mptcp = 1; 100 101 set_state: 102 if (state != MPTCP_SAME_STATE) 103 inet_sk_state_store(sk, state); 104 return ssock; 105 } 106 107 static struct sock *mptcp_subflow_get(const struct mptcp_sock *msk) 108 { 109 struct mptcp_subflow_context *subflow; 110 111 sock_owned_by_me((const struct sock *)msk); 112 113 mptcp_for_each_subflow(msk, subflow) { 114 return mptcp_subflow_tcp_sock(subflow); 115 } 116 117 return NULL; 118 } 119 120 static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 121 struct sk_buff *skb, 122 unsigned int offset, size_t copy_len) 123 { 124 struct sock *sk = (struct sock *)msk; 125 126 __skb_unlink(skb, &ssk->sk_receive_queue); 127 skb_set_owner_r(skb, sk); 128 __skb_queue_tail(&sk->sk_receive_queue, skb); 129 130 msk->ack_seq += copy_len; 131 MPTCP_SKB_CB(skb)->offset = offset; 132 } 133 134 static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 135 struct sock *ssk, 136 unsigned int *bytes) 137 { 138 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 139 struct sock *sk = (struct sock *)msk; 140 unsigned int moved = 0; 141 bool more_data_avail; 142 struct tcp_sock *tp; 143 bool done = false; 144 int rcvbuf; 145 146 rcvbuf = max(ssk->sk_rcvbuf, sk->sk_rcvbuf); 147 if (rcvbuf > sk->sk_rcvbuf) 148 sk->sk_rcvbuf = rcvbuf; 149 150 tp = tcp_sk(ssk); 151 do { 152 u32 map_remaining, offset; 153 u32 seq = tp->copied_seq; 154 struct sk_buff *skb; 155 bool fin; 156 157 /* try to move as much data as available */ 158 map_remaining = subflow->map_data_len - 159 mptcp_subflow_get_map_offset(subflow); 160 161 skb = skb_peek(&ssk->sk_receive_queue); 162 if (!skb) 163 break; 164 165 offset = seq - TCP_SKB_CB(skb)->seq; 166 fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 167 if (fin) { 168 done = true; 169 seq++; 170 } 171 172 if (offset < skb->len) { 173 size_t len = skb->len - offset; 174 175 if (tp->urg_data) 176 done = true; 177 178 __mptcp_move_skb(msk, ssk, skb, offset, len); 179 seq += len; 180 moved += len; 181 182 if (WARN_ON_ONCE(map_remaining < len)) 183 break; 184 } else { 185 WARN_ON_ONCE(!fin); 186 sk_eat_skb(ssk, skb); 187 done = true; 188 } 189 190 WRITE_ONCE(tp->copied_seq, seq); 191 more_data_avail = mptcp_subflow_data_available(ssk); 192 193 if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { 194 done = true; 195 break; 196 } 197 } while (more_data_avail); 198 199 *bytes = moved; 200 201 return done; 202 } 203 204 /* In most cases we will be able to lock the mptcp socket. If its already 205 * owned, we need to defer to the work queue to avoid ABBA deadlock. 206 */ 207 static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 208 { 209 struct sock *sk = (struct sock *)msk; 210 unsigned int moved = 0; 211 212 if (READ_ONCE(sk->sk_lock.owned)) 213 return false; 214 215 if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) 216 return false; 217 218 /* must re-check after taking the lock */ 219 if (!READ_ONCE(sk->sk_lock.owned)) 220 __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 221 222 spin_unlock_bh(&sk->sk_lock.slock); 223 224 return moved > 0; 225 } 226 227 void mptcp_data_ready(struct sock *sk, struct sock *ssk) 228 { 229 struct mptcp_sock *msk = mptcp_sk(sk); 230 231 set_bit(MPTCP_DATA_READY, &msk->flags); 232 233 if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && 234 move_skbs_to_msk(msk, ssk)) 235 goto wake; 236 237 /* don't schedule if mptcp sk is (still) over limit */ 238 if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) 239 goto wake; 240 241 /* mptcp socket is owned, release_cb should retry */ 242 if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, 243 &sk->sk_tsq_flags)) { 244 sock_hold(sk); 245 246 /* need to try again, its possible release_cb() has already 247 * been called after the test_and_set_bit() above. 248 */ 249 move_skbs_to_msk(msk, ssk); 250 } 251 wake: 252 sk->sk_data_ready(sk); 253 } 254 255 static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 256 { 257 if (!msk->cached_ext) 258 msk->cached_ext = __skb_ext_alloc(); 259 260 return !!msk->cached_ext; 261 } 262 263 static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 264 { 265 struct mptcp_subflow_context *subflow; 266 struct sock *sk = (struct sock *)msk; 267 268 sock_owned_by_me(sk); 269 270 mptcp_for_each_subflow(msk, subflow) { 271 if (subflow->data_avail) 272 return mptcp_subflow_tcp_sock(subflow); 273 } 274 275 return NULL; 276 } 277 278 static inline bool mptcp_skb_can_collapse_to(const struct mptcp_sock *msk, 279 const struct sk_buff *skb, 280 const struct mptcp_ext *mpext) 281 { 282 if (!tcp_skb_can_collapse_to(skb)) 283 return false; 284 285 /* can collapse only if MPTCP level sequence is in order */ 286 return mpext && mpext->data_seq + mpext->data_len == msk->write_seq; 287 } 288 289 static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 290 struct msghdr *msg, long *timeo, int *pmss_now, 291 int *ps_goal) 292 { 293 int mss_now, avail_size, size_goal, ret; 294 struct mptcp_sock *msk = mptcp_sk(sk); 295 struct mptcp_ext *mpext = NULL; 296 struct sk_buff *skb, *tail; 297 bool can_collapse = false; 298 struct page_frag *pfrag; 299 size_t psize; 300 301 /* use the mptcp page cache so that we can easily move the data 302 * from one substream to another, but do per subflow memory accounting 303 */ 304 pfrag = sk_page_frag(sk); 305 while (!sk_page_frag_refill(ssk, pfrag) || 306 !mptcp_ext_cache_refill(msk)) { 307 ret = sk_stream_wait_memory(ssk, timeo); 308 if (ret) 309 return ret; 310 if (unlikely(__mptcp_needs_tcp_fallback(msk))) 311 return 0; 312 } 313 314 /* compute copy limit */ 315 mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 316 *pmss_now = mss_now; 317 *ps_goal = size_goal; 318 avail_size = size_goal; 319 skb = tcp_write_queue_tail(ssk); 320 if (skb) { 321 mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 322 323 /* Limit the write to the size available in the 324 * current skb, if any, so that we create at most a new skb. 325 * Explicitly tells TCP internals to avoid collapsing on later 326 * queue management operation, to avoid breaking the ext <-> 327 * SSN association set here 328 */ 329 can_collapse = (size_goal - skb->len > 0) && 330 mptcp_skb_can_collapse_to(msk, skb, mpext); 331 if (!can_collapse) 332 TCP_SKB_CB(skb)->eor = 1; 333 else 334 avail_size = size_goal - skb->len; 335 } 336 psize = min_t(size_t, pfrag->size - pfrag->offset, avail_size); 337 338 /* Copy to page */ 339 pr_debug("left=%zu", msg_data_left(msg)); 340 psize = copy_page_from_iter(pfrag->page, pfrag->offset, 341 min_t(size_t, msg_data_left(msg), psize), 342 &msg->msg_iter); 343 pr_debug("left=%zu", msg_data_left(msg)); 344 if (!psize) 345 return -EINVAL; 346 347 /* tell the TCP stack to delay the push so that we can safely 348 * access the skb after the sendpages call 349 */ 350 ret = do_tcp_sendpages(ssk, pfrag->page, pfrag->offset, psize, 351 msg->msg_flags | MSG_SENDPAGE_NOTLAST); 352 if (ret <= 0) 353 return ret; 354 if (unlikely(ret < psize)) 355 iov_iter_revert(&msg->msg_iter, psize - ret); 356 357 /* if the tail skb extension is still the cached one, collapsing 358 * really happened. Note: we can't check for 'same skb' as the sk_buff 359 * hdr on tail can be transmitted, freed and re-allocated by the 360 * do_tcp_sendpages() call 361 */ 362 tail = tcp_write_queue_tail(ssk); 363 if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 364 WARN_ON_ONCE(!can_collapse); 365 mpext->data_len += ret; 366 goto out; 367 } 368 369 skb = tcp_write_queue_tail(ssk); 370 mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 371 msk->cached_ext = NULL; 372 373 memset(mpext, 0, sizeof(*mpext)); 374 mpext->data_seq = msk->write_seq; 375 mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 376 mpext->data_len = ret; 377 mpext->use_map = 1; 378 mpext->dsn64 = 1; 379 380 pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 381 mpext->data_seq, mpext->subflow_seq, mpext->data_len, 382 mpext->dsn64); 383 384 out: 385 pfrag->offset += ret; 386 msk->write_seq += ret; 387 mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 388 389 return ret; 390 } 391 392 static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk) 393 { 394 struct socket *sock; 395 396 if (likely(sk_stream_is_writeable(ssk))) 397 return; 398 399 sock = READ_ONCE(ssk->sk_socket); 400 401 if (sock) { 402 clear_bit(MPTCP_SEND_SPACE, &msk->flags); 403 smp_mb__after_atomic(); 404 /* set NOSPACE only after clearing SEND_SPACE flag */ 405 set_bit(SOCK_NOSPACE, &sock->flags); 406 } 407 } 408 409 static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 410 { 411 int mss_now = 0, size_goal = 0, ret = 0; 412 struct mptcp_sock *msk = mptcp_sk(sk); 413 struct socket *ssock; 414 size_t copied = 0; 415 struct sock *ssk; 416 long timeo; 417 418 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 419 return -EOPNOTSUPP; 420 421 lock_sock(sk); 422 ssock = __mptcp_tcp_fallback(msk); 423 if (unlikely(ssock)) { 424 fallback: 425 pr_debug("fallback passthrough"); 426 ret = sock_sendmsg(ssock, msg); 427 return ret >= 0 ? ret + copied : (copied ? copied : ret); 428 } 429 430 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 431 432 ssk = mptcp_subflow_get(msk); 433 if (!ssk) { 434 release_sock(sk); 435 return -ENOTCONN; 436 } 437 438 pr_debug("conn_list->subflow=%p", ssk); 439 440 lock_sock(ssk); 441 while (msg_data_left(msg)) { 442 ret = mptcp_sendmsg_frag(sk, ssk, msg, &timeo, &mss_now, 443 &size_goal); 444 if (ret < 0) 445 break; 446 if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) { 447 release_sock(ssk); 448 ssock = __mptcp_tcp_fallback(msk); 449 goto fallback; 450 } 451 452 copied += ret; 453 } 454 455 if (copied) { 456 ret = copied; 457 tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 458 size_goal); 459 } 460 461 ssk_check_wmem(msk, ssk); 462 release_sock(ssk); 463 release_sock(sk); 464 return ret; 465 } 466 467 static void mptcp_wait_data(struct sock *sk, long *timeo) 468 { 469 DEFINE_WAIT_FUNC(wait, woken_wake_function); 470 struct mptcp_sock *msk = mptcp_sk(sk); 471 472 add_wait_queue(sk_sleep(sk), &wait); 473 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 474 475 sk_wait_event(sk, timeo, 476 test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 477 478 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 479 remove_wait_queue(sk_sleep(sk), &wait); 480 } 481 482 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 483 struct msghdr *msg, 484 size_t len) 485 { 486 struct sock *sk = (struct sock *)msk; 487 struct sk_buff *skb; 488 int copied = 0; 489 490 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 491 u32 offset = MPTCP_SKB_CB(skb)->offset; 492 u32 data_len = skb->len - offset; 493 u32 count = min_t(size_t, len - copied, data_len); 494 int err; 495 496 err = skb_copy_datagram_msg(skb, offset, msg, count); 497 if (unlikely(err < 0)) { 498 if (!copied) 499 return err; 500 break; 501 } 502 503 copied += count; 504 505 if (count < data_len) { 506 MPTCP_SKB_CB(skb)->offset += count; 507 break; 508 } 509 510 __skb_unlink(skb, &sk->sk_receive_queue); 511 __kfree_skb(skb); 512 513 if (copied >= len) 514 break; 515 } 516 517 return copied; 518 } 519 520 static bool __mptcp_move_skbs(struct mptcp_sock *msk) 521 { 522 unsigned int moved = 0; 523 bool done; 524 525 do { 526 struct sock *ssk = mptcp_subflow_recv_lookup(msk); 527 528 if (!ssk) 529 break; 530 531 lock_sock(ssk); 532 done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 533 release_sock(ssk); 534 } while (!done); 535 536 return moved > 0; 537 } 538 539 static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 540 int nonblock, int flags, int *addr_len) 541 { 542 struct mptcp_sock *msk = mptcp_sk(sk); 543 struct socket *ssock; 544 int copied = 0; 545 int target; 546 long timeo; 547 548 if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 549 return -EOPNOTSUPP; 550 551 lock_sock(sk); 552 ssock = __mptcp_tcp_fallback(msk); 553 if (unlikely(ssock)) { 554 fallback: 555 pr_debug("fallback-read subflow=%p", 556 mptcp_subflow_ctx(ssock->sk)); 557 copied = sock_recvmsg(ssock, msg, flags); 558 return copied; 559 } 560 561 timeo = sock_rcvtimeo(sk, nonblock); 562 563 len = min_t(size_t, len, INT_MAX); 564 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 565 566 while (len > (size_t)copied) { 567 int bytes_read; 568 569 bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 570 if (unlikely(bytes_read < 0)) { 571 if (!copied) 572 copied = bytes_read; 573 goto out_err; 574 } 575 576 copied += bytes_read; 577 578 if (skb_queue_empty(&sk->sk_receive_queue) && 579 __mptcp_move_skbs(msk)) 580 continue; 581 582 /* only the master socket status is relevant here. The exit 583 * conditions mirror closely tcp_recvmsg() 584 */ 585 if (copied >= target) 586 break; 587 588 if (copied) { 589 if (sk->sk_err || 590 sk->sk_state == TCP_CLOSE || 591 (sk->sk_shutdown & RCV_SHUTDOWN) || 592 !timeo || 593 signal_pending(current)) 594 break; 595 } else { 596 if (sk->sk_err) { 597 copied = sock_error(sk); 598 break; 599 } 600 601 if (sk->sk_shutdown & RCV_SHUTDOWN) 602 break; 603 604 if (sk->sk_state == TCP_CLOSE) { 605 copied = -ENOTCONN; 606 break; 607 } 608 609 if (!timeo) { 610 copied = -EAGAIN; 611 break; 612 } 613 614 if (signal_pending(current)) { 615 copied = sock_intr_errno(timeo); 616 break; 617 } 618 } 619 620 pr_debug("block timeout %ld", timeo); 621 mptcp_wait_data(sk, &timeo); 622 if (unlikely(__mptcp_tcp_fallback(msk))) 623 goto fallback; 624 } 625 626 if (skb_queue_empty(&sk->sk_receive_queue)) { 627 /* entire backlog drained, clear DATA_READY. */ 628 clear_bit(MPTCP_DATA_READY, &msk->flags); 629 630 /* .. race-breaker: ssk might have gotten new data 631 * after last __mptcp_move_skbs() returned false. 632 */ 633 if (unlikely(__mptcp_move_skbs(msk))) 634 set_bit(MPTCP_DATA_READY, &msk->flags); 635 } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 636 /* data to read but mptcp_wait_data() cleared DATA_READY */ 637 set_bit(MPTCP_DATA_READY, &msk->flags); 638 } 639 out_err: 640 release_sock(sk); 641 return copied; 642 } 643 644 /* subflow sockets can be either outgoing (connect) or incoming 645 * (accept). 646 * 647 * Outgoing subflows use in-kernel sockets. 648 * Incoming subflows do not have their own 'struct socket' allocated, 649 * so we need to use tcp_close() after detaching them from the mptcp 650 * parent socket. 651 */ 652 static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 653 struct mptcp_subflow_context *subflow, 654 long timeout) 655 { 656 struct socket *sock = READ_ONCE(ssk->sk_socket); 657 658 list_del(&subflow->node); 659 660 if (sock && sock != sk->sk_socket) { 661 /* outgoing subflow */ 662 sock_release(sock); 663 } else { 664 /* incoming subflow */ 665 tcp_close(ssk, timeout); 666 } 667 } 668 669 static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 670 { 671 return 0; 672 } 673 674 static void mptcp_worker(struct work_struct *work) 675 { 676 struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 677 struct sock *sk = &msk->sk.icsk_inet.sk; 678 679 lock_sock(sk); 680 __mptcp_move_skbs(msk); 681 release_sock(sk); 682 sock_put(sk); 683 } 684 685 static int __mptcp_init_sock(struct sock *sk) 686 { 687 struct mptcp_sock *msk = mptcp_sk(sk); 688 689 INIT_LIST_HEAD(&msk->conn_list); 690 __set_bit(MPTCP_SEND_SPACE, &msk->flags); 691 INIT_WORK(&msk->work, mptcp_worker); 692 693 msk->first = NULL; 694 inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 695 696 return 0; 697 } 698 699 static int mptcp_init_sock(struct sock *sk) 700 { 701 if (!mptcp_is_enabled(sock_net(sk))) 702 return -ENOPROTOOPT; 703 704 return __mptcp_init_sock(sk); 705 } 706 707 static void mptcp_cancel_work(struct sock *sk) 708 { 709 struct mptcp_sock *msk = mptcp_sk(sk); 710 711 if (cancel_work_sync(&msk->work)) 712 sock_put(sk); 713 } 714 715 static void mptcp_subflow_shutdown(struct sock *ssk, int how) 716 { 717 lock_sock(ssk); 718 719 switch (ssk->sk_state) { 720 case TCP_LISTEN: 721 if (!(how & RCV_SHUTDOWN)) 722 break; 723 /* fall through */ 724 case TCP_SYN_SENT: 725 tcp_disconnect(ssk, O_NONBLOCK); 726 break; 727 default: 728 ssk->sk_shutdown |= how; 729 tcp_shutdown(ssk, how); 730 break; 731 } 732 733 /* Wake up anyone sleeping in poll. */ 734 ssk->sk_state_change(ssk); 735 release_sock(ssk); 736 } 737 738 /* Called with msk lock held, releases such lock before returning */ 739 static void mptcp_close(struct sock *sk, long timeout) 740 { 741 struct mptcp_subflow_context *subflow, *tmp; 742 struct mptcp_sock *msk = mptcp_sk(sk); 743 LIST_HEAD(conn_list); 744 745 lock_sock(sk); 746 747 mptcp_token_destroy(msk->token); 748 inet_sk_state_store(sk, TCP_CLOSE); 749 750 list_splice_init(&msk->conn_list, &conn_list); 751 752 release_sock(sk); 753 754 list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 755 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 756 757 __mptcp_close_ssk(sk, ssk, subflow, timeout); 758 } 759 760 mptcp_cancel_work(sk); 761 762 __skb_queue_purge(&sk->sk_receive_queue); 763 764 sk_common_release(sk); 765 } 766 767 static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 768 { 769 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 770 const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 771 struct ipv6_pinfo *msk6 = inet6_sk(msk); 772 773 msk->sk_v6_daddr = ssk->sk_v6_daddr; 774 msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 775 776 if (msk6 && ssk6) { 777 msk6->saddr = ssk6->saddr; 778 msk6->flow_label = ssk6->flow_label; 779 } 780 #endif 781 782 inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 783 inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 784 inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 785 inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 786 inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 787 inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 788 } 789 790 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 791 static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 792 { 793 unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 794 795 return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 796 } 797 #endif 798 799 static struct sock *mptcp_sk_clone_lock(const struct sock *sk) 800 { 801 struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 802 803 if (!nsk) 804 return NULL; 805 806 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 807 if (nsk->sk_family == AF_INET6) 808 inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 809 #endif 810 811 return nsk; 812 } 813 814 static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 815 bool kern) 816 { 817 struct mptcp_sock *msk = mptcp_sk(sk); 818 struct socket *listener; 819 struct sock *newsk; 820 821 listener = __mptcp_nmpc_socket(msk); 822 if (WARN_ON_ONCE(!listener)) { 823 *err = -EINVAL; 824 return NULL; 825 } 826 827 pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 828 newsk = inet_csk_accept(listener->sk, flags, err, kern); 829 if (!newsk) 830 return NULL; 831 832 pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 833 834 if (sk_is_mptcp(newsk)) { 835 struct mptcp_subflow_context *subflow; 836 struct sock *new_mptcp_sock; 837 struct sock *ssk = newsk; 838 u64 ack_seq; 839 840 subflow = mptcp_subflow_ctx(newsk); 841 lock_sock(sk); 842 843 local_bh_disable(); 844 new_mptcp_sock = mptcp_sk_clone_lock(sk); 845 if (!new_mptcp_sock) { 846 *err = -ENOBUFS; 847 local_bh_enable(); 848 release_sock(sk); 849 mptcp_subflow_shutdown(newsk, SHUT_RDWR + 1); 850 tcp_close(newsk, 0); 851 return NULL; 852 } 853 854 __mptcp_init_sock(new_mptcp_sock); 855 856 msk = mptcp_sk(new_mptcp_sock); 857 msk->local_key = subflow->local_key; 858 msk->token = subflow->token; 859 msk->subflow = NULL; 860 msk->first = newsk; 861 862 mptcp_token_update_accept(newsk, new_mptcp_sock); 863 864 msk->write_seq = subflow->idsn + 1; 865 if (subflow->can_ack) { 866 msk->can_ack = true; 867 msk->remote_key = subflow->remote_key; 868 mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 869 ack_seq++; 870 msk->ack_seq = ack_seq; 871 } 872 newsk = new_mptcp_sock; 873 mptcp_copy_inaddrs(newsk, ssk); 874 list_add(&subflow->node, &msk->conn_list); 875 876 /* will be fully established at mptcp_stream_accept() 877 * completion. 878 */ 879 inet_sk_state_store(new_mptcp_sock, TCP_SYN_RECV); 880 bh_unlock_sock(new_mptcp_sock); 881 local_bh_enable(); 882 release_sock(sk); 883 884 /* the subflow can already receive packet, avoid racing with 885 * the receive path and process the pending ones 886 */ 887 lock_sock(ssk); 888 subflow->rel_write_seq = 1; 889 subflow->tcp_sock = ssk; 890 subflow->conn = new_mptcp_sock; 891 if (unlikely(!skb_queue_empty(&ssk->sk_receive_queue))) 892 mptcp_subflow_data_available(ssk); 893 release_sock(ssk); 894 } 895 896 return newsk; 897 } 898 899 static void mptcp_destroy(struct sock *sk) 900 { 901 struct mptcp_sock *msk = mptcp_sk(sk); 902 903 if (msk->cached_ext) 904 __skb_ext_put(msk->cached_ext); 905 } 906 907 static int mptcp_setsockopt(struct sock *sk, int level, int optname, 908 char __user *optval, unsigned int optlen) 909 { 910 struct mptcp_sock *msk = mptcp_sk(sk); 911 struct socket *ssock; 912 913 pr_debug("msk=%p", msk); 914 915 /* @@ the meaning of setsockopt() when the socket is connected and 916 * there are multiple subflows is not yet defined. It is up to the 917 * MPTCP-level socket to configure the subflows until the subflow 918 * is in TCP fallback, when TCP socket options are passed through 919 * to the one remaining subflow. 920 */ 921 lock_sock(sk); 922 ssock = __mptcp_tcp_fallback(msk); 923 if (ssock) 924 return tcp_setsockopt(ssock->sk, level, optname, optval, 925 optlen); 926 927 release_sock(sk); 928 929 return -EOPNOTSUPP; 930 } 931 932 static int mptcp_getsockopt(struct sock *sk, int level, int optname, 933 char __user *optval, int __user *option) 934 { 935 struct mptcp_sock *msk = mptcp_sk(sk); 936 struct socket *ssock; 937 938 pr_debug("msk=%p", msk); 939 940 /* @@ the meaning of setsockopt() when the socket is connected and 941 * there are multiple subflows is not yet defined. It is up to the 942 * MPTCP-level socket to configure the subflows until the subflow 943 * is in TCP fallback, when socket options are passed through 944 * to the one remaining subflow. 945 */ 946 lock_sock(sk); 947 ssock = __mptcp_tcp_fallback(msk); 948 if (ssock) 949 return tcp_getsockopt(ssock->sk, level, optname, optval, 950 option); 951 952 release_sock(sk); 953 954 return -EOPNOTSUPP; 955 } 956 957 #define MPTCP_DEFERRED_ALL TCPF_DELACK_TIMER_DEFERRED 958 959 /* this is very alike tcp_release_cb() but we must handle differently a 960 * different set of events 961 */ 962 static void mptcp_release_cb(struct sock *sk) 963 { 964 unsigned long flags, nflags; 965 966 do { 967 flags = sk->sk_tsq_flags; 968 if (!(flags & MPTCP_DEFERRED_ALL)) 969 return; 970 nflags = flags & ~MPTCP_DEFERRED_ALL; 971 } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 972 973 if (flags & TCPF_DELACK_TIMER_DEFERRED) { 974 struct mptcp_sock *msk = mptcp_sk(sk); 975 struct sock *ssk; 976 977 ssk = mptcp_subflow_recv_lookup(msk); 978 if (!ssk || !schedule_work(&msk->work)) 979 __sock_put(sk); 980 } 981 } 982 983 static int mptcp_get_port(struct sock *sk, unsigned short snum) 984 { 985 struct mptcp_sock *msk = mptcp_sk(sk); 986 struct socket *ssock; 987 988 ssock = __mptcp_nmpc_socket(msk); 989 pr_debug("msk=%p, subflow=%p", msk, ssock); 990 if (WARN_ON_ONCE(!ssock)) 991 return -EINVAL; 992 993 return inet_csk_get_port(ssock->sk, snum); 994 } 995 996 void mptcp_finish_connect(struct sock *ssk) 997 { 998 struct mptcp_subflow_context *subflow; 999 struct mptcp_sock *msk; 1000 struct sock *sk; 1001 u64 ack_seq; 1002 1003 subflow = mptcp_subflow_ctx(ssk); 1004 1005 if (!subflow->mp_capable) 1006 return; 1007 1008 sk = subflow->conn; 1009 msk = mptcp_sk(sk); 1010 1011 pr_debug("msk=%p, token=%u", sk, subflow->token); 1012 1013 mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 1014 ack_seq++; 1015 subflow->map_seq = ack_seq; 1016 subflow->map_subflow_seq = 1; 1017 subflow->rel_write_seq = 1; 1018 1019 /* the socket is not connected yet, no msk/subflow ops can access/race 1020 * accessing the field below 1021 */ 1022 WRITE_ONCE(msk->remote_key, subflow->remote_key); 1023 WRITE_ONCE(msk->local_key, subflow->local_key); 1024 WRITE_ONCE(msk->token, subflow->token); 1025 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 1026 WRITE_ONCE(msk->ack_seq, ack_seq); 1027 WRITE_ONCE(msk->can_ack, 1); 1028 } 1029 1030 static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 1031 { 1032 write_lock_bh(&sk->sk_callback_lock); 1033 rcu_assign_pointer(sk->sk_wq, &parent->wq); 1034 sk_set_socket(sk, parent); 1035 sk->sk_uid = SOCK_INODE(parent)->i_uid; 1036 write_unlock_bh(&sk->sk_callback_lock); 1037 } 1038 1039 static bool mptcp_memory_free(const struct sock *sk, int wake) 1040 { 1041 struct mptcp_sock *msk = mptcp_sk(sk); 1042 1043 return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 1044 } 1045 1046 static struct proto mptcp_prot = { 1047 .name = "MPTCP", 1048 .owner = THIS_MODULE, 1049 .init = mptcp_init_sock, 1050 .close = mptcp_close, 1051 .accept = mptcp_accept, 1052 .setsockopt = mptcp_setsockopt, 1053 .getsockopt = mptcp_getsockopt, 1054 .shutdown = tcp_shutdown, 1055 .destroy = mptcp_destroy, 1056 .sendmsg = mptcp_sendmsg, 1057 .recvmsg = mptcp_recvmsg, 1058 .release_cb = mptcp_release_cb, 1059 .hash = inet_hash, 1060 .unhash = inet_unhash, 1061 .get_port = mptcp_get_port, 1062 .stream_memory_free = mptcp_memory_free, 1063 .obj_size = sizeof(struct mptcp_sock), 1064 .no_autobind = true, 1065 }; 1066 1067 static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 1068 { 1069 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1070 struct socket *ssock; 1071 int err; 1072 1073 lock_sock(sock->sk); 1074 ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE); 1075 if (IS_ERR(ssock)) { 1076 err = PTR_ERR(ssock); 1077 goto unlock; 1078 } 1079 1080 err = ssock->ops->bind(ssock, uaddr, addr_len); 1081 if (!err) 1082 mptcp_copy_inaddrs(sock->sk, ssock->sk); 1083 1084 unlock: 1085 release_sock(sock->sk); 1086 return err; 1087 } 1088 1089 static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 1090 int addr_len, int flags) 1091 { 1092 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1093 struct socket *ssock; 1094 int err; 1095 1096 lock_sock(sock->sk); 1097 ssock = __mptcp_socket_create(msk, TCP_SYN_SENT); 1098 if (IS_ERR(ssock)) { 1099 err = PTR_ERR(ssock); 1100 goto unlock; 1101 } 1102 1103 #ifdef CONFIG_TCP_MD5SIG 1104 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 1105 * TCP option space. 1106 */ 1107 if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 1108 mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0; 1109 #endif 1110 1111 err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 1112 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1113 mptcp_copy_inaddrs(sock->sk, ssock->sk); 1114 1115 unlock: 1116 release_sock(sock->sk); 1117 return err; 1118 } 1119 1120 static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr, 1121 int peer) 1122 { 1123 if (sock->sk->sk_prot == &tcp_prot) { 1124 /* we are being invoked from __sys_accept4, after 1125 * mptcp_accept() has just accepted a non-mp-capable 1126 * flow: sk is a tcp_sk, not an mptcp one. 1127 * 1128 * Hand the socket over to tcp so all further socket ops 1129 * bypass mptcp. 1130 */ 1131 sock->ops = &inet_stream_ops; 1132 } 1133 1134 return inet_getname(sock, uaddr, peer); 1135 } 1136 1137 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1138 static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr, 1139 int peer) 1140 { 1141 if (sock->sk->sk_prot == &tcpv6_prot) { 1142 /* we are being invoked from __sys_accept4 after 1143 * mptcp_accept() has accepted a non-mp-capable 1144 * subflow: sk is a tcp_sk, not mptcp. 1145 * 1146 * Hand the socket over to tcp so all further 1147 * socket ops bypass mptcp. 1148 */ 1149 sock->ops = &inet6_stream_ops; 1150 } 1151 1152 return inet6_getname(sock, uaddr, peer); 1153 } 1154 #endif 1155 1156 static int mptcp_listen(struct socket *sock, int backlog) 1157 { 1158 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1159 struct socket *ssock; 1160 int err; 1161 1162 pr_debug("msk=%p", msk); 1163 1164 lock_sock(sock->sk); 1165 ssock = __mptcp_socket_create(msk, TCP_LISTEN); 1166 if (IS_ERR(ssock)) { 1167 err = PTR_ERR(ssock); 1168 goto unlock; 1169 } 1170 1171 err = ssock->ops->listen(ssock, backlog); 1172 inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1173 if (!err) 1174 mptcp_copy_inaddrs(sock->sk, ssock->sk); 1175 1176 unlock: 1177 release_sock(sock->sk); 1178 return err; 1179 } 1180 1181 static bool is_tcp_proto(const struct proto *p) 1182 { 1183 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1184 return p == &tcp_prot || p == &tcpv6_prot; 1185 #else 1186 return p == &tcp_prot; 1187 #endif 1188 } 1189 1190 static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 1191 int flags, bool kern) 1192 { 1193 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1194 struct socket *ssock; 1195 int err; 1196 1197 pr_debug("msk=%p", msk); 1198 1199 lock_sock(sock->sk); 1200 if (sock->sk->sk_state != TCP_LISTEN) 1201 goto unlock_fail; 1202 1203 ssock = __mptcp_nmpc_socket(msk); 1204 if (!ssock) 1205 goto unlock_fail; 1206 1207 sock_hold(ssock->sk); 1208 release_sock(sock->sk); 1209 1210 err = ssock->ops->accept(sock, newsock, flags, kern); 1211 if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) { 1212 struct mptcp_sock *msk = mptcp_sk(newsock->sk); 1213 struct mptcp_subflow_context *subflow; 1214 1215 /* set ssk->sk_socket of accept()ed flows to mptcp socket. 1216 * This is needed so NOSPACE flag can be set from tcp stack. 1217 */ 1218 list_for_each_entry(subflow, &msk->conn_list, node) { 1219 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1220 1221 if (!ssk->sk_socket) 1222 mptcp_sock_graft(ssk, newsock); 1223 } 1224 1225 inet_sk_state_store(newsock->sk, TCP_ESTABLISHED); 1226 } 1227 1228 sock_put(ssock->sk); 1229 return err; 1230 1231 unlock_fail: 1232 release_sock(sock->sk); 1233 return -EINVAL; 1234 } 1235 1236 static __poll_t mptcp_poll(struct file *file, struct socket *sock, 1237 struct poll_table_struct *wait) 1238 { 1239 struct sock *sk = sock->sk; 1240 struct mptcp_sock *msk; 1241 struct socket *ssock; 1242 __poll_t mask = 0; 1243 1244 msk = mptcp_sk(sk); 1245 lock_sock(sk); 1246 ssock = __mptcp_nmpc_socket(msk); 1247 if (ssock) { 1248 mask = ssock->ops->poll(file, ssock, wait); 1249 release_sock(sk); 1250 return mask; 1251 } 1252 1253 release_sock(sk); 1254 sock_poll_wait(file, sock, wait); 1255 lock_sock(sk); 1256 ssock = __mptcp_tcp_fallback(msk); 1257 if (unlikely(ssock)) 1258 return ssock->ops->poll(file, ssock, NULL); 1259 1260 if (test_bit(MPTCP_DATA_READY, &msk->flags)) 1261 mask = EPOLLIN | EPOLLRDNORM; 1262 if (sk_stream_is_writeable(sk) && 1263 test_bit(MPTCP_SEND_SPACE, &msk->flags)) 1264 mask |= EPOLLOUT | EPOLLWRNORM; 1265 if (sk->sk_shutdown & RCV_SHUTDOWN) 1266 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 1267 1268 release_sock(sk); 1269 1270 return mask; 1271 } 1272 1273 static int mptcp_shutdown(struct socket *sock, int how) 1274 { 1275 struct mptcp_sock *msk = mptcp_sk(sock->sk); 1276 struct mptcp_subflow_context *subflow; 1277 int ret = 0; 1278 1279 pr_debug("sk=%p, how=%d", msk, how); 1280 1281 lock_sock(sock->sk); 1282 1283 if (how == SHUT_WR || how == SHUT_RDWR) 1284 inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 1285 1286 how++; 1287 1288 if ((how & ~SHUTDOWN_MASK) || !how) { 1289 ret = -EINVAL; 1290 goto out_unlock; 1291 } 1292 1293 if (sock->state == SS_CONNECTING) { 1294 if ((1 << sock->sk->sk_state) & 1295 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 1296 sock->state = SS_DISCONNECTING; 1297 else 1298 sock->state = SS_CONNECTED; 1299 } 1300 1301 mptcp_for_each_subflow(msk, subflow) { 1302 struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 1303 1304 mptcp_subflow_shutdown(tcp_sk, how); 1305 } 1306 1307 out_unlock: 1308 release_sock(sock->sk); 1309 1310 return ret; 1311 } 1312 1313 static const struct proto_ops mptcp_stream_ops = { 1314 .family = PF_INET, 1315 .owner = THIS_MODULE, 1316 .release = inet_release, 1317 .bind = mptcp_bind, 1318 .connect = mptcp_stream_connect, 1319 .socketpair = sock_no_socketpair, 1320 .accept = mptcp_stream_accept, 1321 .getname = mptcp_v4_getname, 1322 .poll = mptcp_poll, 1323 .ioctl = inet_ioctl, 1324 .gettstamp = sock_gettstamp, 1325 .listen = mptcp_listen, 1326 .shutdown = mptcp_shutdown, 1327 .setsockopt = sock_common_setsockopt, 1328 .getsockopt = sock_common_getsockopt, 1329 .sendmsg = inet_sendmsg, 1330 .recvmsg = inet_recvmsg, 1331 .mmap = sock_no_mmap, 1332 .sendpage = inet_sendpage, 1333 #ifdef CONFIG_COMPAT 1334 .compat_setsockopt = compat_sock_common_setsockopt, 1335 .compat_getsockopt = compat_sock_common_getsockopt, 1336 #endif 1337 }; 1338 1339 static struct inet_protosw mptcp_protosw = { 1340 .type = SOCK_STREAM, 1341 .protocol = IPPROTO_MPTCP, 1342 .prot = &mptcp_prot, 1343 .ops = &mptcp_stream_ops, 1344 .flags = INET_PROTOSW_ICSK, 1345 }; 1346 1347 void mptcp_proto_init(void) 1348 { 1349 mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 1350 1351 mptcp_subflow_init(); 1352 1353 if (proto_register(&mptcp_prot, 1) != 0) 1354 panic("Failed to register MPTCP proto.\n"); 1355 1356 inet_register_protosw(&mptcp_protosw); 1357 1358 BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 1359 } 1360 1361 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1362 static const struct proto_ops mptcp_v6_stream_ops = { 1363 .family = PF_INET6, 1364 .owner = THIS_MODULE, 1365 .release = inet6_release, 1366 .bind = mptcp_bind, 1367 .connect = mptcp_stream_connect, 1368 .socketpair = sock_no_socketpair, 1369 .accept = mptcp_stream_accept, 1370 .getname = mptcp_v6_getname, 1371 .poll = mptcp_poll, 1372 .ioctl = inet6_ioctl, 1373 .gettstamp = sock_gettstamp, 1374 .listen = mptcp_listen, 1375 .shutdown = mptcp_shutdown, 1376 .setsockopt = sock_common_setsockopt, 1377 .getsockopt = sock_common_getsockopt, 1378 .sendmsg = inet6_sendmsg, 1379 .recvmsg = inet6_recvmsg, 1380 .mmap = sock_no_mmap, 1381 .sendpage = inet_sendpage, 1382 #ifdef CONFIG_COMPAT 1383 .compat_setsockopt = compat_sock_common_setsockopt, 1384 .compat_getsockopt = compat_sock_common_getsockopt, 1385 #endif 1386 }; 1387 1388 static struct proto mptcp_v6_prot; 1389 1390 static void mptcp_v6_destroy(struct sock *sk) 1391 { 1392 mptcp_destroy(sk); 1393 inet6_destroy_sock(sk); 1394 } 1395 1396 static struct inet_protosw mptcp_v6_protosw = { 1397 .type = SOCK_STREAM, 1398 .protocol = IPPROTO_MPTCP, 1399 .prot = &mptcp_v6_prot, 1400 .ops = &mptcp_v6_stream_ops, 1401 .flags = INET_PROTOSW_ICSK, 1402 }; 1403 1404 int mptcp_proto_v6_init(void) 1405 { 1406 int err; 1407 1408 mptcp_v6_prot = mptcp_prot; 1409 strcpy(mptcp_v6_prot.name, "MPTCPv6"); 1410 mptcp_v6_prot.slab = NULL; 1411 mptcp_v6_prot.destroy = mptcp_v6_destroy; 1412 mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 1413 1414 err = proto_register(&mptcp_v6_prot, 1); 1415 if (err) 1416 return err; 1417 1418 err = inet6_register_protosw(&mptcp_v6_protosw); 1419 if (err) 1420 proto_unregister(&mptcp_v6_prot); 1421 1422 return err; 1423 } 1424 #endif 1425