1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <crypto/sha2.h> 13 #include <crypto/utils.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #include <net/tcp.h> 19 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20 #include <net/ip6_route.h> 21 #include <net/transp_v6.h> 22 #endif 23 #include <net/mptcp.h> 24 #include <uapi/linux/mptcp.h> 25 #include "protocol.h" 26 #include "mib.h" 27 28 #include <trace/events/mptcp.h> 29 #include <trace/events/sock.h> 30 31 static void mptcp_subflow_ops_undo_override(struct sock *ssk); 32 33 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 34 enum linux_mptcp_mib_field field) 35 { 36 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 37 } 38 39 static void subflow_req_destructor(struct request_sock *req) 40 { 41 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 42 43 pr_debug("subflow_req=%p", subflow_req); 44 45 if (subflow_req->msk) 46 sock_put((struct sock *)subflow_req->msk); 47 48 mptcp_token_destroy_request(req); 49 } 50 51 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 52 void *hmac) 53 { 54 u8 msg[8]; 55 56 put_unaligned_be32(nonce1, &msg[0]); 57 put_unaligned_be32(nonce2, &msg[4]); 58 59 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 60 } 61 62 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk) 63 { 64 return mptcp_is_fully_established((void *)msk) && 65 ((mptcp_pm_is_userspace(msk) && 66 mptcp_userspace_pm_active(msk)) || 67 READ_ONCE(msk->pm.accept_subflow)); 68 } 69 70 /* validate received token and create truncated hmac and nonce for SYN-ACK */ 71 static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req) 72 { 73 struct mptcp_sock *msk = subflow_req->msk; 74 u8 hmac[SHA256_DIGEST_SIZE]; 75 76 get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); 77 78 subflow_generate_hmac(READ_ONCE(msk->local_key), 79 READ_ONCE(msk->remote_key), 80 subflow_req->local_nonce, 81 subflow_req->remote_nonce, hmac); 82 83 subflow_req->thmac = get_unaligned_be64(hmac); 84 } 85 86 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req) 87 { 88 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 89 struct mptcp_sock *msk; 90 int local_id; 91 92 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token); 93 if (!msk) { 94 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 95 return NULL; 96 } 97 98 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 99 if (local_id < 0) { 100 sock_put((struct sock *)msk); 101 return NULL; 102 } 103 subflow_req->local_id = local_id; 104 105 return msk; 106 } 107 108 static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener) 109 { 110 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 111 112 subflow_req->mp_capable = 0; 113 subflow_req->mp_join = 0; 114 subflow_req->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk_listener)); 115 subflow_req->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk_listener)); 116 subflow_req->msk = NULL; 117 mptcp_token_init_request(req); 118 } 119 120 static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk) 121 { 122 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport; 123 } 124 125 static void subflow_add_reset_reason(struct sk_buff *skb, u8 reason) 126 { 127 struct mptcp_ext *mpext = skb_ext_add(skb, SKB_EXT_MPTCP); 128 129 if (mpext) { 130 memset(mpext, 0, sizeof(*mpext)); 131 mpext->reset_reason = reason; 132 } 133 } 134 135 /* Init mptcp request socket. 136 * 137 * Returns an error code if a JOIN has failed and a TCP reset 138 * should be sent. 139 */ 140 static int subflow_check_req(struct request_sock *req, 141 const struct sock *sk_listener, 142 struct sk_buff *skb) 143 { 144 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 145 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 146 struct mptcp_options_received mp_opt; 147 bool opt_mp_capable, opt_mp_join; 148 149 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); 150 151 #ifdef CONFIG_TCP_MD5SIG 152 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 153 * TCP option space. 154 */ 155 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) 156 return -EINVAL; 157 #endif 158 159 mptcp_get_options(skb, &mp_opt); 160 161 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYN); 162 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYN); 163 if (opt_mp_capable) { 164 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 165 166 if (opt_mp_join) 167 return 0; 168 } else if (opt_mp_join) { 169 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 170 } 171 172 if (opt_mp_capable && listener->request_mptcp) { 173 int err, retries = MPTCP_TOKEN_MAX_RETRIES; 174 175 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 176 again: 177 do { 178 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key)); 179 } while (subflow_req->local_key == 0); 180 181 if (unlikely(req->syncookie)) { 182 mptcp_crypto_key_sha(subflow_req->local_key, 183 &subflow_req->token, 184 &subflow_req->idsn); 185 if (mptcp_token_exists(subflow_req->token)) { 186 if (retries-- > 0) 187 goto again; 188 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 189 } else { 190 subflow_req->mp_capable = 1; 191 } 192 return 0; 193 } 194 195 err = mptcp_token_new_request(req); 196 if (err == 0) 197 subflow_req->mp_capable = 1; 198 else if (retries-- > 0) 199 goto again; 200 else 201 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 202 203 } else if (opt_mp_join && listener->request_mptcp) { 204 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 205 subflow_req->mp_join = 1; 206 subflow_req->backup = mp_opt.backup; 207 subflow_req->remote_id = mp_opt.join_id; 208 subflow_req->token = mp_opt.token; 209 subflow_req->remote_nonce = mp_opt.nonce; 210 subflow_req->msk = subflow_token_join_request(req); 211 212 /* Can't fall back to TCP in this case. */ 213 if (!subflow_req->msk) { 214 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 215 return -EPERM; 216 } 217 218 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) { 219 pr_debug("syn inet_sport=%d %d", 220 ntohs(inet_sk(sk_listener)->inet_sport), 221 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport)); 222 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) { 223 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX); 224 return -EPERM; 225 } 226 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX); 227 } 228 229 subflow_req_create_thmac(subflow_req); 230 231 if (unlikely(req->syncookie)) { 232 if (mptcp_can_accept_new_subflow(subflow_req->msk)) 233 subflow_init_req_cookie_join_save(subflow_req, skb); 234 else 235 return -EPERM; 236 } 237 238 pr_debug("token=%u, remote_nonce=%u msk=%p", subflow_req->token, 239 subflow_req->remote_nonce, subflow_req->msk); 240 } 241 242 return 0; 243 } 244 245 int mptcp_subflow_init_cookie_req(struct request_sock *req, 246 const struct sock *sk_listener, 247 struct sk_buff *skb) 248 { 249 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 250 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 251 struct mptcp_options_received mp_opt; 252 bool opt_mp_capable, opt_mp_join; 253 int err; 254 255 subflow_init_req(req, sk_listener); 256 mptcp_get_options(skb, &mp_opt); 257 258 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_ACK); 259 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK); 260 if (opt_mp_capable && opt_mp_join) 261 return -EINVAL; 262 263 if (opt_mp_capable && listener->request_mptcp) { 264 if (mp_opt.sndr_key == 0) 265 return -EINVAL; 266 267 subflow_req->local_key = mp_opt.rcvr_key; 268 err = mptcp_token_new_request(req); 269 if (err) 270 return err; 271 272 subflow_req->mp_capable = 1; 273 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 274 } else if (opt_mp_join && listener->request_mptcp) { 275 if (!mptcp_token_join_cookie_init_state(subflow_req, skb)) 276 return -EINVAL; 277 278 subflow_req->mp_join = 1; 279 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 280 } 281 282 return 0; 283 } 284 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req); 285 286 static struct dst_entry *subflow_v4_route_req(const struct sock *sk, 287 struct sk_buff *skb, 288 struct flowi *fl, 289 struct request_sock *req) 290 { 291 struct dst_entry *dst; 292 int err; 293 294 tcp_rsk(req)->is_mptcp = 1; 295 subflow_init_req(req, sk); 296 297 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req); 298 if (!dst) 299 return NULL; 300 301 err = subflow_check_req(req, sk, skb); 302 if (err == 0) 303 return dst; 304 305 dst_release(dst); 306 if (!req->syncookie) 307 tcp_request_sock_ops.send_reset(sk, skb); 308 return NULL; 309 } 310 311 static void subflow_prep_synack(const struct sock *sk, struct request_sock *req, 312 struct tcp_fastopen_cookie *foc, 313 enum tcp_synack_type synack_type) 314 { 315 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 316 struct inet_request_sock *ireq = inet_rsk(req); 317 318 /* clear tstamp_ok, as needed depending on cookie */ 319 if (foc && foc->len > -1) 320 ireq->tstamp_ok = 0; 321 322 if (synack_type == TCP_SYNACK_FASTOPEN) 323 mptcp_fastopen_subflow_synack_set_params(subflow, req); 324 } 325 326 static int subflow_v4_send_synack(const struct sock *sk, struct dst_entry *dst, 327 struct flowi *fl, 328 struct request_sock *req, 329 struct tcp_fastopen_cookie *foc, 330 enum tcp_synack_type synack_type, 331 struct sk_buff *syn_skb) 332 { 333 subflow_prep_synack(sk, req, foc, synack_type); 334 335 return tcp_request_sock_ipv4_ops.send_synack(sk, dst, fl, req, foc, 336 synack_type, syn_skb); 337 } 338 339 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 340 static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 341 struct flowi *fl, 342 struct request_sock *req, 343 struct tcp_fastopen_cookie *foc, 344 enum tcp_synack_type synack_type, 345 struct sk_buff *syn_skb) 346 { 347 subflow_prep_synack(sk, req, foc, synack_type); 348 349 return tcp_request_sock_ipv6_ops.send_synack(sk, dst, fl, req, foc, 350 synack_type, syn_skb); 351 } 352 353 static struct dst_entry *subflow_v6_route_req(const struct sock *sk, 354 struct sk_buff *skb, 355 struct flowi *fl, 356 struct request_sock *req) 357 { 358 struct dst_entry *dst; 359 int err; 360 361 tcp_rsk(req)->is_mptcp = 1; 362 subflow_init_req(req, sk); 363 364 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req); 365 if (!dst) 366 return NULL; 367 368 err = subflow_check_req(req, sk, skb); 369 if (err == 0) 370 return dst; 371 372 dst_release(dst); 373 if (!req->syncookie) 374 tcp6_request_sock_ops.send_reset(sk, skb); 375 return NULL; 376 } 377 #endif 378 379 /* validate received truncated hmac and create hmac for third ACK */ 380 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 381 { 382 u8 hmac[SHA256_DIGEST_SIZE]; 383 u64 thmac; 384 385 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 386 subflow->remote_nonce, subflow->local_nonce, 387 hmac); 388 389 thmac = get_unaligned_be64(hmac); 390 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 391 subflow, subflow->token, thmac, subflow->thmac); 392 393 return thmac == subflow->thmac; 394 } 395 396 void mptcp_subflow_reset(struct sock *ssk) 397 { 398 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 399 struct sock *sk = subflow->conn; 400 401 /* mptcp_mp_fail_no_response() can reach here on an already closed 402 * socket 403 */ 404 if (ssk->sk_state == TCP_CLOSE) 405 return; 406 407 /* must hold: tcp_done() could drop last reference on parent */ 408 sock_hold(sk); 409 410 tcp_send_active_reset(ssk, GFP_ATOMIC); 411 tcp_done(ssk); 412 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags)) 413 mptcp_schedule_work(sk); 414 415 sock_put(sk); 416 } 417 418 static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk) 419 { 420 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport; 421 } 422 423 void __mptcp_sync_state(struct sock *sk, int state) 424 { 425 struct mptcp_sock *msk = mptcp_sk(sk); 426 427 __mptcp_propagate_sndbuf(sk, msk->first); 428 if (sk->sk_state == TCP_SYN_SENT) { 429 mptcp_set_state(sk, state); 430 sk->sk_state_change(sk); 431 } 432 } 433 434 static void mptcp_propagate_state(struct sock *sk, struct sock *ssk) 435 { 436 struct mptcp_sock *msk = mptcp_sk(sk); 437 438 mptcp_data_lock(sk); 439 if (!sock_owned_by_user(sk)) { 440 __mptcp_sync_state(sk, ssk->sk_state); 441 } else { 442 msk->pending_state = ssk->sk_state; 443 __set_bit(MPTCP_SYNC_STATE, &msk->cb_flags); 444 } 445 mptcp_data_unlock(sk); 446 } 447 448 static void subflow_set_remote_key(struct mptcp_sock *msk, 449 struct mptcp_subflow_context *subflow, 450 const struct mptcp_options_received *mp_opt) 451 { 452 /* active MPC subflow will reach here multiple times: 453 * at subflow_finish_connect() time and at 4th ack time 454 */ 455 if (subflow->remote_key_valid) 456 return; 457 458 subflow->remote_key_valid = 1; 459 subflow->remote_key = mp_opt->sndr_key; 460 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn); 461 subflow->iasn++; 462 463 WRITE_ONCE(msk->remote_key, subflow->remote_key); 464 WRITE_ONCE(msk->ack_seq, subflow->iasn); 465 WRITE_ONCE(msk->can_ack, true); 466 atomic64_set(&msk->rcv_wnd_sent, subflow->iasn); 467 } 468 469 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 470 { 471 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 472 struct mptcp_options_received mp_opt; 473 struct sock *parent = subflow->conn; 474 struct mptcp_sock *msk; 475 476 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 477 478 /* be sure no special action on any packet other than syn-ack */ 479 if (subflow->conn_finished) 480 return; 481 482 msk = mptcp_sk(parent); 483 subflow->rel_write_seq = 1; 484 subflow->conn_finished = 1; 485 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 486 pr_debug("subflow=%p synack seq=%x", subflow, subflow->ssn_offset); 487 488 mptcp_get_options(skb, &mp_opt); 489 if (subflow->request_mptcp) { 490 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) { 491 MPTCP_INC_STATS(sock_net(sk), 492 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK); 493 mptcp_do_fallback(sk); 494 pr_fallback(msk); 495 goto fallback; 496 } 497 498 if (mp_opt.suboptions & OPTION_MPTCP_CSUMREQD) 499 WRITE_ONCE(msk->csum_enabled, true); 500 if (mp_opt.deny_join_id0) 501 WRITE_ONCE(msk->pm.remote_deny_join_id0, true); 502 subflow->mp_capable = 1; 503 subflow_set_remote_key(msk, subflow, &mp_opt); 504 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK); 505 mptcp_finish_connect(sk); 506 mptcp_propagate_state(parent, sk); 507 } else if (subflow->request_join) { 508 u8 hmac[SHA256_DIGEST_SIZE]; 509 510 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYNACK)) { 511 subflow->reset_reason = MPTCP_RST_EMPTCP; 512 goto do_reset; 513 } 514 515 subflow->backup = mp_opt.backup; 516 subflow->thmac = mp_opt.thmac; 517 subflow->remote_nonce = mp_opt.nonce; 518 subflow->remote_id = mp_opt.join_id; 519 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d", 520 subflow, subflow->thmac, subflow->remote_nonce, 521 subflow->backup); 522 523 if (!subflow_thmac_valid(subflow)) { 524 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); 525 subflow->reset_reason = MPTCP_RST_EMPTCP; 526 goto do_reset; 527 } 528 529 if (!mptcp_finish_join(sk)) 530 goto do_reset; 531 532 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 533 subflow->local_nonce, 534 subflow->remote_nonce, 535 hmac); 536 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN); 537 538 subflow->mp_join = 1; 539 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 540 541 if (subflow_use_different_dport(msk, sk)) { 542 pr_debug("synack inet_dport=%d %d", 543 ntohs(inet_sk(sk)->inet_dport), 544 ntohs(inet_sk(parent)->inet_dport)); 545 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX); 546 } 547 } else if (mptcp_check_fallback(sk)) { 548 fallback: 549 mptcp_rcv_space_init(msk, sk); 550 mptcp_propagate_state(parent, sk); 551 } 552 return; 553 554 do_reset: 555 subflow->reset_transient = 0; 556 mptcp_subflow_reset(sk); 557 } 558 559 static void subflow_set_local_id(struct mptcp_subflow_context *subflow, int local_id) 560 { 561 subflow->local_id = local_id; 562 subflow->local_id_valid = 1; 563 } 564 565 static int subflow_chk_local_id(struct sock *sk) 566 { 567 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 568 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 569 int err; 570 571 if (likely(subflow->local_id_valid)) 572 return 0; 573 574 err = mptcp_pm_get_local_id(msk, (struct sock_common *)sk); 575 if (err < 0) 576 return err; 577 578 subflow_set_local_id(subflow, err); 579 return 0; 580 } 581 582 static int subflow_rebuild_header(struct sock *sk) 583 { 584 int err = subflow_chk_local_id(sk); 585 586 if (unlikely(err < 0)) 587 return err; 588 589 return inet_sk_rebuild_header(sk); 590 } 591 592 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 593 static int subflow_v6_rebuild_header(struct sock *sk) 594 { 595 int err = subflow_chk_local_id(sk); 596 597 if (unlikely(err < 0)) 598 return err; 599 600 return inet6_sk_rebuild_header(sk); 601 } 602 #endif 603 604 static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init; 605 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init; 606 607 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 608 { 609 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 610 611 pr_debug("subflow=%p", subflow); 612 613 /* Never answer to SYNs sent to broadcast or multicast */ 614 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 615 goto drop; 616 617 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops, 618 &subflow_request_sock_ipv4_ops, 619 sk, skb); 620 drop: 621 tcp_listendrop(sk); 622 return 0; 623 } 624 625 static void subflow_v4_req_destructor(struct request_sock *req) 626 { 627 subflow_req_destructor(req); 628 tcp_request_sock_ops.destructor(req); 629 } 630 631 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 632 static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init; 633 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init; 634 static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init; 635 static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init; 636 static struct proto tcpv6_prot_override __ro_after_init; 637 638 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 639 { 640 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 641 642 pr_debug("subflow=%p", subflow); 643 644 if (skb->protocol == htons(ETH_P_IP)) 645 return subflow_v4_conn_request(sk, skb); 646 647 if (!ipv6_unicast_destination(skb)) 648 goto drop; 649 650 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 651 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 652 return 0; 653 } 654 655 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops, 656 &subflow_request_sock_ipv6_ops, sk, skb); 657 658 drop: 659 tcp_listendrop(sk); 660 return 0; /* don't send reset */ 661 } 662 663 static void subflow_v6_req_destructor(struct request_sock *req) 664 { 665 subflow_req_destructor(req); 666 tcp6_request_sock_ops.destructor(req); 667 } 668 #endif 669 670 struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops, 671 struct sock *sk_listener, 672 bool attach_listener) 673 { 674 if (ops->family == AF_INET) 675 ops = &mptcp_subflow_v4_request_sock_ops; 676 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 677 else if (ops->family == AF_INET6) 678 ops = &mptcp_subflow_v6_request_sock_ops; 679 #endif 680 681 return inet_reqsk_alloc(ops, sk_listener, attach_listener); 682 } 683 EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc); 684 685 /* validate hmac received in third ACK */ 686 static bool subflow_hmac_valid(const struct request_sock *req, 687 const struct mptcp_options_received *mp_opt) 688 { 689 const struct mptcp_subflow_request_sock *subflow_req; 690 u8 hmac[SHA256_DIGEST_SIZE]; 691 struct mptcp_sock *msk; 692 693 subflow_req = mptcp_subflow_rsk(req); 694 msk = subflow_req->msk; 695 if (!msk) 696 return false; 697 698 subflow_generate_hmac(READ_ONCE(msk->remote_key), 699 READ_ONCE(msk->local_key), 700 subflow_req->remote_nonce, 701 subflow_req->local_nonce, hmac); 702 703 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); 704 } 705 706 static void subflow_ulp_fallback(struct sock *sk, 707 struct mptcp_subflow_context *old_ctx) 708 { 709 struct inet_connection_sock *icsk = inet_csk(sk); 710 711 mptcp_subflow_tcp_fallback(sk, old_ctx); 712 icsk->icsk_ulp_ops = NULL; 713 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 714 tcp_sk(sk)->is_mptcp = 0; 715 716 mptcp_subflow_ops_undo_override(sk); 717 } 718 719 void mptcp_subflow_drop_ctx(struct sock *ssk) 720 { 721 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 722 723 if (!ctx) 724 return; 725 726 list_del(&mptcp_subflow_ctx(ssk)->node); 727 if (inet_csk(ssk)->icsk_ulp_ops) { 728 subflow_ulp_fallback(ssk, ctx); 729 if (ctx->conn) 730 sock_put(ctx->conn); 731 } 732 733 kfree_rcu(ctx, rcu); 734 } 735 736 void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, 737 const struct mptcp_options_received *mp_opt) 738 { 739 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 740 741 subflow_set_remote_key(msk, subflow, mp_opt); 742 subflow->fully_established = 1; 743 WRITE_ONCE(msk->fully_established, true); 744 745 if (subflow->is_mptfo) 746 mptcp_fastopen_gen_msk_ackseq(msk, subflow, mp_opt); 747 } 748 749 static struct sock *subflow_syn_recv_sock(const struct sock *sk, 750 struct sk_buff *skb, 751 struct request_sock *req, 752 struct dst_entry *dst, 753 struct request_sock *req_unhash, 754 bool *own_req) 755 { 756 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 757 struct mptcp_subflow_request_sock *subflow_req; 758 struct mptcp_options_received mp_opt; 759 bool fallback, fallback_is_fatal; 760 struct mptcp_sock *owner; 761 struct sock *child; 762 763 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); 764 765 /* After child creation we must look for MPC even when options 766 * are not parsed 767 */ 768 mp_opt.suboptions = 0; 769 770 /* hopefully temporary handling for MP_JOIN+syncookie */ 771 subflow_req = mptcp_subflow_rsk(req); 772 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join; 773 fallback = !tcp_rsk(req)->is_mptcp; 774 if (fallback) 775 goto create_child; 776 777 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 778 if (subflow_req->mp_capable) { 779 /* we can receive and accept an in-window, out-of-order pkt, 780 * which may not carry the MP_CAPABLE opt even on mptcp enabled 781 * paths: always try to extract the peer key, and fallback 782 * for packets missing it. 783 * Even OoO DSS packets coming legitly after dropped or 784 * reordered MPC will cause fallback, but we don't have other 785 * options. 786 */ 787 mptcp_get_options(skb, &mp_opt); 788 if (!(mp_opt.suboptions & 789 (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_ACK))) 790 fallback = true; 791 792 } else if (subflow_req->mp_join) { 793 mptcp_get_options(skb, &mp_opt); 794 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK) || 795 !subflow_hmac_valid(req, &mp_opt) || 796 !mptcp_can_accept_new_subflow(subflow_req->msk)) { 797 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 798 fallback = true; 799 } 800 } 801 802 create_child: 803 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 804 req_unhash, own_req); 805 806 if (child && *own_req) { 807 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 808 809 tcp_rsk(req)->drop_req = false; 810 811 /* we need to fallback on ctx allocation failure and on pre-reqs 812 * checking above. In the latter scenario we additionally need 813 * to reset the context to non MPTCP status. 814 */ 815 if (!ctx || fallback) { 816 if (fallback_is_fatal) { 817 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 818 goto dispose_child; 819 } 820 goto fallback; 821 } 822 823 /* ssk inherits options of listener sk */ 824 ctx->setsockopt_seq = listener->setsockopt_seq; 825 826 if (ctx->mp_capable) { 827 ctx->conn = mptcp_sk_clone_init(listener->conn, &mp_opt, child, req); 828 if (!ctx->conn) 829 goto fallback; 830 831 ctx->subflow_id = 1; 832 owner = mptcp_sk(ctx->conn); 833 mptcp_pm_new_connection(owner, child, 1); 834 835 /* with OoO packets we can reach here without ingress 836 * mpc option 837 */ 838 if (mp_opt.suboptions & OPTION_MPTCP_MPC_ACK) { 839 mptcp_subflow_fully_established(ctx, &mp_opt); 840 mptcp_pm_fully_established(owner, child); 841 ctx->pm_notified = 1; 842 } 843 } else if (ctx->mp_join) { 844 owner = subflow_req->msk; 845 if (!owner) { 846 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 847 goto dispose_child; 848 } 849 850 /* move the msk reference ownership to the subflow */ 851 subflow_req->msk = NULL; 852 ctx->conn = (struct sock *)owner; 853 854 if (subflow_use_different_sport(owner, sk)) { 855 pr_debug("ack inet_sport=%d %d", 856 ntohs(inet_sk(sk)->inet_sport), 857 ntohs(inet_sk((struct sock *)owner)->inet_sport)); 858 if (!mptcp_pm_sport_in_anno_list(owner, sk)) { 859 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX); 860 goto dispose_child; 861 } 862 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX); 863 } 864 865 if (!mptcp_finish_join(child)) 866 goto dispose_child; 867 868 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 869 tcp_rsk(req)->drop_req = true; 870 } 871 } 872 873 /* check for expected invariant - should never trigger, just help 874 * catching eariler subtle bugs 875 */ 876 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 877 (!mptcp_subflow_ctx(child) || 878 !mptcp_subflow_ctx(child)->conn)); 879 return child; 880 881 dispose_child: 882 mptcp_subflow_drop_ctx(child); 883 tcp_rsk(req)->drop_req = true; 884 inet_csk_prepare_for_destroy_sock(child); 885 tcp_done(child); 886 req->rsk_ops->send_reset(sk, skb); 887 888 /* The last child reference will be released by the caller */ 889 return child; 890 891 fallback: 892 mptcp_subflow_drop_ctx(child); 893 return child; 894 } 895 896 static struct inet_connection_sock_af_ops subflow_specific __ro_after_init; 897 static struct proto tcp_prot_override __ro_after_init; 898 899 enum mapping_status { 900 MAPPING_OK, 901 MAPPING_INVALID, 902 MAPPING_EMPTY, 903 MAPPING_DATA_FIN, 904 MAPPING_DUMMY, 905 MAPPING_BAD_CSUM 906 }; 907 908 static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 909 { 910 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d", 911 ssn, subflow->map_subflow_seq, subflow->map_data_len); 912 } 913 914 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 915 { 916 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 917 unsigned int skb_consumed; 918 919 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 920 if (WARN_ON_ONCE(skb_consumed >= skb->len)) 921 return true; 922 923 return skb->len - skb_consumed <= subflow->map_data_len - 924 mptcp_subflow_get_map_offset(subflow); 925 } 926 927 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 928 { 929 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 930 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 931 932 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 933 /* Mapping covers data later in the subflow stream, 934 * currently unsupported. 935 */ 936 dbg_bad_map(subflow, ssn); 937 return false; 938 } 939 if (unlikely(!before(ssn, subflow->map_subflow_seq + 940 subflow->map_data_len))) { 941 /* Mapping does covers past subflow data, invalid */ 942 dbg_bad_map(subflow, ssn); 943 return false; 944 } 945 return true; 946 } 947 948 static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb, 949 bool csum_reqd) 950 { 951 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 952 u32 offset, seq, delta; 953 __sum16 csum; 954 int len; 955 956 if (!csum_reqd) 957 return MAPPING_OK; 958 959 /* mapping already validated on previous traversal */ 960 if (subflow->map_csum_len == subflow->map_data_len) 961 return MAPPING_OK; 962 963 /* traverse the receive queue, ensuring it contains a full 964 * DSS mapping and accumulating the related csum. 965 * Preserve the accoumlate csum across multiple calls, to compute 966 * the csum only once 967 */ 968 delta = subflow->map_data_len - subflow->map_csum_len; 969 for (;;) { 970 seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len; 971 offset = seq - TCP_SKB_CB(skb)->seq; 972 973 /* if the current skb has not been accounted yet, csum its contents 974 * up to the amount covered by the current DSS 975 */ 976 if (offset < skb->len) { 977 __wsum csum; 978 979 len = min(skb->len - offset, delta); 980 csum = skb_checksum(skb, offset, len, 0); 981 subflow->map_data_csum = csum_block_add(subflow->map_data_csum, csum, 982 subflow->map_csum_len); 983 984 delta -= len; 985 subflow->map_csum_len += len; 986 } 987 if (delta == 0) 988 break; 989 990 if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) { 991 /* if this subflow is closed, the partial mapping 992 * will be never completed; flush the pending skbs, so 993 * that subflow_sched_work_if_closed() can kick in 994 */ 995 if (unlikely(ssk->sk_state == TCP_CLOSE)) 996 while ((skb = skb_peek(&ssk->sk_receive_queue))) 997 sk_eat_skb(ssk, skb); 998 999 /* not enough data to validate the csum */ 1000 return MAPPING_EMPTY; 1001 } 1002 1003 /* the DSS mapping for next skbs will be validated later, 1004 * when a get_mapping_status call will process such skb 1005 */ 1006 skb = skb->next; 1007 } 1008 1009 /* note that 'map_data_len' accounts only for the carried data, does 1010 * not include the eventual seq increment due to the data fin, 1011 * while the pseudo header requires the original DSS data len, 1012 * including that 1013 */ 1014 csum = __mptcp_make_csum(subflow->map_seq, 1015 subflow->map_subflow_seq, 1016 subflow->map_data_len + subflow->map_data_fin, 1017 subflow->map_data_csum); 1018 if (unlikely(csum)) { 1019 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR); 1020 return MAPPING_BAD_CSUM; 1021 } 1022 1023 subflow->valid_csum_seen = 1; 1024 return MAPPING_OK; 1025 } 1026 1027 static enum mapping_status get_mapping_status(struct sock *ssk, 1028 struct mptcp_sock *msk) 1029 { 1030 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1031 bool csum_reqd = READ_ONCE(msk->csum_enabled); 1032 struct mptcp_ext *mpext; 1033 struct sk_buff *skb; 1034 u16 data_len; 1035 u64 map_seq; 1036 1037 skb = skb_peek(&ssk->sk_receive_queue); 1038 if (!skb) 1039 return MAPPING_EMPTY; 1040 1041 if (mptcp_check_fallback(ssk)) 1042 return MAPPING_DUMMY; 1043 1044 mpext = mptcp_get_ext(skb); 1045 if (!mpext || !mpext->use_map) { 1046 if (!subflow->map_valid && !skb->len) { 1047 /* the TCP stack deliver 0 len FIN pkt to the receive 1048 * queue, that is the only 0len pkts ever expected here, 1049 * and we can admit no mapping only for 0 len pkts 1050 */ 1051 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 1052 WARN_ONCE(1, "0len seq %d:%d flags %x", 1053 TCP_SKB_CB(skb)->seq, 1054 TCP_SKB_CB(skb)->end_seq, 1055 TCP_SKB_CB(skb)->tcp_flags); 1056 sk_eat_skb(ssk, skb); 1057 return MAPPING_EMPTY; 1058 } 1059 1060 if (!subflow->map_valid) 1061 return MAPPING_INVALID; 1062 1063 goto validate_seq; 1064 } 1065 1066 trace_get_mapping_status(mpext); 1067 1068 data_len = mpext->data_len; 1069 if (data_len == 0) { 1070 pr_debug("infinite mapping received"); 1071 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 1072 subflow->map_data_len = 0; 1073 return MAPPING_INVALID; 1074 } 1075 1076 if (mpext->data_fin == 1) { 1077 if (data_len == 1) { 1078 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq, 1079 mpext->dsn64); 1080 pr_debug("DATA_FIN with no payload seq=%llu", mpext->data_seq); 1081 if (subflow->map_valid) { 1082 /* A DATA_FIN might arrive in a DSS 1083 * option before the previous mapping 1084 * has been fully consumed. Continue 1085 * handling the existing mapping. 1086 */ 1087 skb_ext_del(skb, SKB_EXT_MPTCP); 1088 return MAPPING_OK; 1089 } else { 1090 if (updated) 1091 mptcp_schedule_work((struct sock *)msk); 1092 1093 return MAPPING_DATA_FIN; 1094 } 1095 } else { 1096 u64 data_fin_seq = mpext->data_seq + data_len - 1; 1097 1098 /* If mpext->data_seq is a 32-bit value, data_fin_seq 1099 * must also be limited to 32 bits. 1100 */ 1101 if (!mpext->dsn64) 1102 data_fin_seq &= GENMASK_ULL(31, 0); 1103 1104 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64); 1105 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d", 1106 data_fin_seq, mpext->dsn64); 1107 } 1108 1109 /* Adjust for DATA_FIN using 1 byte of sequence space */ 1110 data_len--; 1111 } 1112 1113 map_seq = mptcp_expand_seq(READ_ONCE(msk->ack_seq), mpext->data_seq, mpext->dsn64); 1114 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64); 1115 1116 if (subflow->map_valid) { 1117 /* Allow replacing only with an identical map */ 1118 if (subflow->map_seq == map_seq && 1119 subflow->map_subflow_seq == mpext->subflow_seq && 1120 subflow->map_data_len == data_len && 1121 subflow->map_csum_reqd == mpext->csum_reqd) { 1122 skb_ext_del(skb, SKB_EXT_MPTCP); 1123 goto validate_csum; 1124 } 1125 1126 /* If this skb data are fully covered by the current mapping, 1127 * the new map would need caching, which is not supported 1128 */ 1129 if (skb_is_fully_mapped(ssk, skb)) { 1130 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 1131 return MAPPING_INVALID; 1132 } 1133 1134 /* will validate the next map after consuming the current one */ 1135 goto validate_csum; 1136 } 1137 1138 subflow->map_seq = map_seq; 1139 subflow->map_subflow_seq = mpext->subflow_seq; 1140 subflow->map_data_len = data_len; 1141 subflow->map_valid = 1; 1142 subflow->map_data_fin = mpext->data_fin; 1143 subflow->mpc_map = mpext->mpc_map; 1144 subflow->map_csum_reqd = mpext->csum_reqd; 1145 subflow->map_csum_len = 0; 1146 subflow->map_data_csum = csum_unfold(mpext->csum); 1147 1148 /* Cfr RFC 8684 Section 3.3.0 */ 1149 if (unlikely(subflow->map_csum_reqd != csum_reqd)) 1150 return MAPPING_INVALID; 1151 1152 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u csum=%d:%u", 1153 subflow->map_seq, subflow->map_subflow_seq, 1154 subflow->map_data_len, subflow->map_csum_reqd, 1155 subflow->map_data_csum); 1156 1157 validate_seq: 1158 /* we revalidate valid mapping on new skb, because we must ensure 1159 * the current skb is completely covered by the available mapping 1160 */ 1161 if (!validate_mapping(ssk, skb)) { 1162 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH); 1163 return MAPPING_INVALID; 1164 } 1165 1166 skb_ext_del(skb, SKB_EXT_MPTCP); 1167 1168 validate_csum: 1169 return validate_data_csum(ssk, skb, csum_reqd); 1170 } 1171 1172 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb, 1173 u64 limit) 1174 { 1175 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1176 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 1177 u32 incr; 1178 1179 incr = limit >= skb->len ? skb->len + fin : limit; 1180 1181 pr_debug("discarding=%d len=%d seq=%d", incr, skb->len, 1182 subflow->map_subflow_seq); 1183 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA); 1184 tcp_sk(ssk)->copied_seq += incr; 1185 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq)) 1186 sk_eat_skb(ssk, skb); 1187 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) 1188 subflow->map_valid = 0; 1189 } 1190 1191 /* sched mptcp worker to remove the subflow if no more data is pending */ 1192 static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk) 1193 { 1194 if (likely(ssk->sk_state != TCP_CLOSE)) 1195 return; 1196 1197 if (skb_queue_empty(&ssk->sk_receive_queue) && 1198 !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 1199 mptcp_schedule_work((struct sock *)msk); 1200 } 1201 1202 static bool subflow_can_fallback(struct mptcp_subflow_context *subflow) 1203 { 1204 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 1205 1206 if (subflow->mp_join) 1207 return false; 1208 else if (READ_ONCE(msk->csum_enabled)) 1209 return !subflow->valid_csum_seen; 1210 else 1211 return !subflow->fully_established; 1212 } 1213 1214 static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) 1215 { 1216 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1217 unsigned long fail_tout; 1218 1219 /* greceful failure can happen only on the MPC subflow */ 1220 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first))) 1221 return; 1222 1223 /* since the close timeout take precedence on the fail one, 1224 * no need to start the latter when the first is already set 1225 */ 1226 if (sock_flag((struct sock *)msk, SOCK_DEAD)) 1227 return; 1228 1229 /* we don't need extreme accuracy here, use a zero fail_tout as special 1230 * value meaning no fail timeout at all; 1231 */ 1232 fail_tout = jiffies + TCP_RTO_MAX; 1233 if (!fail_tout) 1234 fail_tout = 1; 1235 WRITE_ONCE(subflow->fail_tout, fail_tout); 1236 tcp_send_ack(ssk); 1237 1238 mptcp_reset_tout_timer(msk, subflow->fail_tout); 1239 } 1240 1241 static bool subflow_check_data_avail(struct sock *ssk) 1242 { 1243 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1244 enum mapping_status status; 1245 struct mptcp_sock *msk; 1246 struct sk_buff *skb; 1247 1248 if (!skb_peek(&ssk->sk_receive_queue)) 1249 WRITE_ONCE(subflow->data_avail, false); 1250 if (subflow->data_avail) 1251 return true; 1252 1253 msk = mptcp_sk(subflow->conn); 1254 for (;;) { 1255 u64 ack_seq; 1256 u64 old_ack; 1257 1258 status = get_mapping_status(ssk, msk); 1259 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue)); 1260 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY || 1261 status == MAPPING_BAD_CSUM)) 1262 goto fallback; 1263 1264 if (status != MAPPING_OK) 1265 goto no_data; 1266 1267 skb = skb_peek(&ssk->sk_receive_queue); 1268 if (WARN_ON_ONCE(!skb)) 1269 goto no_data; 1270 1271 if (unlikely(!READ_ONCE(msk->can_ack))) 1272 goto fallback; 1273 1274 old_ack = READ_ONCE(msk->ack_seq); 1275 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 1276 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, 1277 ack_seq); 1278 if (unlikely(before64(ack_seq, old_ack))) { 1279 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq); 1280 continue; 1281 } 1282 1283 WRITE_ONCE(subflow->data_avail, true); 1284 break; 1285 } 1286 return true; 1287 1288 no_data: 1289 subflow_sched_work_if_closed(msk, ssk); 1290 return false; 1291 1292 fallback: 1293 if (!__mptcp_check_fallback(msk)) { 1294 /* RFC 8684 section 3.7. */ 1295 if (status == MAPPING_BAD_CSUM && 1296 (subflow->mp_join || subflow->valid_csum_seen)) { 1297 subflow->send_mp_fail = 1; 1298 1299 if (!READ_ONCE(msk->allow_infinite_fallback)) { 1300 subflow->reset_transient = 0; 1301 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX; 1302 goto reset; 1303 } 1304 mptcp_subflow_fail(msk, ssk); 1305 WRITE_ONCE(subflow->data_avail, true); 1306 return true; 1307 } 1308 1309 if (!subflow_can_fallback(subflow) && subflow->map_data_len) { 1310 /* fatal protocol error, close the socket. 1311 * subflow_error_report() will introduce the appropriate barriers 1312 */ 1313 subflow->reset_transient = 0; 1314 subflow->reset_reason = MPTCP_RST_EMPTCP; 1315 1316 reset: 1317 WRITE_ONCE(ssk->sk_err, EBADMSG); 1318 tcp_set_state(ssk, TCP_CLOSE); 1319 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1320 sk_eat_skb(ssk, skb); 1321 tcp_send_active_reset(ssk, GFP_ATOMIC); 1322 WRITE_ONCE(subflow->data_avail, false); 1323 return false; 1324 } 1325 1326 mptcp_do_fallback(ssk); 1327 } 1328 1329 skb = skb_peek(&ssk->sk_receive_queue); 1330 subflow->map_valid = 1; 1331 subflow->map_seq = READ_ONCE(msk->ack_seq); 1332 subflow->map_data_len = skb->len; 1333 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 1334 WRITE_ONCE(subflow->data_avail, true); 1335 return true; 1336 } 1337 1338 bool mptcp_subflow_data_available(struct sock *sk) 1339 { 1340 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1341 1342 /* check if current mapping is still valid */ 1343 if (subflow->map_valid && 1344 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 1345 subflow->map_valid = 0; 1346 WRITE_ONCE(subflow->data_avail, false); 1347 1348 pr_debug("Done with mapping: seq=%u data_len=%u", 1349 subflow->map_subflow_seq, 1350 subflow->map_data_len); 1351 } 1352 1353 return subflow_check_data_avail(sk); 1354 } 1355 1356 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy, 1357 * not the ssk one. 1358 * 1359 * In mptcp, rwin is about the mptcp-level connection data. 1360 * 1361 * Data that is still on the ssk rx queue can thus be ignored, 1362 * as far as mptcp peer is concerned that data is still inflight. 1363 * DSS ACK is updated when skb is moved to the mptcp rx queue. 1364 */ 1365 void mptcp_space(const struct sock *ssk, int *space, int *full_space) 1366 { 1367 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1368 const struct sock *sk = subflow->conn; 1369 1370 *space = __mptcp_space(sk); 1371 *full_space = mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 1372 } 1373 1374 static void subflow_error_report(struct sock *ssk) 1375 { 1376 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1377 1378 /* bail early if this is a no-op, so that we avoid introducing a 1379 * problematic lockdep dependency between TCP accept queue lock 1380 * and msk socket spinlock 1381 */ 1382 if (!sk->sk_socket) 1383 return; 1384 1385 mptcp_data_lock(sk); 1386 if (!sock_owned_by_user(sk)) 1387 __mptcp_error_report(sk); 1388 else 1389 __set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->cb_flags); 1390 mptcp_data_unlock(sk); 1391 } 1392 1393 static void subflow_data_ready(struct sock *sk) 1394 { 1395 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1396 u16 state = 1 << inet_sk_state_load(sk); 1397 struct sock *parent = subflow->conn; 1398 struct mptcp_sock *msk; 1399 1400 trace_sk_data_ready(sk); 1401 1402 msk = mptcp_sk(parent); 1403 if (state & TCPF_LISTEN) { 1404 /* MPJ subflow are removed from accept queue before reaching here, 1405 * avoid stray wakeups 1406 */ 1407 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue)) 1408 return; 1409 1410 parent->sk_data_ready(parent); 1411 return; 1412 } 1413 1414 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable && 1415 !subflow->mp_join && !(state & TCPF_CLOSE)); 1416 1417 if (mptcp_subflow_data_available(sk)) { 1418 mptcp_data_ready(parent, sk); 1419 1420 /* subflow-level lowat test are not relevant. 1421 * respect the msk-level threshold eventually mandating an immediate ack 1422 */ 1423 if (mptcp_data_avail(msk) < parent->sk_rcvlowat && 1424 (tcp_sk(sk)->rcv_nxt - tcp_sk(sk)->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss) 1425 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; 1426 } else if (unlikely(sk->sk_err)) { 1427 subflow_error_report(sk); 1428 } 1429 } 1430 1431 static void subflow_write_space(struct sock *ssk) 1432 { 1433 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1434 1435 mptcp_propagate_sndbuf(sk, ssk); 1436 mptcp_write_space(sk); 1437 } 1438 1439 static const struct inet_connection_sock_af_ops * 1440 subflow_default_af_ops(struct sock *sk) 1441 { 1442 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1443 if (sk->sk_family == AF_INET6) 1444 return &subflow_v6_specific; 1445 #endif 1446 return &subflow_specific; 1447 } 1448 1449 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1450 void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 1451 { 1452 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1453 struct inet_connection_sock *icsk = inet_csk(sk); 1454 const struct inet_connection_sock_af_ops *target; 1455 1456 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 1457 1458 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", 1459 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 1460 1461 if (likely(icsk->icsk_af_ops == target)) 1462 return; 1463 1464 subflow->icsk_af_ops = icsk->icsk_af_ops; 1465 icsk->icsk_af_ops = target; 1466 } 1467 #endif 1468 1469 void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 1470 struct sockaddr_storage *addr, 1471 unsigned short family) 1472 { 1473 memset(addr, 0, sizeof(*addr)); 1474 addr->ss_family = family; 1475 if (addr->ss_family == AF_INET) { 1476 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 1477 1478 if (info->family == AF_INET) 1479 in_addr->sin_addr = info->addr; 1480 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1481 else if (ipv6_addr_v4mapped(&info->addr6)) 1482 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3]; 1483 #endif 1484 in_addr->sin_port = info->port; 1485 } 1486 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1487 else if (addr->ss_family == AF_INET6) { 1488 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 1489 1490 if (info->family == AF_INET) 1491 ipv6_addr_set_v4mapped(info->addr.s_addr, 1492 &in6_addr->sin6_addr); 1493 else 1494 in6_addr->sin6_addr = info->addr6; 1495 in6_addr->sin6_port = info->port; 1496 } 1497 #endif 1498 } 1499 1500 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, 1501 const struct mptcp_addr_info *remote) 1502 { 1503 struct mptcp_sock *msk = mptcp_sk(sk); 1504 struct mptcp_subflow_context *subflow; 1505 struct sockaddr_storage addr; 1506 int remote_id = remote->id; 1507 int local_id = loc->id; 1508 int err = -ENOTCONN; 1509 struct socket *sf; 1510 struct sock *ssk; 1511 u32 remote_token; 1512 int addrlen; 1513 int ifindex; 1514 u8 flags; 1515 1516 if (!mptcp_is_fully_established(sk)) 1517 goto err_out; 1518 1519 err = mptcp_subflow_create_socket(sk, loc->family, &sf); 1520 if (err) 1521 goto err_out; 1522 1523 ssk = sf->sk; 1524 subflow = mptcp_subflow_ctx(ssk); 1525 do { 1526 get_random_bytes(&subflow->local_nonce, sizeof(u32)); 1527 } while (!subflow->local_nonce); 1528 1529 if (local_id) 1530 subflow_set_local_id(subflow, local_id); 1531 1532 mptcp_pm_get_flags_and_ifindex_by_id(msk, local_id, 1533 &flags, &ifindex); 1534 subflow->remote_key_valid = 1; 1535 subflow->remote_key = READ_ONCE(msk->remote_key); 1536 subflow->local_key = READ_ONCE(msk->local_key); 1537 subflow->token = msk->token; 1538 mptcp_info2sockaddr(loc, &addr, ssk->sk_family); 1539 1540 addrlen = sizeof(struct sockaddr_in); 1541 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1542 if (addr.ss_family == AF_INET6) 1543 addrlen = sizeof(struct sockaddr_in6); 1544 #endif 1545 ssk->sk_bound_dev_if = ifindex; 1546 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); 1547 if (err) 1548 goto failed; 1549 1550 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 1551 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d", msk, 1552 remote_token, local_id, remote_id); 1553 subflow->remote_token = remote_token; 1554 subflow->remote_id = remote_id; 1555 subflow->request_join = 1; 1556 subflow->request_bkup = !!(flags & MPTCP_PM_ADDR_FLAG_BACKUP); 1557 subflow->subflow_id = msk->subflow_id++; 1558 mptcp_info2sockaddr(remote, &addr, ssk->sk_family); 1559 1560 sock_hold(ssk); 1561 list_add_tail(&subflow->node, &msk->conn_list); 1562 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); 1563 if (err && err != -EINPROGRESS) 1564 goto failed_unlink; 1565 1566 /* discard the subflow socket */ 1567 mptcp_sock_graft(ssk, sk->sk_socket); 1568 iput(SOCK_INODE(sf)); 1569 WRITE_ONCE(msk->allow_infinite_fallback, false); 1570 mptcp_stop_tout_timer(sk); 1571 return 0; 1572 1573 failed_unlink: 1574 list_del(&subflow->node); 1575 sock_put(mptcp_subflow_tcp_sock(subflow)); 1576 1577 failed: 1578 subflow->disposable = 1; 1579 sock_release(sf); 1580 1581 err_out: 1582 /* we account subflows before the creation, and this failures will not 1583 * be caught by sk_state_change() 1584 */ 1585 mptcp_pm_close_subflow(msk); 1586 return err; 1587 } 1588 1589 static void mptcp_attach_cgroup(struct sock *parent, struct sock *child) 1590 { 1591 #ifdef CONFIG_SOCK_CGROUP_DATA 1592 struct sock_cgroup_data *parent_skcd = &parent->sk_cgrp_data, 1593 *child_skcd = &child->sk_cgrp_data; 1594 1595 /* only the additional subflows created by kworkers have to be modified */ 1596 if (cgroup_id(sock_cgroup_ptr(parent_skcd)) != 1597 cgroup_id(sock_cgroup_ptr(child_skcd))) { 1598 #ifdef CONFIG_MEMCG 1599 struct mem_cgroup *memcg = parent->sk_memcg; 1600 1601 mem_cgroup_sk_free(child); 1602 if (memcg && css_tryget(&memcg->css)) 1603 child->sk_memcg = memcg; 1604 #endif /* CONFIG_MEMCG */ 1605 1606 cgroup_sk_free(child_skcd); 1607 *child_skcd = *parent_skcd; 1608 cgroup_sk_clone(child_skcd); 1609 } 1610 #endif /* CONFIG_SOCK_CGROUP_DATA */ 1611 } 1612 1613 static void mptcp_subflow_ops_override(struct sock *ssk) 1614 { 1615 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1616 if (ssk->sk_prot == &tcpv6_prot) 1617 ssk->sk_prot = &tcpv6_prot_override; 1618 else 1619 #endif 1620 ssk->sk_prot = &tcp_prot_override; 1621 } 1622 1623 static void mptcp_subflow_ops_undo_override(struct sock *ssk) 1624 { 1625 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1626 if (ssk->sk_prot == &tcpv6_prot_override) 1627 ssk->sk_prot = &tcpv6_prot; 1628 else 1629 #endif 1630 ssk->sk_prot = &tcp_prot; 1631 } 1632 1633 int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, 1634 struct socket **new_sock) 1635 { 1636 struct mptcp_subflow_context *subflow; 1637 struct net *net = sock_net(sk); 1638 struct socket *sf; 1639 int err; 1640 1641 /* un-accepted server sockets can reach here - on bad configuration 1642 * bail early to avoid greater trouble later 1643 */ 1644 if (unlikely(!sk->sk_socket)) 1645 return -EINVAL; 1646 1647 err = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, &sf); 1648 if (err) 1649 return err; 1650 1651 lock_sock_nested(sf->sk, SINGLE_DEPTH_NESTING); 1652 1653 err = security_mptcp_add_subflow(sk, sf->sk); 1654 if (err) 1655 goto err_free; 1656 1657 /* the newly created socket has to be in the same cgroup as its parent */ 1658 mptcp_attach_cgroup(sk, sf->sk); 1659 1660 /* kernel sockets do not by default acquire net ref, but TCP timer 1661 * needs it. 1662 * Update ns_tracker to current stack trace and refcounted tracker. 1663 */ 1664 __netns_tracker_free(net, &sf->sk->ns_tracker, false); 1665 sf->sk->sk_net_refcnt = 1; 1666 get_net_track(net, &sf->sk->ns_tracker, GFP_KERNEL); 1667 sock_inuse_add(net, 1); 1668 err = tcp_set_ulp(sf->sk, "mptcp"); 1669 if (err) 1670 goto err_free; 1671 1672 mptcp_sockopt_sync_locked(mptcp_sk(sk), sf->sk); 1673 release_sock(sf->sk); 1674 1675 /* the newly created socket really belongs to the owning MPTCP master 1676 * socket, even if for additional subflows the allocation is performed 1677 * by a kernel workqueue. Adjust inode references, so that the 1678 * procfs/diag interfaces really show this one belonging to the correct 1679 * user. 1680 */ 1681 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; 1682 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; 1683 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; 1684 1685 subflow = mptcp_subflow_ctx(sf->sk); 1686 pr_debug("subflow=%p", subflow); 1687 1688 *new_sock = sf; 1689 sock_hold(sk); 1690 subflow->conn = sk; 1691 mptcp_subflow_ops_override(sf->sk); 1692 1693 return 0; 1694 1695 err_free: 1696 release_sock(sf->sk); 1697 sock_release(sf); 1698 return err; 1699 } 1700 1701 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1702 gfp_t priority) 1703 { 1704 struct inet_connection_sock *icsk = inet_csk(sk); 1705 struct mptcp_subflow_context *ctx; 1706 1707 ctx = kzalloc(sizeof(*ctx), priority); 1708 if (!ctx) 1709 return NULL; 1710 1711 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1712 INIT_LIST_HEAD(&ctx->node); 1713 INIT_LIST_HEAD(&ctx->delegated_node); 1714 1715 pr_debug("subflow=%p", ctx); 1716 1717 ctx->tcp_sock = sk; 1718 1719 return ctx; 1720 } 1721 1722 static void __subflow_state_change(struct sock *sk) 1723 { 1724 struct socket_wq *wq; 1725 1726 rcu_read_lock(); 1727 wq = rcu_dereference(sk->sk_wq); 1728 if (skwq_has_sleeper(wq)) 1729 wake_up_interruptible_all(&wq->wait); 1730 rcu_read_unlock(); 1731 } 1732 1733 static bool subflow_is_done(const struct sock *sk) 1734 { 1735 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1736 } 1737 1738 static void subflow_state_change(struct sock *sk) 1739 { 1740 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1741 struct sock *parent = subflow->conn; 1742 struct mptcp_sock *msk; 1743 1744 __subflow_state_change(sk); 1745 1746 msk = mptcp_sk(parent); 1747 if (subflow_simultaneous_connect(sk)) { 1748 mptcp_do_fallback(sk); 1749 mptcp_rcv_space_init(msk, sk); 1750 pr_fallback(msk); 1751 subflow->conn_finished = 1; 1752 mptcp_propagate_state(parent, sk); 1753 } 1754 1755 /* as recvmsg() does not acquire the subflow socket for ssk selection 1756 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1757 * the data available machinery here. 1758 */ 1759 if (mptcp_subflow_data_available(sk)) 1760 mptcp_data_ready(parent, sk); 1761 else if (unlikely(sk->sk_err)) 1762 subflow_error_report(sk); 1763 1764 subflow_sched_work_if_closed(mptcp_sk(parent), sk); 1765 1766 /* when the fallback subflow closes the rx side, trigger a 'dummy' 1767 * ingress data fin, so that the msk state will follow along 1768 */ 1769 if (__mptcp_check_fallback(msk) && subflow_is_done(sk) && msk->first == sk && 1770 mptcp_update_rcv_data_fin(msk, READ_ONCE(msk->ack_seq), true)) 1771 mptcp_schedule_work(parent); 1772 } 1773 1774 void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk) 1775 { 1776 struct request_sock_queue *queue = &inet_csk(listener_ssk)->icsk_accept_queue; 1777 struct request_sock *req, *head, *tail; 1778 struct mptcp_subflow_context *subflow; 1779 struct sock *sk, *ssk; 1780 1781 /* Due to lock dependencies no relevant lock can be acquired under rskq_lock. 1782 * Splice the req list, so that accept() can not reach the pending ssk after 1783 * the listener socket is released below. 1784 */ 1785 spin_lock_bh(&queue->rskq_lock); 1786 head = queue->rskq_accept_head; 1787 tail = queue->rskq_accept_tail; 1788 queue->rskq_accept_head = NULL; 1789 queue->rskq_accept_tail = NULL; 1790 spin_unlock_bh(&queue->rskq_lock); 1791 1792 if (!head) 1793 return; 1794 1795 /* can't acquire the msk socket lock under the subflow one, 1796 * or will cause ABBA deadlock 1797 */ 1798 release_sock(listener_ssk); 1799 1800 for (req = head; req; req = req->dl_next) { 1801 ssk = req->sk; 1802 if (!sk_is_mptcp(ssk)) 1803 continue; 1804 1805 subflow = mptcp_subflow_ctx(ssk); 1806 if (!subflow || !subflow->conn) 1807 continue; 1808 1809 sk = subflow->conn; 1810 sock_hold(sk); 1811 1812 lock_sock_nested(sk, SINGLE_DEPTH_NESTING); 1813 __mptcp_unaccepted_force_close(sk); 1814 release_sock(sk); 1815 1816 /* lockdep will report a false positive ABBA deadlock 1817 * between cancel_work_sync and the listener socket. 1818 * The involved locks belong to different sockets WRT 1819 * the existing AB chain. 1820 * Using a per socket key is problematic as key 1821 * deregistration requires process context and must be 1822 * performed at socket disposal time, in atomic 1823 * context. 1824 * Just tell lockdep to consider the listener socket 1825 * released here. 1826 */ 1827 mutex_release(&listener_sk->sk_lock.dep_map, _RET_IP_); 1828 mptcp_cancel_work(sk); 1829 mutex_acquire(&listener_sk->sk_lock.dep_map, 0, 0, _RET_IP_); 1830 1831 sock_put(sk); 1832 } 1833 1834 /* we are still under the listener msk socket lock */ 1835 lock_sock_nested(listener_ssk, SINGLE_DEPTH_NESTING); 1836 1837 /* restore the listener queue, to let the TCP code clean it up */ 1838 spin_lock_bh(&queue->rskq_lock); 1839 WARN_ON_ONCE(queue->rskq_accept_head); 1840 queue->rskq_accept_head = head; 1841 queue->rskq_accept_tail = tail; 1842 spin_unlock_bh(&queue->rskq_lock); 1843 } 1844 1845 static int subflow_ulp_init(struct sock *sk) 1846 { 1847 struct inet_connection_sock *icsk = inet_csk(sk); 1848 struct mptcp_subflow_context *ctx; 1849 struct tcp_sock *tp = tcp_sk(sk); 1850 int err = 0; 1851 1852 /* disallow attaching ULP to a socket unless it has been 1853 * created with sock_create_kern() 1854 */ 1855 if (!sk->sk_kern_sock) { 1856 err = -EOPNOTSUPP; 1857 goto out; 1858 } 1859 1860 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1861 if (!ctx) { 1862 err = -ENOMEM; 1863 goto out; 1864 } 1865 1866 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); 1867 1868 tp->is_mptcp = 1; 1869 ctx->icsk_af_ops = icsk->icsk_af_ops; 1870 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1871 ctx->tcp_state_change = sk->sk_state_change; 1872 ctx->tcp_error_report = sk->sk_error_report; 1873 1874 WARN_ON_ONCE(sk->sk_data_ready != sock_def_readable); 1875 WARN_ON_ONCE(sk->sk_write_space != sk_stream_write_space); 1876 1877 sk->sk_data_ready = subflow_data_ready; 1878 sk->sk_write_space = subflow_write_space; 1879 sk->sk_state_change = subflow_state_change; 1880 sk->sk_error_report = subflow_error_report; 1881 out: 1882 return err; 1883 } 1884 1885 static void subflow_ulp_release(struct sock *ssk) 1886 { 1887 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 1888 bool release = true; 1889 struct sock *sk; 1890 1891 if (!ctx) 1892 return; 1893 1894 sk = ctx->conn; 1895 if (sk) { 1896 /* if the msk has been orphaned, keep the ctx 1897 * alive, will be freed by __mptcp_close_ssk(), 1898 * when the subflow is still unaccepted 1899 */ 1900 release = ctx->disposable || list_empty(&ctx->node); 1901 1902 /* inet_child_forget() does not call sk_state_change(), 1903 * explicitly trigger the socket close machinery 1904 */ 1905 if (!release && !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, 1906 &mptcp_sk(sk)->flags)) 1907 mptcp_schedule_work(sk); 1908 sock_put(sk); 1909 } 1910 1911 mptcp_subflow_ops_undo_override(ssk); 1912 if (release) 1913 kfree_rcu(ctx, rcu); 1914 } 1915 1916 static void subflow_ulp_clone(const struct request_sock *req, 1917 struct sock *newsk, 1918 const gfp_t priority) 1919 { 1920 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1921 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 1922 struct mptcp_subflow_context *new_ctx; 1923 1924 if (!tcp_rsk(req)->is_mptcp || 1925 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 1926 subflow_ulp_fallback(newsk, old_ctx); 1927 return; 1928 } 1929 1930 new_ctx = subflow_create_ctx(newsk, priority); 1931 if (!new_ctx) { 1932 subflow_ulp_fallback(newsk, old_ctx); 1933 return; 1934 } 1935 1936 new_ctx->conn_finished = 1; 1937 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 1938 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 1939 new_ctx->tcp_error_report = old_ctx->tcp_error_report; 1940 new_ctx->rel_write_seq = 1; 1941 new_ctx->tcp_sock = newsk; 1942 1943 if (subflow_req->mp_capable) { 1944 /* see comments in subflow_syn_recv_sock(), MPTCP connection 1945 * is fully established only after we receive the remote key 1946 */ 1947 new_ctx->mp_capable = 1; 1948 new_ctx->local_key = subflow_req->local_key; 1949 new_ctx->token = subflow_req->token; 1950 new_ctx->ssn_offset = subflow_req->ssn_offset; 1951 new_ctx->idsn = subflow_req->idsn; 1952 1953 /* this is the first subflow, id is always 0 */ 1954 new_ctx->local_id_valid = 1; 1955 } else if (subflow_req->mp_join) { 1956 new_ctx->ssn_offset = subflow_req->ssn_offset; 1957 new_ctx->mp_join = 1; 1958 new_ctx->fully_established = 1; 1959 new_ctx->remote_key_valid = 1; 1960 new_ctx->backup = subflow_req->backup; 1961 new_ctx->remote_id = subflow_req->remote_id; 1962 new_ctx->token = subflow_req->token; 1963 new_ctx->thmac = subflow_req->thmac; 1964 1965 /* the subflow req id is valid, fetched via subflow_check_req() 1966 * and subflow_token_join_request() 1967 */ 1968 subflow_set_local_id(new_ctx, subflow_req->local_id); 1969 } 1970 } 1971 1972 static void tcp_release_cb_override(struct sock *ssk) 1973 { 1974 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1975 long status; 1976 1977 /* process and clear all the pending actions, but leave the subflow into 1978 * the napi queue. To respect locking, only the same CPU that originated 1979 * the action can touch the list. mptcp_napi_poll will take care of it. 1980 */ 1981 status = set_mask_bits(&subflow->delegated_status, MPTCP_DELEGATE_ACTIONS_MASK, 0); 1982 if (status) 1983 mptcp_subflow_process_delegated(ssk, status); 1984 1985 tcp_release_cb(ssk); 1986 } 1987 1988 static int tcp_abort_override(struct sock *ssk, int err) 1989 { 1990 /* closing a listener subflow requires a great deal of care. 1991 * keep it simple and just prevent such operation 1992 */ 1993 if (inet_sk_state_load(ssk) == TCP_LISTEN) 1994 return -EINVAL; 1995 1996 return tcp_abort(ssk, err); 1997 } 1998 1999 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 2000 .name = "mptcp", 2001 .owner = THIS_MODULE, 2002 .init = subflow_ulp_init, 2003 .release = subflow_ulp_release, 2004 .clone = subflow_ulp_clone, 2005 }; 2006 2007 static int subflow_ops_init(struct request_sock_ops *subflow_ops) 2008 { 2009 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 2010 2011 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 2012 subflow_ops->obj_size, 0, 2013 SLAB_ACCOUNT | 2014 SLAB_TYPESAFE_BY_RCU, 2015 NULL); 2016 if (!subflow_ops->slab) 2017 return -ENOMEM; 2018 2019 return 0; 2020 } 2021 2022 void __init mptcp_subflow_init(void) 2023 { 2024 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops; 2025 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4"; 2026 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor; 2027 2028 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0) 2029 panic("MPTCP: failed to init subflow v4 request sock ops\n"); 2030 2031 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 2032 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req; 2033 subflow_request_sock_ipv4_ops.send_synack = subflow_v4_send_synack; 2034 2035 subflow_specific = ipv4_specific; 2036 subflow_specific.conn_request = subflow_v4_conn_request; 2037 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 2038 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 2039 subflow_specific.rebuild_header = subflow_rebuild_header; 2040 2041 tcp_prot_override = tcp_prot; 2042 tcp_prot_override.release_cb = tcp_release_cb_override; 2043 tcp_prot_override.diag_destroy = tcp_abort_override; 2044 2045 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2046 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock 2047 * structures for v4 and v6 have the same size. It should not changed in 2048 * the future but better to make sure to be warned if it is no longer 2049 * the case. 2050 */ 2051 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock)); 2052 2053 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops; 2054 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6"; 2055 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor; 2056 2057 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0) 2058 panic("MPTCP: failed to init subflow v6 request sock ops\n"); 2059 2060 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 2061 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req; 2062 subflow_request_sock_ipv6_ops.send_synack = subflow_v6_send_synack; 2063 2064 subflow_v6_specific = ipv6_specific; 2065 subflow_v6_specific.conn_request = subflow_v6_conn_request; 2066 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 2067 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 2068 subflow_v6_specific.rebuild_header = subflow_v6_rebuild_header; 2069 2070 subflow_v6m_specific = subflow_v6_specific; 2071 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 2072 subflow_v6m_specific.send_check = ipv4_specific.send_check; 2073 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 2074 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 2075 subflow_v6m_specific.rebuild_header = subflow_rebuild_header; 2076 2077 tcpv6_prot_override = tcpv6_prot; 2078 tcpv6_prot_override.release_cb = tcp_release_cb_override; 2079 tcpv6_prot_override.diag_destroy = tcp_abort_override; 2080 #endif 2081 2082 mptcp_diag_subflow_init(&subflow_ulp_ops); 2083 2084 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 2085 panic("MPTCP: failed to register subflows to ULP\n"); 2086 } 2087