1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <crypto/sha2.h> 13 #include <crypto/utils.h> 14 #include <net/sock.h> 15 #include <net/inet_common.h> 16 #include <net/inet_hashtables.h> 17 #include <net/protocol.h> 18 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 19 #include <net/ip6_route.h> 20 #include <net/transp_v6.h> 21 #endif 22 #include <net/mptcp.h> 23 24 #include "protocol.h" 25 #include "mib.h" 26 27 #include <trace/events/mptcp.h> 28 #include <trace/events/sock.h> 29 30 static void mptcp_subflow_ops_undo_override(struct sock *ssk); 31 32 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 33 enum linux_mptcp_mib_field field) 34 { 35 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 36 } 37 38 static void subflow_req_destructor(struct request_sock *req) 39 { 40 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 41 42 pr_debug("subflow_req=%p\n", subflow_req); 43 44 if (subflow_req->msk) 45 sock_put((struct sock *)subflow_req->msk); 46 47 mptcp_token_destroy_request(req); 48 } 49 50 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 51 void *hmac) 52 { 53 u8 msg[8]; 54 55 put_unaligned_be32(nonce1, &msg[0]); 56 put_unaligned_be32(nonce2, &msg[4]); 57 58 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 59 } 60 61 static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk) 62 { 63 return mptcp_is_fully_established((void *)msk) && 64 ((mptcp_pm_is_userspace(msk) && 65 mptcp_userspace_pm_active(msk)) || 66 READ_ONCE(msk->pm.accept_subflow)); 67 } 68 69 /* validate received token and create truncated hmac and nonce for SYN-ACK */ 70 static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req) 71 { 72 struct mptcp_sock *msk = subflow_req->msk; 73 u8 hmac[SHA256_DIGEST_SIZE]; 74 75 subflow_req->local_nonce = get_random_u32(); 76 77 subflow_generate_hmac(READ_ONCE(msk->local_key), 78 READ_ONCE(msk->remote_key), 79 subflow_req->local_nonce, 80 subflow_req->remote_nonce, hmac); 81 82 subflow_req->thmac = get_unaligned_be64(hmac); 83 } 84 85 static struct mptcp_sock *subflow_token_join_request(struct request_sock *req) 86 { 87 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 88 struct mptcp_sock *msk; 89 int local_id; 90 91 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token); 92 if (!msk) { 93 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 94 return NULL; 95 } 96 97 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 98 if (local_id < 0) { 99 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPJOINNOIDFOUND); 100 sock_put((struct sock *)msk); 101 return NULL; 102 } 103 subflow_req->local_id = local_id; 104 subflow_req->request_bkup = mptcp_pm_is_backup(msk, (struct sock_common *)req); 105 106 return msk; 107 } 108 109 static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener) 110 { 111 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 112 113 subflow_req->mp_capable = 0; 114 subflow_req->mp_join = 0; 115 subflow_req->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk_listener)); 116 subflow_req->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk_listener)); 117 subflow_req->msk = NULL; 118 mptcp_token_init_request(req); 119 } 120 121 static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk) 122 { 123 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport; 124 } 125 126 static void subflow_add_reset_reason(struct sk_buff *skb, u8 reason) 127 { 128 struct mptcp_ext *mpext = skb_ext_add(skb, SKB_EXT_MPTCP); 129 130 if (mpext) { 131 memset(mpext, 0, sizeof(*mpext)); 132 mpext->reset_reason = reason; 133 } 134 } 135 136 static int subflow_reset_req_endp(struct request_sock *req, struct sk_buff *skb) 137 { 138 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEENDPATTEMPT); 139 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 140 return -EPERM; 141 } 142 143 /* Init mptcp request socket. 144 * 145 * Returns an error code if a JOIN has failed and a TCP reset 146 * should be sent. 147 */ 148 static int subflow_check_req(struct request_sock *req, 149 const struct sock *sk_listener, 150 struct sk_buff *skb) 151 { 152 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 153 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 154 struct mptcp_options_received mp_opt; 155 bool opt_mp_capable, opt_mp_join; 156 157 pr_debug("subflow_req=%p, listener=%p\n", subflow_req, listener); 158 159 #ifdef CONFIG_TCP_MD5SIG 160 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 161 * TCP option space. 162 */ 163 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) { 164 MPTCP_INC_STATS(sock_net(sk_listener), MPTCP_MIB_MD5SIGRESET); 165 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 166 return -EINVAL; 167 } 168 #endif 169 170 mptcp_get_options(skb, &mp_opt); 171 172 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYN); 173 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYN); 174 if (opt_mp_capable) { 175 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 176 177 if (unlikely(listener->pm_listener)) 178 return subflow_reset_req_endp(req, skb); 179 } else if (opt_mp_join) { 180 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 181 182 if (mp_opt.backup) 183 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNBACKUPRX); 184 } else if (unlikely(listener->pm_listener)) { 185 return subflow_reset_req_endp(req, skb); 186 } 187 188 if (opt_mp_capable && listener->request_mptcp) { 189 int err, retries = MPTCP_TOKEN_MAX_RETRIES; 190 191 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 192 again: 193 do { 194 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key)); 195 } while (subflow_req->local_key == 0); 196 197 if (unlikely(req->syncookie)) { 198 mptcp_crypto_key_sha(subflow_req->local_key, 199 &subflow_req->token, 200 &subflow_req->idsn); 201 if (mptcp_token_exists(subflow_req->token)) { 202 if (retries-- > 0) 203 goto again; 204 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 205 } else { 206 subflow_req->mp_capable = 1; 207 } 208 return 0; 209 } 210 211 err = mptcp_token_new_request(req); 212 if (err == 0) 213 subflow_req->mp_capable = 1; 214 else if (retries-- > 0) 215 goto again; 216 else 217 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT); 218 219 } else if (opt_mp_join && listener->request_mptcp) { 220 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 221 subflow_req->mp_join = 1; 222 subflow_req->backup = mp_opt.backup; 223 subflow_req->remote_id = mp_opt.join_id; 224 subflow_req->token = mp_opt.token; 225 subflow_req->remote_nonce = mp_opt.nonce; 226 subflow_req->msk = subflow_token_join_request(req); 227 228 /* Can't fall back to TCP in this case. */ 229 if (!subflow_req->msk) { 230 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 231 return -EPERM; 232 } 233 234 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) { 235 pr_debug("syn inet_sport=%d %d\n", 236 ntohs(inet_sk(sk_listener)->inet_sport), 237 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport)); 238 if (!mptcp_pm_announced_has_ssk(subflow_req->msk, sk_listener)) { 239 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX); 240 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 241 return -EPERM; 242 } 243 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX); 244 } 245 246 subflow_req_create_thmac(subflow_req); 247 248 if (unlikely(req->syncookie)) { 249 if (!mptcp_can_accept_new_subflow(subflow_req->msk)) { 250 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED); 251 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 252 return -EPERM; 253 } 254 255 subflow_init_req_cookie_join_save(subflow_req, skb); 256 } 257 258 pr_debug("token=%u, remote_nonce=%u msk=%p\n", subflow_req->token, 259 subflow_req->remote_nonce, subflow_req->msk); 260 } 261 262 return 0; 263 } 264 265 int mptcp_subflow_init_cookie_req(struct request_sock *req, 266 const struct sock *sk_listener, 267 struct sk_buff *skb) 268 { 269 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 270 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 271 struct mptcp_options_received mp_opt; 272 bool opt_mp_capable, opt_mp_join; 273 int err; 274 275 subflow_init_req(req, sk_listener); 276 mptcp_get_options(skb, &mp_opt); 277 278 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_ACK); 279 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK); 280 if (opt_mp_capable && listener->request_mptcp) { 281 if (mp_opt.sndr_key == 0) 282 return -EINVAL; 283 284 subflow_req->local_key = mp_opt.rcvr_key; 285 err = mptcp_token_new_request(req); 286 if (err) 287 return err; 288 289 subflow_req->mp_capable = 1; 290 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 291 } else if (opt_mp_join && listener->request_mptcp) { 292 if (!mptcp_token_join_cookie_init_state(subflow_req, skb)) 293 return -EINVAL; 294 295 subflow_req->mp_join = 1; 296 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1; 297 } 298 299 return 0; 300 } 301 EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req); 302 303 static enum sk_rst_reason mptcp_get_rst_reason(const struct sk_buff *skb) 304 { 305 const struct mptcp_ext *mpext = mptcp_get_ext(skb); 306 307 if (!mpext) 308 return SK_RST_REASON_NOT_SPECIFIED; 309 310 return sk_rst_convert_mptcp_reason(mpext->reset_reason); 311 } 312 313 static struct dst_entry *subflow_v4_route_req(const struct sock *sk, 314 struct sk_buff *skb, 315 struct flowi *fl, 316 struct request_sock *req, 317 u32 tw_isn) 318 { 319 struct dst_entry *dst; 320 int err; 321 322 tcp_rsk(req)->is_mptcp = 1; 323 subflow_init_req(req, sk); 324 325 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req, tw_isn); 326 if (!dst) 327 return NULL; 328 329 err = subflow_check_req(req, sk, skb); 330 if (err == 0) 331 return dst; 332 333 dst_release(dst); 334 if (!req->syncookie) 335 tcp_request_sock_ops.send_reset(sk, skb, 336 mptcp_get_rst_reason(skb)); 337 return NULL; 338 } 339 340 static void subflow_prep_synack(const struct sock *sk, struct request_sock *req, 341 struct tcp_fastopen_cookie *foc, 342 enum tcp_synack_type synack_type) 343 { 344 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 345 struct inet_request_sock *ireq = inet_rsk(req); 346 347 /* clear tstamp_ok, as needed depending on cookie */ 348 if (foc && foc->len > -1) 349 ireq->tstamp_ok = 0; 350 351 if (synack_type == TCP_SYNACK_FASTOPEN) 352 mptcp_fastopen_subflow_synack_set_params(subflow, req); 353 } 354 355 static int subflow_v4_send_synack(const struct sock *sk, struct dst_entry *dst, 356 struct flowi *fl, 357 struct request_sock *req, 358 struct tcp_fastopen_cookie *foc, 359 enum tcp_synack_type synack_type, 360 struct sk_buff *syn_skb) 361 { 362 subflow_prep_synack(sk, req, foc, synack_type); 363 364 return tcp_request_sock_ipv4_ops.send_synack(sk, dst, fl, req, foc, 365 synack_type, syn_skb); 366 } 367 368 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 369 static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst, 370 struct flowi *fl, 371 struct request_sock *req, 372 struct tcp_fastopen_cookie *foc, 373 enum tcp_synack_type synack_type, 374 struct sk_buff *syn_skb) 375 { 376 subflow_prep_synack(sk, req, foc, synack_type); 377 378 return tcp_request_sock_ipv6_ops.send_synack(sk, dst, fl, req, foc, 379 synack_type, syn_skb); 380 } 381 382 static struct dst_entry *subflow_v6_route_req(const struct sock *sk, 383 struct sk_buff *skb, 384 struct flowi *fl, 385 struct request_sock *req, 386 u32 tw_isn) 387 { 388 struct dst_entry *dst; 389 int err; 390 391 tcp_rsk(req)->is_mptcp = 1; 392 subflow_init_req(req, sk); 393 394 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req, tw_isn); 395 if (!dst) 396 return NULL; 397 398 err = subflow_check_req(req, sk, skb); 399 if (err == 0) 400 return dst; 401 402 dst_release(dst); 403 if (!req->syncookie) 404 tcp6_request_sock_ops.send_reset(sk, skb, 405 mptcp_get_rst_reason(skb)); 406 return NULL; 407 } 408 #endif 409 410 /* validate received truncated hmac and create hmac for third ACK */ 411 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 412 { 413 u8 hmac[SHA256_DIGEST_SIZE]; 414 u64 thmac; 415 416 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 417 subflow->remote_nonce, subflow->local_nonce, 418 hmac); 419 420 thmac = get_unaligned_be64(hmac); 421 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 422 subflow, subflow->token, thmac, subflow->thmac); 423 424 return thmac == subflow->thmac; 425 } 426 427 void mptcp_subflow_reset(struct sock *ssk) 428 { 429 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 430 struct sock *sk = subflow->conn; 431 432 /* mptcp_mp_fail_no_response() can reach here on an already closed 433 * socket 434 */ 435 if (ssk->sk_state == TCP_CLOSE) 436 return; 437 438 /* must hold: tcp_done() could drop last reference on parent */ 439 sock_hold(sk); 440 441 mptcp_send_active_reset_reason(ssk); 442 tcp_done(ssk); 443 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags)) 444 mptcp_schedule_work(sk); 445 446 sock_put(sk); 447 } 448 449 static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk) 450 { 451 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport; 452 } 453 454 void __mptcp_sync_state(struct sock *sk, int state) 455 { 456 struct mptcp_subflow_context *subflow; 457 struct mptcp_sock *msk = mptcp_sk(sk); 458 struct sock *ssk = msk->first; 459 460 subflow = mptcp_subflow_ctx(ssk); 461 __mptcp_propagate_sndbuf(sk, ssk); 462 463 if (sk->sk_state == TCP_SYN_SENT) { 464 /* subflow->idsn is always available is TCP_SYN_SENT state, 465 * even for the FASTOPEN scenarios 466 */ 467 WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 468 WRITE_ONCE(msk->snd_nxt, msk->write_seq); 469 mptcp_set_state(sk, state); 470 sk->sk_state_change(sk); 471 } 472 } 473 474 static void subflow_set_remote_key(struct mptcp_sock *msk, 475 struct mptcp_subflow_context *subflow, 476 const struct mptcp_options_received *mp_opt) 477 { 478 /* active MPC subflow will reach here multiple times: 479 * at subflow_finish_connect() time and at 4th ack time 480 */ 481 if (subflow->remote_key_valid) 482 return; 483 484 subflow->remote_key_valid = 1; 485 subflow->remote_key = mp_opt->sndr_key; 486 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn); 487 subflow->iasn++; 488 489 /* for fallback's sake */ 490 subflow->map_seq = subflow->iasn; 491 492 WRITE_ONCE(msk->remote_key, subflow->remote_key); 493 WRITE_ONCE(msk->ack_seq, subflow->iasn); 494 WRITE_ONCE(msk->can_ack, true); 495 atomic64_set(&msk->rcv_wnd_sent, subflow->iasn); 496 } 497 498 static void mptcp_propagate_state(struct sock *sk, struct sock *ssk, 499 struct mptcp_subflow_context *subflow, 500 const struct mptcp_options_received *mp_opt) 501 { 502 struct mptcp_sock *msk = mptcp_sk(sk); 503 504 mptcp_data_lock(sk); 505 if (mp_opt) { 506 /* Options are available only in the non fallback cases 507 * avoid updating rx path fields otherwise 508 */ 509 WRITE_ONCE(msk->snd_una, subflow->idsn + 1); 510 WRITE_ONCE(msk->wnd_end, subflow->idsn + 1 + tcp_sk(ssk)->snd_wnd); 511 subflow_set_remote_key(msk, subflow, mp_opt); 512 } 513 514 if (!sock_owned_by_user(sk)) { 515 __mptcp_sync_state(sk, ssk->sk_state); 516 } else { 517 msk->pending_state = ssk->sk_state; 518 __set_bit(MPTCP_SYNC_STATE, &msk->cb_flags); 519 } 520 mptcp_data_unlock(sk); 521 } 522 523 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 524 { 525 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 526 struct mptcp_options_received mp_opt; 527 struct sock *parent = subflow->conn; 528 struct mptcp_sock *msk; 529 530 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 531 532 /* be sure no special action on any packet other than syn-ack */ 533 if (subflow->conn_finished) 534 return; 535 536 msk = mptcp_sk(parent); 537 subflow->rel_write_seq = 1; 538 subflow->conn_finished = 1; 539 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 540 pr_debug("subflow=%p synack seq=%x\n", subflow, subflow->ssn_offset); 541 542 mptcp_get_options(skb, &mp_opt); 543 if (subflow->request_mptcp) { 544 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) { 545 if (!mptcp_try_fallback(sk, 546 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK)) { 547 MPTCP_INC_STATS(sock_net(sk), 548 MPTCP_MIB_FALLBACKFAILED); 549 goto do_reset; 550 } 551 552 goto fallback; 553 } 554 555 if (mp_opt.suboptions & OPTION_MPTCP_CSUMREQD) 556 WRITE_ONCE(msk->csum_enabled, true); 557 if (mp_opt.deny_join_id0) 558 WRITE_ONCE(msk->pm.remote_deny_join_id0, true); 559 subflow->mp_capable = 1; 560 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK); 561 mptcp_finish_connect(sk); 562 mptcp_active_enable(parent); 563 mptcp_propagate_state(parent, sk, subflow, &mp_opt); 564 } else if (subflow->request_join) { 565 u8 hmac[SHA256_DIGEST_SIZE]; 566 567 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYNACK)) { 568 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPJOINSYNACKNOMPJOIN); 569 subflow->reset_reason = MPTCP_RST_EMPTCP; 570 goto do_reset; 571 } 572 573 subflow->backup = mp_opt.backup; 574 subflow->thmac = mp_opt.thmac; 575 subflow->remote_nonce = mp_opt.nonce; 576 WRITE_ONCE(subflow->remote_id, mp_opt.join_id); 577 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d\n", 578 subflow, subflow->thmac, subflow->remote_nonce, 579 subflow->backup); 580 581 if (!subflow_thmac_valid(subflow)) { 582 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKMAC); 583 subflow->reset_reason = MPTCP_RST_EMPTCP; 584 goto do_reset; 585 } 586 587 if (!mptcp_finish_join(sk)) 588 goto do_reset; 589 590 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 591 subflow->local_nonce, 592 subflow->remote_nonce, 593 hmac); 594 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN); 595 596 subflow->mp_join = 1; 597 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 598 599 if (subflow->backup) 600 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKBACKUPRX); 601 602 if (subflow_use_different_dport(msk, sk)) { 603 pr_debug("synack inet_dport=%d %d\n", 604 ntohs(inet_sk(sk)->inet_dport), 605 ntohs(inet_sk(parent)->inet_dport)); 606 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX); 607 } 608 } else if (mptcp_check_fallback(sk)) { 609 /* It looks like MPTCP is blocked, while TCP is not */ 610 if (subflow->mpc_drop) 611 mptcp_active_disable(parent); 612 fallback: 613 mptcp_propagate_state(parent, sk, subflow, NULL); 614 } 615 return; 616 617 do_reset: 618 subflow->reset_transient = 0; 619 mptcp_subflow_reset(sk); 620 } 621 622 static void subflow_set_local_id(struct mptcp_subflow_context *subflow, int local_id) 623 { 624 WARN_ON_ONCE(local_id < 0 || local_id > 255); 625 WRITE_ONCE(subflow->local_id, local_id); 626 } 627 628 static int subflow_chk_local_id(struct sock *sk) 629 { 630 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 631 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 632 int err; 633 634 if (likely(subflow->local_id >= 0)) 635 return 0; 636 637 err = mptcp_pm_get_local_id(msk, (struct sock_common *)sk); 638 if (err < 0) 639 return err; 640 641 subflow_set_local_id(subflow, err); 642 subflow->request_bkup = mptcp_pm_is_backup(msk, (struct sock_common *)sk); 643 644 return 0; 645 } 646 647 static int subflow_rebuild_header(struct sock *sk) 648 { 649 int err = subflow_chk_local_id(sk); 650 651 if (unlikely(err < 0)) 652 return err; 653 654 return inet_sk_rebuild_header(sk); 655 } 656 657 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 658 static int subflow_v6_rebuild_header(struct sock *sk) 659 { 660 int err = subflow_chk_local_id(sk); 661 662 if (unlikely(err < 0)) 663 return err; 664 665 return inet6_sk_rebuild_header(sk); 666 } 667 #endif 668 669 static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init; 670 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init; 671 672 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 673 { 674 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 675 676 pr_debug("subflow=%p\n", subflow); 677 678 /* Never answer to SYNs sent to broadcast or multicast */ 679 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 680 goto drop; 681 682 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops, 683 &subflow_request_sock_ipv4_ops, 684 sk, skb); 685 drop: 686 tcp_listendrop(sk); 687 return 0; 688 } 689 690 static void subflow_v4_req_destructor(struct request_sock *req) 691 { 692 subflow_req_destructor(req); 693 tcp_request_sock_ops.destructor(req); 694 } 695 696 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 697 static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init; 698 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init; 699 static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init; 700 static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init; 701 static struct proto tcpv6_prot_override __ro_after_init; 702 703 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 704 { 705 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 706 707 pr_debug("subflow=%p\n", subflow); 708 709 if (skb->protocol == htons(ETH_P_IP)) 710 return subflow_v4_conn_request(sk, skb); 711 712 if (!ipv6_unicast_destination(skb)) 713 goto drop; 714 715 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) { 716 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS); 717 return 0; 718 } 719 720 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops, 721 &subflow_request_sock_ipv6_ops, sk, skb); 722 723 drop: 724 tcp_listendrop(sk); 725 return 0; /* don't send reset */ 726 } 727 728 static void subflow_v6_req_destructor(struct request_sock *req) 729 { 730 subflow_req_destructor(req); 731 tcp6_request_sock_ops.destructor(req); 732 } 733 #endif 734 735 struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops, 736 struct sock *sk_listener, 737 bool attach_listener) 738 { 739 if (ops->family == AF_INET) 740 ops = &mptcp_subflow_v4_request_sock_ops; 741 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 742 else if (ops->family == AF_INET6) 743 ops = &mptcp_subflow_v6_request_sock_ops; 744 #endif 745 746 return inet_reqsk_alloc(ops, sk_listener, attach_listener); 747 } 748 EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc); 749 750 /* validate hmac received in third ACK */ 751 static bool subflow_hmac_valid(const struct mptcp_subflow_request_sock *subflow_req, 752 const struct mptcp_options_received *mp_opt) 753 { 754 struct mptcp_sock *msk = subflow_req->msk; 755 u8 hmac[SHA256_DIGEST_SIZE]; 756 757 subflow_generate_hmac(READ_ONCE(msk->remote_key), 758 READ_ONCE(msk->local_key), 759 subflow_req->remote_nonce, 760 subflow_req->local_nonce, hmac); 761 762 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); 763 } 764 765 static void subflow_ulp_fallback(struct sock *sk, 766 struct mptcp_subflow_context *old_ctx) 767 { 768 struct inet_connection_sock *icsk = inet_csk(sk); 769 770 mptcp_subflow_tcp_fallback(sk, old_ctx); 771 icsk->icsk_ulp_ops = NULL; 772 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 773 tcp_sk(sk)->is_mptcp = 0; 774 775 mptcp_subflow_ops_undo_override(sk); 776 } 777 778 void mptcp_subflow_drop_ctx(struct sock *ssk) 779 { 780 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 781 782 if (!ctx) 783 return; 784 785 list_del(&mptcp_subflow_ctx(ssk)->node); 786 if (inet_csk(ssk)->icsk_ulp_ops) { 787 subflow_ulp_fallback(ssk, ctx); 788 if (ctx->conn) 789 sock_put(ctx->conn); 790 } 791 792 kfree_rcu(ctx, rcu); 793 } 794 795 void __mptcp_subflow_fully_established(struct mptcp_sock *msk, 796 struct mptcp_subflow_context *subflow, 797 const struct mptcp_options_received *mp_opt) 798 { 799 subflow_set_remote_key(msk, subflow, mp_opt); 800 WRITE_ONCE(subflow->fully_established, true); 801 WRITE_ONCE(msk->fully_established, true); 802 } 803 804 static struct sock *subflow_syn_recv_sock(const struct sock *sk, 805 struct sk_buff *skb, 806 struct request_sock *req, 807 struct dst_entry *dst, 808 struct request_sock *req_unhash, 809 bool *own_req, 810 void (*opt_child_init)(struct sock *newsk, 811 const struct sock *sk)) 812 { 813 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 814 struct mptcp_subflow_request_sock *subflow_req; 815 struct mptcp_options_received mp_opt; 816 bool fallback, fallback_is_fatal; 817 enum sk_rst_reason reason; 818 struct mptcp_sock *owner; 819 struct sock *child; 820 821 pr_debug("listener=%p, req=%p, conn=%p\n", listener, req, listener->conn); 822 823 /* After child creation we must look for MPC even when options 824 * are not parsed 825 */ 826 mp_opt.suboptions = 0; 827 828 /* hopefully temporary handling for MP_JOIN+syncookie */ 829 subflow_req = mptcp_subflow_rsk(req); 830 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join; 831 fallback = !tcp_rsk(req)->is_mptcp; 832 if (fallback) 833 goto create_child; 834 835 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 836 if (subflow_req->mp_capable) { 837 /* we can receive and accept an in-window, out-of-order pkt, 838 * which may not carry the MP_CAPABLE opt even on mptcp enabled 839 * paths: always try to extract the peer key, and fallback 840 * for packets missing it. 841 * Even OoO DSS packets coming legitly after dropped or 842 * reordered MPC will cause fallback, but we don't have other 843 * options. 844 */ 845 mptcp_get_options(skb, &mp_opt); 846 if (!(mp_opt.suboptions & 847 (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_ACK))) 848 fallback = true; 849 850 } else if (subflow_req->mp_join) { 851 mptcp_get_options(skb, &mp_opt); 852 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK)) 853 fallback = true; 854 } 855 856 create_child: 857 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 858 req_unhash, own_req, opt_child_init); 859 860 if (child && *own_req) { 861 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 862 863 tcp_rsk(req)->drop_req = false; 864 865 /* we need to fallback on ctx allocation failure and on pre-reqs 866 * checking above. In the latter scenario we additionally need 867 * to reset the context to non MPTCP status. 868 */ 869 if (!ctx || fallback) { 870 if (fallback_is_fatal) { 871 if (!ctx) 872 MPTCP_INC_STATS(sock_net(sk), 873 MPTCP_MIB_MPJOINACKNOCTX); 874 else 875 MPTCP_INC_STATS(sock_net(sk), 876 MPTCP_MIB_MPJOINACKNOMPJOIN); 877 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 878 goto dispose_child; 879 } 880 goto fallback; 881 } 882 883 /* ssk inherits options of listener sk */ 884 ctx->setsockopt_seq = listener->setsockopt_seq; 885 886 if (ctx->mp_capable) { 887 ctx->conn = mptcp_sk_clone_init(listener->conn, &mp_opt, child, req); 888 if (!ctx->conn) 889 goto fallback; 890 891 ctx->subflow_id = 1; 892 owner = mptcp_sk(ctx->conn); 893 894 if (mp_opt.deny_join_id0) 895 WRITE_ONCE(owner->pm.remote_deny_join_id0, true); 896 897 mptcp_pm_new_connection(owner, child, 1); 898 899 /* with OoO packets we can reach here without ingress 900 * mpc option 901 */ 902 if (mp_opt.suboptions & OPTION_MPTCP_MPC_ACK) { 903 mptcp_pm_fully_established(owner, child); 904 ctx->pm_notified = 1; 905 } 906 } else if (ctx->mp_join) { 907 owner = subflow_req->msk; 908 if (!owner) { 909 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 910 goto dispose_child; 911 } 912 913 if (!subflow_hmac_valid(subflow_req, &mp_opt)) { 914 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 915 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); 916 goto dispose_child; 917 } 918 919 if (!mptcp_can_accept_new_subflow(owner)) { 920 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED); 921 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 922 goto dispose_child; 923 } 924 925 /* move the msk reference ownership to the subflow */ 926 subflow_req->msk = NULL; 927 ctx->conn = (struct sock *)owner; 928 929 if (subflow_use_different_sport(owner, sk)) { 930 pr_debug("ack inet_sport=%d %d\n", 931 ntohs(inet_sk(sk)->inet_sport), 932 ntohs(inet_sk((struct sock *)owner)->inet_sport)); 933 if (!mptcp_pm_announced_has_ssk(owner, sk)) { 934 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX); 935 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); 936 goto dispose_child; 937 } 938 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX); 939 } 940 941 if (!mptcp_finish_join(child)) { 942 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(child); 943 944 subflow_add_reset_reason(skb, subflow->reset_reason); 945 goto dispose_child; 946 } 947 948 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 949 tcp_rsk(req)->drop_req = true; 950 } 951 } 952 953 /* check for expected invariant - should never trigger, just help 954 * catching earlier subtle bugs 955 */ 956 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 957 (!mptcp_subflow_ctx(child) || 958 !mptcp_subflow_ctx(child)->conn)); 959 return child; 960 961 dispose_child: 962 mptcp_subflow_drop_ctx(child); 963 tcp_rsk(req)->drop_req = true; 964 inet_csk_prepare_for_destroy_sock(child); 965 tcp_done(child); 966 reason = mptcp_get_rst_reason(skb); 967 req->rsk_ops->send_reset(sk, skb, reason); 968 969 /* The last child reference will be released by the caller */ 970 return child; 971 972 fallback: 973 if (fallback) 974 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 975 mptcp_subflow_drop_ctx(child); 976 return child; 977 } 978 979 static struct inet_connection_sock_af_ops subflow_specific __ro_after_init; 980 static struct proto tcp_prot_override __ro_after_init; 981 982 enum mapping_status { 983 MAPPING_OK, 984 MAPPING_INVALID, 985 MAPPING_EMPTY, 986 MAPPING_DATA_FIN, 987 MAPPING_DUMMY, 988 MAPPING_BAD_CSUM, 989 MAPPING_NODSS 990 }; 991 992 static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 993 { 994 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d\n", 995 ssn, subflow->map_subflow_seq, subflow->map_data_len); 996 } 997 998 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 999 { 1000 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1001 unsigned int skb_consumed; 1002 1003 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 1004 if (unlikely(skb_consumed >= skb->len)) { 1005 DEBUG_NET_WARN_ON_ONCE(1); 1006 return true; 1007 } 1008 1009 return skb->len - skb_consumed <= subflow->map_data_len - 1010 mptcp_subflow_get_map_offset(subflow); 1011 } 1012 1013 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 1014 { 1015 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1016 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 1017 1018 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 1019 /* Mapping covers data later in the subflow stream, 1020 * currently unsupported. 1021 */ 1022 dbg_bad_map(subflow, ssn); 1023 return false; 1024 } 1025 if (unlikely(!before(ssn, subflow->map_subflow_seq + 1026 subflow->map_data_len))) { 1027 /* Mapping does covers past subflow data, invalid */ 1028 dbg_bad_map(subflow, ssn); 1029 return false; 1030 } 1031 return true; 1032 } 1033 1034 static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb, 1035 bool csum_reqd) 1036 { 1037 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1038 u32 offset, seq, delta; 1039 __sum16 csum; 1040 int len; 1041 1042 if (!csum_reqd) 1043 return MAPPING_OK; 1044 1045 /* mapping already validated on previous traversal */ 1046 if (subflow->map_csum_len == subflow->map_data_len) 1047 return MAPPING_OK; 1048 1049 /* traverse the receive queue, ensuring it contains a full 1050 * DSS mapping and accumulating the related csum. 1051 * Preserve the accoumlate csum across multiple calls, to compute 1052 * the csum only once 1053 */ 1054 delta = subflow->map_data_len - subflow->map_csum_len; 1055 for (;;) { 1056 seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len; 1057 offset = seq - TCP_SKB_CB(skb)->seq; 1058 1059 /* if the current skb has not been accounted yet, csum its contents 1060 * up to the amount covered by the current DSS 1061 */ 1062 if (offset < skb->len) { 1063 __wsum csum; 1064 1065 len = min(skb->len - offset, delta); 1066 csum = skb_checksum(skb, offset, len, 0); 1067 subflow->map_data_csum = csum_block_add(subflow->map_data_csum, csum, 1068 subflow->map_csum_len); 1069 1070 delta -= len; 1071 subflow->map_csum_len += len; 1072 } 1073 if (delta == 0) 1074 break; 1075 1076 if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) { 1077 /* if this subflow is closed, the partial mapping 1078 * will be never completed; flush the pending skbs, so 1079 * that subflow_sched_work_if_closed() can kick in 1080 */ 1081 if (unlikely(ssk->sk_state == TCP_CLOSE)) 1082 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1083 sk_eat_skb(ssk, skb); 1084 1085 /* not enough data to validate the csum */ 1086 return MAPPING_EMPTY; 1087 } 1088 1089 /* the DSS mapping for next skbs will be validated later, 1090 * when a get_mapping_status call will process such skb 1091 */ 1092 skb = skb->next; 1093 } 1094 1095 /* note that 'map_data_len' accounts only for the carried data, does 1096 * not include the eventual seq increment due to the data fin, 1097 * while the pseudo header requires the original DSS data len, 1098 * including that 1099 */ 1100 csum = __mptcp_make_csum(subflow->map_seq, 1101 subflow->map_subflow_seq, 1102 subflow->map_data_len + subflow->map_data_fin, 1103 subflow->map_data_csum); 1104 if (unlikely(csum)) { 1105 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR); 1106 return MAPPING_BAD_CSUM; 1107 } 1108 1109 subflow->valid_csum_seen = 1; 1110 return MAPPING_OK; 1111 } 1112 1113 static enum mapping_status get_mapping_status(struct sock *ssk, 1114 struct mptcp_sock *msk) 1115 { 1116 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1117 bool csum_reqd = READ_ONCE(msk->csum_enabled); 1118 struct mptcp_ext *mpext; 1119 struct sk_buff *skb; 1120 u16 data_len; 1121 u64 map_seq; 1122 1123 skb = skb_peek(&ssk->sk_receive_queue); 1124 if (!skb) 1125 return MAPPING_EMPTY; 1126 1127 if (mptcp_check_fallback(ssk)) 1128 return MAPPING_DUMMY; 1129 1130 mpext = mptcp_get_ext(skb); 1131 if (!mpext || !mpext->use_map) { 1132 if (!subflow->map_valid && !skb->len) { 1133 /* the TCP stack deliver 0 len FIN pkt to the receive 1134 * queue, that is the only 0len pkts ever expected here, 1135 * and we can admit no mapping only for 0 len pkts 1136 */ 1137 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 1138 WARN_ONCE(1, "0len seq %d:%d flags %x", 1139 TCP_SKB_CB(skb)->seq, 1140 TCP_SKB_CB(skb)->end_seq, 1141 TCP_SKB_CB(skb)->tcp_flags); 1142 sk_eat_skb(ssk, skb); 1143 return MAPPING_EMPTY; 1144 } 1145 1146 /* If the required DSS has likely been dropped by a middlebox */ 1147 if (!subflow->map_valid) 1148 return MAPPING_NODSS; 1149 1150 goto validate_seq; 1151 } 1152 1153 trace_get_mapping_status(mpext); 1154 1155 data_len = mpext->data_len; 1156 if (data_len == 0) { 1157 pr_debug("infinite mapping received\n"); 1158 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 1159 return MAPPING_INVALID; 1160 } 1161 1162 if (mpext->data_fin == 1) { 1163 u64 data_fin_seq; 1164 1165 if (data_len == 1) { 1166 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq, 1167 mpext->dsn64); 1168 pr_debug("DATA_FIN with no payload seq=%llu\n", mpext->data_seq); 1169 if (subflow->map_valid) { 1170 /* A DATA_FIN might arrive in a DSS 1171 * option before the previous mapping 1172 * has been fully consumed. Continue 1173 * handling the existing mapping. 1174 */ 1175 skb_ext_del(skb, SKB_EXT_MPTCP); 1176 return MAPPING_OK; 1177 } 1178 1179 if (updated) 1180 mptcp_schedule_work((struct sock *)msk); 1181 1182 return MAPPING_DATA_FIN; 1183 } 1184 1185 data_fin_seq = mpext->data_seq + data_len - 1; 1186 1187 /* If mpext->data_seq is a 32-bit value, data_fin_seq must also 1188 * be limited to 32 bits. 1189 */ 1190 if (!mpext->dsn64) 1191 data_fin_seq &= GENMASK_ULL(31, 0); 1192 1193 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64); 1194 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d\n", 1195 data_fin_seq, mpext->dsn64); 1196 1197 /* Adjust for DATA_FIN using 1 byte of sequence space */ 1198 data_len--; 1199 } 1200 1201 map_seq = mptcp_expand_seq(READ_ONCE(msk->ack_seq), mpext->data_seq, mpext->dsn64); 1202 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64); 1203 1204 if (subflow->map_valid) { 1205 /* Allow replacing only with an identical map */ 1206 if (subflow->map_seq == map_seq && 1207 subflow->map_subflow_seq == mpext->subflow_seq && 1208 subflow->map_data_len == data_len && 1209 subflow->map_csum_reqd == mpext->csum_reqd) { 1210 skb_ext_del(skb, SKB_EXT_MPTCP); 1211 goto validate_csum; 1212 } 1213 1214 /* If this skb data are fully covered by the current mapping, 1215 * the new map would need caching, which is not supported 1216 */ 1217 if (skb_is_fully_mapped(ssk, skb)) { 1218 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 1219 return MAPPING_INVALID; 1220 } 1221 1222 /* will validate the next map after consuming the current one */ 1223 goto validate_csum; 1224 } 1225 1226 subflow->map_seq = map_seq; 1227 subflow->map_subflow_seq = mpext->subflow_seq; 1228 subflow->map_data_len = data_len; 1229 subflow->map_valid = 1; 1230 subflow->map_data_fin = mpext->data_fin; 1231 subflow->mpc_map = mpext->mpc_map; 1232 subflow->map_csum_reqd = mpext->csum_reqd; 1233 subflow->map_csum_len = 0; 1234 subflow->map_data_csum = csum_unfold(mpext->csum); 1235 1236 /* Cfr RFC 8684 Section 3.3.0 */ 1237 if (unlikely(subflow->map_csum_reqd != csum_reqd)) 1238 return MAPPING_INVALID; 1239 1240 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u csum=%d:%u\n", 1241 subflow->map_seq, subflow->map_subflow_seq, 1242 subflow->map_data_len, subflow->map_csum_reqd, 1243 subflow->map_data_csum); 1244 1245 validate_seq: 1246 /* we revalidate valid mapping on new skb, because we must ensure 1247 * the current skb is completely covered by the available mapping 1248 */ 1249 if (!validate_mapping(ssk, skb)) { 1250 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH); 1251 return MAPPING_INVALID; 1252 } 1253 1254 skb_ext_del(skb, SKB_EXT_MPTCP); 1255 1256 validate_csum: 1257 return validate_data_csum(ssk, skb, csum_reqd); 1258 } 1259 1260 static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb, 1261 u64 limit) 1262 { 1263 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1264 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 1265 struct tcp_sock *tp = tcp_sk(ssk); 1266 u32 offset, incr, avail_len; 1267 1268 offset = tp->copied_seq - TCP_SKB_CB(skb)->seq; 1269 if (WARN_ON_ONCE(offset > skb->len)) 1270 goto out; 1271 1272 avail_len = skb->len - offset; 1273 incr = limit >= avail_len ? avail_len + fin : limit; 1274 1275 pr_debug("discarding=%d len=%d offset=%d seq=%d\n", incr, skb->len, 1276 offset, subflow->map_subflow_seq); 1277 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA); 1278 tcp_sk(ssk)->copied_seq += incr; 1279 1280 out: 1281 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq)) 1282 sk_eat_skb(ssk, skb); 1283 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) 1284 subflow->map_valid = 0; 1285 } 1286 1287 static bool subflow_is_done(const struct sock *sk) 1288 { 1289 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1290 } 1291 1292 /* sched mptcp worker for subflow cleanup if no more data is pending */ 1293 static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk) 1294 { 1295 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1296 struct sock *sk = (struct sock *)msk; 1297 1298 if (likely(ssk->sk_state != TCP_CLOSE && 1299 (ssk->sk_state != TCP_CLOSE_WAIT || 1300 inet_sk_state_load(sk) != TCP_ESTABLISHED))) 1301 return; 1302 1303 if (!skb_queue_empty(&ssk->sk_receive_queue)) 1304 return; 1305 1306 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 1307 mptcp_schedule_work(sk); 1308 1309 /* when the fallback subflow closes the rx side, trigger a 'dummy' 1310 * ingress data fin, so that the msk state will follow along 1311 */ 1312 if (__mptcp_check_fallback(msk) && subflow_is_done(ssk) && 1313 msk->first == ssk && 1314 mptcp_update_rcv_data_fin(msk, subflow->map_seq + 1315 subflow->map_data_len, true)) 1316 mptcp_schedule_work(sk); 1317 } 1318 1319 static bool mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk) 1320 { 1321 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1322 unsigned long fail_tout; 1323 1324 /* we are really failing, prevent any later subflow join */ 1325 spin_lock_bh(&msk->fallback_lock); 1326 if (!msk->allow_infinite_fallback) { 1327 spin_unlock_bh(&msk->fallback_lock); 1328 return false; 1329 } 1330 msk->allow_subflows = false; 1331 spin_unlock_bh(&msk->fallback_lock); 1332 1333 /* graceful failure can happen only on the MPC subflow */ 1334 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first))) 1335 return false; 1336 1337 /* since the close timeout take precedence on the fail one, 1338 * no need to start the latter when the first is already set 1339 */ 1340 if (sock_flag((struct sock *)msk, SOCK_DEAD)) 1341 return true; 1342 1343 /* we don't need extreme accuracy here, use a zero fail_tout as special 1344 * value meaning no fail timeout at all; 1345 */ 1346 fail_tout = jiffies + TCP_RTO_MAX; 1347 if (!fail_tout) 1348 fail_tout = 1; 1349 WRITE_ONCE(subflow->fail_tout, fail_tout); 1350 tcp_send_ack(ssk); 1351 1352 mptcp_reset_tout_timer(msk, subflow->fail_tout); 1353 return true; 1354 } 1355 1356 static bool subflow_check_data_avail(struct sock *ssk) 1357 { 1358 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1359 enum mapping_status status; 1360 struct mptcp_sock *msk; 1361 struct sk_buff *skb; 1362 1363 if (!skb_peek(&ssk->sk_receive_queue)) 1364 WRITE_ONCE(subflow->data_avail, false); 1365 if (subflow->data_avail) 1366 return true; 1367 1368 msk = mptcp_sk(subflow->conn); 1369 for (;;) { 1370 u64 ack_seq; 1371 u64 old_ack; 1372 1373 status = get_mapping_status(ssk, msk); 1374 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue)); 1375 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY || 1376 status == MAPPING_BAD_CSUM || status == MAPPING_NODSS)) 1377 goto fallback; 1378 1379 if (status != MAPPING_OK) 1380 goto no_data; 1381 1382 skb = skb_peek(&ssk->sk_receive_queue); 1383 if (WARN_ON_ONCE(!skb)) 1384 goto no_data; 1385 1386 if (unlikely(!READ_ONCE(msk->can_ack))) 1387 goto fallback; 1388 1389 old_ack = READ_ONCE(msk->ack_seq); 1390 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 1391 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx\n", old_ack, 1392 ack_seq); 1393 if (unlikely(before64(ack_seq, old_ack))) { 1394 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq); 1395 continue; 1396 } 1397 1398 WRITE_ONCE(subflow->data_avail, true); 1399 break; 1400 } 1401 return true; 1402 1403 no_data: 1404 subflow_sched_work_if_closed(msk, ssk); 1405 return false; 1406 1407 fallback: 1408 if (!__mptcp_check_fallback(msk)) { 1409 /* RFC 8684 section 3.7. */ 1410 if (status == MAPPING_BAD_CSUM && 1411 (subflow->mp_join || subflow->valid_csum_seen)) { 1412 subflow->send_mp_fail = 1; 1413 1414 if (!mptcp_subflow_fail(msk, ssk)) { 1415 subflow->reset_transient = 0; 1416 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX; 1417 goto reset; 1418 } 1419 WRITE_ONCE(subflow->data_avail, true); 1420 return true; 1421 } 1422 1423 if (!mptcp_try_fallback(ssk, MPTCP_MIB_DSSFALLBACK)) { 1424 /* fatal protocol error, close the socket. 1425 * subflow_error_report() will introduce the appropriate barriers 1426 */ 1427 subflow->reset_transient = 0; 1428 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSRESET); 1429 subflow->reset_reason = status == MAPPING_NODSS ? 1430 MPTCP_RST_EMIDDLEBOX : 1431 MPTCP_RST_EMPTCP; 1432 1433 reset: 1434 WRITE_ONCE(ssk->sk_err, EBADMSG); 1435 tcp_set_state(ssk, TCP_CLOSE); 1436 while ((skb = skb_peek(&ssk->sk_receive_queue))) 1437 sk_eat_skb(ssk, skb); 1438 mptcp_send_active_reset_reason(ssk); 1439 WRITE_ONCE(subflow->data_avail, false); 1440 return false; 1441 } 1442 } 1443 1444 skb = skb_peek(&ssk->sk_receive_queue); 1445 subflow->map_valid = 1; 1446 subflow->map_data_len = skb->len; 1447 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 1448 subflow->map_seq = __mptcp_expand_seq(subflow->map_seq, 1449 subflow->iasn + 1450 TCP_SKB_CB(skb)->seq - 1451 subflow->ssn_offset - 1); 1452 WRITE_ONCE(subflow->data_avail, true); 1453 return true; 1454 } 1455 1456 bool mptcp_subflow_data_available(struct sock *sk) 1457 { 1458 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1459 1460 /* check if current mapping is still valid */ 1461 if (subflow->map_valid && 1462 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 1463 subflow->map_valid = 0; 1464 WRITE_ONCE(subflow->data_avail, false); 1465 1466 pr_debug("Done with mapping: seq=%u data_len=%u\n", 1467 subflow->map_subflow_seq, 1468 subflow->map_data_len); 1469 } 1470 1471 return subflow_check_data_avail(sk); 1472 } 1473 1474 /* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy, 1475 * not the ssk one. 1476 * 1477 * In mptcp, rwin is about the mptcp-level connection data. 1478 * 1479 * Data that is still on the ssk rx queue can thus be ignored, 1480 * as far as mptcp peer is concerned that data is still inflight. 1481 * DSS ACK is updated when skb is moved to the mptcp rx queue. 1482 */ 1483 void mptcp_space(const struct sock *ssk, int *space, int *full_space) 1484 { 1485 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 1486 const struct sock *sk = subflow->conn; 1487 1488 *space = __mptcp_space(sk); 1489 *full_space = mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 1490 } 1491 1492 static void subflow_error_report(struct sock *ssk) 1493 { 1494 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1495 1496 /* bail early if this is a no-op, so that we avoid introducing a 1497 * problematic lockdep dependency between TCP accept queue lock 1498 * and msk socket spinlock 1499 */ 1500 if (!sk->sk_socket) 1501 return; 1502 1503 mptcp_data_lock(sk); 1504 if (!sock_owned_by_user(sk)) 1505 __mptcp_error_report(sk); 1506 else 1507 __set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->cb_flags); 1508 mptcp_data_unlock(sk); 1509 } 1510 1511 static void subflow_data_ready(struct sock *sk) 1512 { 1513 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1514 u16 state = 1 << inet_sk_state_load(sk); 1515 struct sock *parent = subflow->conn; 1516 struct mptcp_sock *msk; 1517 1518 trace_sk_data_ready(sk); 1519 1520 msk = mptcp_sk(parent); 1521 if (state & TCPF_LISTEN) { 1522 /* MPJ subflow are removed from accept queue before reaching here, 1523 * avoid stray wakeups 1524 */ 1525 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue)) 1526 return; 1527 1528 parent->sk_data_ready(parent); 1529 return; 1530 } 1531 1532 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable && 1533 !subflow->mp_join && !(state & TCPF_CLOSE)); 1534 1535 if (mptcp_subflow_data_available(sk)) { 1536 mptcp_data_ready(parent, sk); 1537 1538 /* subflow-level lowat test are not relevant. 1539 * respect the msk-level threshold eventually mandating an immediate ack 1540 */ 1541 if (mptcp_data_avail(msk) < parent->sk_rcvlowat && 1542 (tcp_sk(sk)->rcv_nxt - tcp_sk(sk)->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss) 1543 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW; 1544 } else if (unlikely(sk->sk_err)) { 1545 subflow_error_report(sk); 1546 } 1547 } 1548 1549 static void subflow_write_space(struct sock *ssk) 1550 { 1551 struct sock *sk = mptcp_subflow_ctx(ssk)->conn; 1552 1553 mptcp_propagate_sndbuf(sk, ssk); 1554 mptcp_write_space(sk); 1555 } 1556 1557 static const struct inet_connection_sock_af_ops * 1558 subflow_default_af_ops(struct sock *sk) 1559 { 1560 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1561 if (sk->sk_family == AF_INET6) 1562 return &subflow_v6_specific; 1563 #endif 1564 return &subflow_specific; 1565 } 1566 1567 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1568 void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 1569 { 1570 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1571 struct inet_connection_sock *icsk = inet_csk(sk); 1572 const struct inet_connection_sock_af_ops *target; 1573 1574 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 1575 1576 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d\n", 1577 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 1578 1579 if (likely(icsk->icsk_af_ops == target)) 1580 return; 1581 1582 subflow->icsk_af_ops = icsk->icsk_af_ops; 1583 icsk->icsk_af_ops = target; 1584 } 1585 #endif 1586 1587 void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 1588 struct sockaddr_storage *addr, 1589 unsigned short family) 1590 { 1591 memset(addr, 0, sizeof(*addr)); 1592 addr->ss_family = family; 1593 if (addr->ss_family == AF_INET) { 1594 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 1595 1596 if (info->family == AF_INET) 1597 in_addr->sin_addr = info->addr; 1598 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1599 else if (ipv6_addr_v4mapped(&info->addr6)) 1600 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3]; 1601 #endif 1602 in_addr->sin_port = info->port; 1603 } 1604 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1605 else if (addr->ss_family == AF_INET6) { 1606 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 1607 1608 if (info->family == AF_INET) 1609 ipv6_addr_set_v4mapped(info->addr.s_addr, 1610 &in6_addr->sin6_addr); 1611 else 1612 in6_addr->sin6_addr = info->addr6; 1613 in6_addr->sin6_port = info->port; 1614 } 1615 #endif 1616 } 1617 1618 int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_pm_local *local, 1619 const struct mptcp_addr_info *remote) 1620 { 1621 struct mptcp_sock *msk = mptcp_sk(sk); 1622 struct mptcp_subflow_context *subflow; 1623 int local_id = local->addr.id; 1624 struct sockaddr_storage addr; 1625 int remote_id = remote->id; 1626 int err = -ENOTCONN; 1627 struct socket *sf; 1628 struct sock *ssk; 1629 u32 remote_token; 1630 int addrlen; 1631 1632 /* The userspace PM sent the request too early? */ 1633 if (!mptcp_is_fully_established(sk)) 1634 goto err_out; 1635 1636 err = mptcp_subflow_create_socket(sk, local->addr.family, &sf); 1637 if (err) { 1638 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCREATSKERR); 1639 pr_debug("msk=%p local=%d remote=%d create sock error: %d\n", 1640 msk, local_id, remote_id, err); 1641 goto err_out; 1642 } 1643 1644 ssk = sf->sk; 1645 subflow = mptcp_subflow_ctx(ssk); 1646 do { 1647 subflow->local_nonce = get_random_u32(); 1648 } while (!subflow->local_nonce); 1649 1650 /* if 'IPADDRANY', the ID will be set later, after the routing */ 1651 if (local->addr.family == AF_INET) { 1652 if (!local->addr.addr.s_addr) 1653 local_id = -1; 1654 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1655 } else if (sk->sk_family == AF_INET6) { 1656 if (ipv6_addr_any(&local->addr.addr6)) 1657 local_id = -1; 1658 #endif 1659 } 1660 1661 if (local_id >= 0) 1662 subflow_set_local_id(subflow, local_id); 1663 1664 subflow->remote_key_valid = 1; 1665 subflow->remote_key = READ_ONCE(msk->remote_key); 1666 subflow->local_key = READ_ONCE(msk->local_key); 1667 subflow->token = msk->token; 1668 mptcp_info2sockaddr(&local->addr, &addr, ssk->sk_family); 1669 1670 addrlen = sizeof(struct sockaddr_in); 1671 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1672 if (addr.ss_family == AF_INET6) 1673 addrlen = sizeof(struct sockaddr_in6); 1674 #endif 1675 ssk->sk_bound_dev_if = local->ifindex; 1676 err = kernel_bind(sf, (struct sockaddr_unsized *)&addr, addrlen); 1677 if (err) { 1678 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXBINDERR); 1679 pr_debug("msk=%p local=%d remote=%d bind error: %d\n", 1680 msk, local_id, remote_id, err); 1681 goto failed; 1682 } 1683 1684 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 1685 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d\n", msk, 1686 remote_token, local_id, remote_id); 1687 subflow->remote_token = remote_token; 1688 WRITE_ONCE(subflow->remote_id, remote_id); 1689 subflow->request_join = 1; 1690 subflow->request_bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP); 1691 subflow->subflow_id = msk->subflow_id++; 1692 mptcp_info2sockaddr(remote, &addr, ssk->sk_family); 1693 1694 sock_hold(ssk); 1695 list_add_tail(&subflow->node, &msk->conn_list); 1696 err = kernel_connect(sf, (struct sockaddr_unsized *)&addr, addrlen, O_NONBLOCK); 1697 if (err && err != -EINPROGRESS) { 1698 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCONNECTERR); 1699 pr_debug("msk=%p local=%d remote=%d connect error: %d\n", 1700 msk, local_id, remote_id, err); 1701 goto failed_unlink; 1702 } 1703 1704 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTX); 1705 1706 /* discard the subflow socket */ 1707 mptcp_sock_graft(ssk, sk->sk_socket); 1708 iput(SOCK_INODE(sf)); 1709 mptcp_stop_tout_timer(sk); 1710 return 0; 1711 1712 failed_unlink: 1713 list_del(&subflow->node); 1714 sock_put(mptcp_subflow_tcp_sock(subflow)); 1715 1716 failed: 1717 subflow->disposable = 1; 1718 sock_release(sf); 1719 1720 err_out: 1721 /* we account subflows before the creation, and this failures will not 1722 * be caught by sk_state_change() 1723 */ 1724 mptcp_pm_close_subflow(msk); 1725 return err; 1726 } 1727 1728 void __mptcp_inherit_memcg(struct sock *sk, struct sock *ssk, gfp_t gfp) 1729 { 1730 /* Only if the msk has been accepted already (and not orphaned).*/ 1731 if (!mem_cgroup_sockets_enabled || !sk->sk_socket) 1732 return; 1733 1734 mem_cgroup_sk_inherit(sk, ssk); 1735 __sk_charge(ssk, gfp); 1736 } 1737 1738 void __mptcp_inherit_cgrp_data(struct sock *sk, struct sock *ssk) 1739 { 1740 #ifdef CONFIG_SOCK_CGROUP_DATA 1741 struct sock_cgroup_data *sk_cd = &sk->sk_cgrp_data, 1742 *ssk_cd = &ssk->sk_cgrp_data; 1743 1744 /* only the additional subflows created by kworkers have to be modified */ 1745 if (cgroup_id(sock_cgroup_ptr(sk_cd)) != 1746 cgroup_id(sock_cgroup_ptr(ssk_cd))) { 1747 cgroup_sk_free(ssk_cd); 1748 *ssk_cd = *sk_cd; 1749 cgroup_sk_clone(sk_cd); 1750 } 1751 #endif /* CONFIG_SOCK_CGROUP_DATA */ 1752 } 1753 1754 static void mptcp_attach_cgroup(struct sock *parent, struct sock *child) 1755 { 1756 __mptcp_inherit_cgrp_data(parent, child); 1757 if (mem_cgroup_sockets_enabled) 1758 mem_cgroup_sk_inherit(parent, child); 1759 } 1760 1761 static void mptcp_subflow_ops_override(struct sock *ssk) 1762 { 1763 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1764 if (ssk->sk_prot == &tcpv6_prot) 1765 ssk->sk_prot = &tcpv6_prot_override; 1766 else 1767 #endif 1768 ssk->sk_prot = &tcp_prot_override; 1769 } 1770 1771 static void mptcp_subflow_ops_undo_override(struct sock *ssk) 1772 { 1773 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1774 if (ssk->sk_prot == &tcpv6_prot_override) 1775 ssk->sk_prot = &tcpv6_prot; 1776 else 1777 #endif 1778 ssk->sk_prot = &tcp_prot; 1779 } 1780 1781 int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, 1782 struct socket **new_sock) 1783 { 1784 struct mptcp_subflow_context *subflow; 1785 struct net *net = sock_net(sk); 1786 struct socket *sf; 1787 int err; 1788 1789 /* un-accepted server sockets can reach here - on bad configuration 1790 * bail early to avoid greater trouble later 1791 */ 1792 if (unlikely(!sk->sk_socket)) 1793 return -EINVAL; 1794 1795 err = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, &sf); 1796 if (err) 1797 return err; 1798 1799 lock_sock_nested(sf->sk, SINGLE_DEPTH_NESTING); 1800 1801 err = security_mptcp_add_subflow(sk, sf->sk); 1802 if (err) 1803 goto err_free; 1804 1805 /* the newly created socket has to be in the same cgroup as its parent */ 1806 mptcp_attach_cgroup(sk, sf->sk); 1807 1808 /* kernel sockets do not by default acquire net ref, but TCP timer 1809 * needs it. 1810 * Update ns_tracker to current stack trace and refcounted tracker. 1811 */ 1812 sk_net_refcnt_upgrade(sf->sk); 1813 err = tcp_set_ulp(sf->sk, "mptcp"); 1814 if (err) 1815 goto err_free; 1816 1817 mptcp_sockopt_sync_locked(mptcp_sk(sk), sf->sk); 1818 release_sock(sf->sk); 1819 1820 /* the newly created socket really belongs to the owning MPTCP 1821 * socket, even if for additional subflows the allocation is performed 1822 * by a kernel workqueue. Adjust inode references, so that the 1823 * procfs/diag interfaces really show this one belonging to the correct 1824 * user. 1825 */ 1826 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; 1827 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; 1828 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; 1829 1830 subflow = mptcp_subflow_ctx(sf->sk); 1831 pr_debug("subflow=%p\n", subflow); 1832 1833 *new_sock = sf; 1834 sock_hold(sk); 1835 subflow->conn = sk; 1836 mptcp_subflow_ops_override(sf->sk); 1837 1838 return 0; 1839 1840 err_free: 1841 release_sock(sf->sk); 1842 sock_release(sf); 1843 return err; 1844 } 1845 1846 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1847 gfp_t priority) 1848 { 1849 struct inet_connection_sock *icsk = inet_csk(sk); 1850 struct mptcp_subflow_context *ctx; 1851 1852 ctx = kzalloc_obj(*ctx, priority); 1853 if (!ctx) 1854 return NULL; 1855 1856 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1857 INIT_LIST_HEAD(&ctx->node); 1858 INIT_LIST_HEAD(&ctx->delegated_node); 1859 1860 pr_debug("subflow=%p\n", ctx); 1861 1862 ctx->tcp_sock = sk; 1863 WRITE_ONCE(ctx->local_id, -1); 1864 1865 return ctx; 1866 } 1867 1868 static void __subflow_state_change(struct sock *sk) 1869 { 1870 struct socket_wq *wq; 1871 1872 rcu_read_lock(); 1873 wq = rcu_dereference(sk->sk_wq); 1874 if (skwq_has_sleeper(wq)) 1875 wake_up_interruptible_all(&wq->wait); 1876 rcu_read_unlock(); 1877 } 1878 1879 static void subflow_state_change(struct sock *sk) 1880 { 1881 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1882 struct sock *parent = subflow->conn; 1883 1884 __subflow_state_change(sk); 1885 1886 /* as recvmsg() does not acquire the subflow socket for ssk selection 1887 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1888 * the data available machinery here. 1889 */ 1890 if (mptcp_subflow_data_available(sk)) 1891 mptcp_data_ready(parent, sk); 1892 else if (unlikely(sk->sk_err)) 1893 subflow_error_report(sk); 1894 1895 subflow_sched_work_if_closed(mptcp_sk(parent), sk); 1896 } 1897 1898 void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk) 1899 { 1900 struct request_sock_queue *queue = &inet_csk(listener_ssk)->icsk_accept_queue; 1901 struct request_sock *req, *head, *tail; 1902 struct mptcp_subflow_context *subflow; 1903 struct sock *sk, *ssk; 1904 1905 /* Due to lock dependencies no relevant lock can be acquired under rskq_lock. 1906 * Splice the req list, so that accept() can not reach the pending ssk after 1907 * the listener socket is released below. 1908 */ 1909 spin_lock_bh(&queue->rskq_lock); 1910 head = queue->rskq_accept_head; 1911 tail = queue->rskq_accept_tail; 1912 queue->rskq_accept_head = NULL; 1913 queue->rskq_accept_tail = NULL; 1914 spin_unlock_bh(&queue->rskq_lock); 1915 1916 if (!head) 1917 return; 1918 1919 /* can't acquire the msk socket lock under the subflow one, 1920 * or will cause ABBA deadlock 1921 */ 1922 release_sock(listener_ssk); 1923 1924 for (req = head; req; req = req->dl_next) { 1925 ssk = req->sk; 1926 if (!sk_is_mptcp(ssk)) 1927 continue; 1928 1929 subflow = mptcp_subflow_ctx(ssk); 1930 if (!subflow || !subflow->conn) 1931 continue; 1932 1933 sk = subflow->conn; 1934 sock_hold(sk); 1935 1936 lock_sock_nested(sk, SINGLE_DEPTH_NESTING); 1937 __mptcp_unaccepted_force_close(sk); 1938 release_sock(sk); 1939 1940 /* lockdep will report a false positive ABBA deadlock 1941 * between cancel_work_sync and the listener socket. 1942 * The involved locks belong to different sockets WRT 1943 * the existing AB chain. 1944 * Using a per socket key is problematic as key 1945 * deregistration requires process context and must be 1946 * performed at socket disposal time, in atomic 1947 * context. 1948 * Just tell lockdep to consider the listener socket 1949 * released here. 1950 */ 1951 mutex_release(&listener_sk->sk_lock.dep_map, _RET_IP_); 1952 mptcp_cancel_work(sk); 1953 mutex_acquire(&listener_sk->sk_lock.dep_map, 0, 0, _RET_IP_); 1954 1955 sock_put(sk); 1956 } 1957 1958 /* we are still under the listener msk socket lock */ 1959 lock_sock_nested(listener_ssk, SINGLE_DEPTH_NESTING); 1960 1961 /* restore the listener queue, to let the TCP code clean it up */ 1962 spin_lock_bh(&queue->rskq_lock); 1963 WARN_ON_ONCE(queue->rskq_accept_head); 1964 queue->rskq_accept_head = head; 1965 queue->rskq_accept_tail = tail; 1966 spin_unlock_bh(&queue->rskq_lock); 1967 } 1968 1969 static int subflow_ulp_init(struct sock *sk) 1970 { 1971 struct inet_connection_sock *icsk = inet_csk(sk); 1972 struct mptcp_subflow_context *ctx; 1973 struct tcp_sock *tp = tcp_sk(sk); 1974 int err = 0; 1975 1976 /* disallow attaching ULP to a socket unless it has been 1977 * created with sock_create_kern() 1978 */ 1979 if (!sk->sk_kern_sock) { 1980 err = -EOPNOTSUPP; 1981 goto out; 1982 } 1983 1984 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1985 if (!ctx) { 1986 err = -ENOMEM; 1987 goto out; 1988 } 1989 1990 pr_debug("subflow=%p, family=%d\n", ctx, sk->sk_family); 1991 1992 tp->is_mptcp = 1; 1993 ctx->icsk_af_ops = icsk->icsk_af_ops; 1994 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1995 ctx->tcp_state_change = sk->sk_state_change; 1996 ctx->tcp_error_report = sk->sk_error_report; 1997 1998 WARN_ON_ONCE(sk->sk_data_ready != sock_def_readable); 1999 WARN_ON_ONCE(sk->sk_write_space != sk_stream_write_space); 2000 2001 sk->sk_data_ready = subflow_data_ready; 2002 sk->sk_write_space = subflow_write_space; 2003 sk->sk_state_change = subflow_state_change; 2004 sk->sk_error_report = subflow_error_report; 2005 out: 2006 return err; 2007 } 2008 2009 static void subflow_ulp_release(struct sock *ssk) 2010 { 2011 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk); 2012 bool release = true; 2013 struct sock *sk; 2014 2015 if (!ctx) 2016 return; 2017 2018 sk = ctx->conn; 2019 if (sk) { 2020 /* if the msk has been orphaned, keep the ctx 2021 * alive, will be freed by __mptcp_close_ssk(), 2022 * when the subflow is still unaccepted 2023 */ 2024 release = ctx->disposable || list_empty(&ctx->node); 2025 2026 /* inet_child_forget() does not call sk_state_change(), 2027 * explicitly trigger the socket close machinery 2028 */ 2029 if (!release && !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, 2030 &mptcp_sk(sk)->flags)) 2031 mptcp_schedule_work(sk); 2032 sock_put(sk); 2033 } 2034 2035 mptcp_subflow_ops_undo_override(ssk); 2036 if (release) 2037 kfree_rcu(ctx, rcu); 2038 } 2039 2040 static void subflow_ulp_clone(const struct request_sock *req, 2041 struct sock *newsk, 2042 const gfp_t priority) 2043 { 2044 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 2045 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 2046 struct mptcp_subflow_context *new_ctx; 2047 2048 if (!tcp_rsk(req)->is_mptcp || 2049 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 2050 subflow_ulp_fallback(newsk, old_ctx); 2051 return; 2052 } 2053 2054 new_ctx = subflow_create_ctx(newsk, priority); 2055 if (!new_ctx) { 2056 subflow_ulp_fallback(newsk, old_ctx); 2057 return; 2058 } 2059 2060 new_ctx->conn_finished = 1; 2061 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 2062 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 2063 new_ctx->tcp_error_report = old_ctx->tcp_error_report; 2064 new_ctx->rel_write_seq = 1; 2065 2066 if (subflow_req->mp_capable) { 2067 /* see comments in subflow_syn_recv_sock(), MPTCP connection 2068 * is fully established only after we receive the remote key 2069 */ 2070 new_ctx->mp_capable = 1; 2071 new_ctx->local_key = subflow_req->local_key; 2072 new_ctx->token = subflow_req->token; 2073 new_ctx->ssn_offset = subflow_req->ssn_offset; 2074 new_ctx->idsn = subflow_req->idsn; 2075 2076 /* this is the first subflow, id is always 0 */ 2077 subflow_set_local_id(new_ctx, 0); 2078 } else if (subflow_req->mp_join) { 2079 new_ctx->ssn_offset = subflow_req->ssn_offset; 2080 new_ctx->mp_join = 1; 2081 WRITE_ONCE(new_ctx->fully_established, true); 2082 new_ctx->remote_key_valid = 1; 2083 new_ctx->backup = subflow_req->backup; 2084 new_ctx->request_bkup = subflow_req->request_bkup; 2085 WRITE_ONCE(new_ctx->remote_id, subflow_req->remote_id); 2086 new_ctx->token = subflow_req->token; 2087 new_ctx->thmac = subflow_req->thmac; 2088 2089 /* the subflow req id is valid, fetched via subflow_check_req() 2090 * and subflow_token_join_request() 2091 */ 2092 subflow_set_local_id(new_ctx, subflow_req->local_id); 2093 } 2094 } 2095 2096 static void tcp_release_cb_override(struct sock *ssk) 2097 { 2098 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 2099 long status; 2100 2101 /* process and clear all the pending actions, but leave the subflow into 2102 * the napi queue. To respect locking, only the same CPU that originated 2103 * the action can touch the list. mptcp_napi_poll will take care of it. 2104 */ 2105 status = set_mask_bits(&subflow->delegated_status, MPTCP_DELEGATE_ACTIONS_MASK, 0); 2106 if (status) 2107 mptcp_subflow_process_delegated(ssk, status); 2108 2109 tcp_release_cb(ssk); 2110 } 2111 2112 static int tcp_abort_override(struct sock *ssk, int err) 2113 { 2114 /* closing a listener subflow requires a great deal of care. 2115 * keep it simple and just prevent such operation 2116 */ 2117 if (inet_sk_state_load(ssk) == TCP_LISTEN) 2118 return -EINVAL; 2119 2120 return tcp_abort(ssk, err); 2121 } 2122 2123 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 2124 .name = "mptcp", 2125 .owner = THIS_MODULE, 2126 .init = subflow_ulp_init, 2127 .release = subflow_ulp_release, 2128 .clone = subflow_ulp_clone, 2129 }; 2130 2131 static int subflow_ops_init(struct request_sock_ops *subflow_ops) 2132 { 2133 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 2134 2135 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 2136 subflow_ops->obj_size, 0, 2137 SLAB_ACCOUNT | 2138 SLAB_TYPESAFE_BY_RCU, 2139 NULL); 2140 if (!subflow_ops->slab) 2141 return -ENOMEM; 2142 2143 return 0; 2144 } 2145 2146 void __init mptcp_subflow_init(void) 2147 { 2148 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops; 2149 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4"; 2150 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor; 2151 2152 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0) 2153 panic("MPTCP: failed to init subflow v4 request sock ops\n"); 2154 2155 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 2156 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req; 2157 subflow_request_sock_ipv4_ops.send_synack = subflow_v4_send_synack; 2158 2159 subflow_specific = ipv4_specific; 2160 subflow_specific.conn_request = subflow_v4_conn_request; 2161 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 2162 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 2163 subflow_specific.rebuild_header = subflow_rebuild_header; 2164 2165 tcp_prot_override = tcp_prot; 2166 tcp_prot_override.release_cb = tcp_release_cb_override; 2167 tcp_prot_override.diag_destroy = tcp_abort_override; 2168 #ifdef CONFIG_BPF_SYSCALL 2169 /* Disable sockmap processing for subflows */ 2170 tcp_prot_override.psock_update_sk_prot = NULL; 2171 #endif 2172 2173 mptcp_diag_subflow_init(&subflow_ulp_ops); 2174 2175 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 2176 panic("MPTCP: failed to register subflows to ULP\n"); 2177 } 2178 2179 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2180 void __init mptcp_subflow_v6_init(void) 2181 { 2182 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock 2183 * structures for v4 and v6 have the same size. It should not changed in 2184 * the future but better to make sure to be warned if it is no longer 2185 * the case. 2186 */ 2187 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock)); 2188 2189 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops; 2190 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6"; 2191 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor; 2192 2193 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0) 2194 panic("MPTCP: failed to init subflow v6 request sock ops\n"); 2195 2196 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 2197 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req; 2198 subflow_request_sock_ipv6_ops.send_synack = subflow_v6_send_synack; 2199 2200 subflow_v6_specific = ipv6_specific; 2201 subflow_v6_specific.conn_request = subflow_v6_conn_request; 2202 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 2203 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 2204 subflow_v6_specific.rebuild_header = subflow_v6_rebuild_header; 2205 2206 subflow_v6m_specific = subflow_v6_specific; 2207 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 2208 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 2209 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 2210 subflow_v6m_specific.rebuild_header = subflow_rebuild_header; 2211 2212 tcpv6_prot_override = tcpv6_prot; 2213 tcpv6_prot_override.release_cb = tcp_release_cb_override; 2214 tcpv6_prot_override.diag_destroy = tcp_abort_override; 2215 #ifdef CONFIG_BPF_SYSCALL 2216 /* Disable sockmap processing for subflows */ 2217 tcpv6_prot_override.psock_update_sk_prot = NULL; 2218 #endif 2219 } 2220 #endif 2221