1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2017 - 2019, Intel Corporation. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/netdevice.h> 12 #include <crypto/algapi.h> 13 #include <net/sock.h> 14 #include <net/inet_common.h> 15 #include <net/inet_hashtables.h> 16 #include <net/protocol.h> 17 #include <net/tcp.h> 18 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 19 #include <net/ip6_route.h> 20 #endif 21 #include <net/mptcp.h> 22 #include "protocol.h" 23 #include "mib.h" 24 25 static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, 26 enum linux_mptcp_mib_field field) 27 { 28 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); 29 } 30 31 static int subflow_rebuild_header(struct sock *sk) 32 { 33 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 34 int local_id, err = 0; 35 36 if (subflow->request_mptcp && !subflow->token) { 37 pr_debug("subflow=%p", sk); 38 err = mptcp_token_new_connect(sk); 39 } else if (subflow->request_join && !subflow->local_nonce) { 40 struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn; 41 42 pr_debug("subflow=%p", sk); 43 44 do { 45 get_random_bytes(&subflow->local_nonce, sizeof(u32)); 46 } while (!subflow->local_nonce); 47 48 if (subflow->local_id) 49 goto out; 50 51 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)sk); 52 if (local_id < 0) 53 return -EINVAL; 54 55 subflow->local_id = local_id; 56 } 57 58 out: 59 if (err) 60 return err; 61 62 return subflow->icsk_af_ops->rebuild_header(sk); 63 } 64 65 static void subflow_req_destructor(struct request_sock *req) 66 { 67 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 68 69 pr_debug("subflow_req=%p", subflow_req); 70 71 if (subflow_req->mp_capable) 72 mptcp_token_destroy_request(subflow_req->token); 73 tcp_request_sock_ops.destructor(req); 74 } 75 76 static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, 77 void *hmac) 78 { 79 u8 msg[8]; 80 81 put_unaligned_be32(nonce1, &msg[0]); 82 put_unaligned_be32(nonce2, &msg[4]); 83 84 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); 85 } 86 87 /* validate received token and create truncated hmac and nonce for SYN-ACK */ 88 static bool subflow_token_join_request(struct request_sock *req, 89 const struct sk_buff *skb) 90 { 91 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 92 u8 hmac[MPTCPOPT_HMAC_LEN]; 93 struct mptcp_sock *msk; 94 int local_id; 95 96 msk = mptcp_token_get_sock(subflow_req->token); 97 if (!msk) { 98 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); 99 return false; 100 } 101 102 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); 103 if (local_id < 0) { 104 sock_put((struct sock *)msk); 105 return false; 106 } 107 subflow_req->local_id = local_id; 108 109 get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); 110 111 subflow_generate_hmac(msk->local_key, msk->remote_key, 112 subflow_req->local_nonce, 113 subflow_req->remote_nonce, hmac); 114 115 subflow_req->thmac = get_unaligned_be64(hmac); 116 117 sock_put((struct sock *)msk); 118 return true; 119 } 120 121 static void subflow_init_req(struct request_sock *req, 122 const struct sock *sk_listener, 123 struct sk_buff *skb) 124 { 125 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); 126 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 127 struct mptcp_options_received mp_opt; 128 129 pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); 130 131 mptcp_get_options(skb, &mp_opt); 132 133 subflow_req->mp_capable = 0; 134 subflow_req->mp_join = 0; 135 136 #ifdef CONFIG_TCP_MD5SIG 137 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 138 * TCP option space. 139 */ 140 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) 141 return; 142 #endif 143 144 if (mp_opt.mp_capable) { 145 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); 146 147 if (mp_opt.mp_join) 148 return; 149 } else if (mp_opt.mp_join) { 150 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); 151 } 152 153 if (mp_opt.mp_capable && listener->request_mptcp) { 154 int err; 155 156 err = mptcp_token_new_request(req); 157 if (err == 0) 158 subflow_req->mp_capable = 1; 159 160 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 161 } else if (mp_opt.mp_join && listener->request_mptcp) { 162 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; 163 subflow_req->mp_join = 1; 164 subflow_req->backup = mp_opt.backup; 165 subflow_req->remote_id = mp_opt.join_id; 166 subflow_req->token = mp_opt.token; 167 subflow_req->remote_nonce = mp_opt.nonce; 168 pr_debug("token=%u, remote_nonce=%u", subflow_req->token, 169 subflow_req->remote_nonce); 170 if (!subflow_token_join_request(req, skb)) { 171 subflow_req->mp_join = 0; 172 // @@ need to trigger RST 173 } 174 } 175 } 176 177 static void subflow_v4_init_req(struct request_sock *req, 178 const struct sock *sk_listener, 179 struct sk_buff *skb) 180 { 181 tcp_rsk(req)->is_mptcp = 1; 182 183 tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb); 184 185 subflow_init_req(req, sk_listener, skb); 186 } 187 188 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 189 static void subflow_v6_init_req(struct request_sock *req, 190 const struct sock *sk_listener, 191 struct sk_buff *skb) 192 { 193 tcp_rsk(req)->is_mptcp = 1; 194 195 tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb); 196 197 subflow_init_req(req, sk_listener, skb); 198 } 199 #endif 200 201 /* validate received truncated hmac and create hmac for third ACK */ 202 static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) 203 { 204 u8 hmac[MPTCPOPT_HMAC_LEN]; 205 u64 thmac; 206 207 subflow_generate_hmac(subflow->remote_key, subflow->local_key, 208 subflow->remote_nonce, subflow->local_nonce, 209 hmac); 210 211 thmac = get_unaligned_be64(hmac); 212 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", 213 subflow, subflow->token, 214 (unsigned long long)thmac, 215 (unsigned long long)subflow->thmac); 216 217 return thmac == subflow->thmac; 218 } 219 220 static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) 221 { 222 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 223 struct mptcp_options_received mp_opt; 224 struct sock *parent = subflow->conn; 225 struct tcp_sock *tp = tcp_sk(sk); 226 227 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); 228 229 if (inet_sk_state_load(parent) == TCP_SYN_SENT) { 230 inet_sk_state_store(parent, TCP_ESTABLISHED); 231 parent->sk_state_change(parent); 232 } 233 234 /* be sure no special action on any packet other than syn-ack */ 235 if (subflow->conn_finished) 236 return; 237 238 subflow->conn_finished = 1; 239 240 mptcp_get_options(skb, &mp_opt); 241 if (subflow->request_mptcp && mp_opt.mp_capable) { 242 subflow->mp_capable = 1; 243 subflow->can_ack = 1; 244 subflow->remote_key = mp_opt.sndr_key; 245 pr_debug("subflow=%p, remote_key=%llu", subflow, 246 subflow->remote_key); 247 } else if (subflow->request_join && mp_opt.mp_join) { 248 subflow->mp_join = 1; 249 subflow->thmac = mp_opt.thmac; 250 subflow->remote_nonce = mp_opt.nonce; 251 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow, 252 subflow->thmac, subflow->remote_nonce); 253 } else if (subflow->request_mptcp) { 254 tp->is_mptcp = 0; 255 } 256 257 if (!tp->is_mptcp) 258 return; 259 260 if (subflow->mp_capable) { 261 pr_debug("subflow=%p, remote_key=%llu", mptcp_subflow_ctx(sk), 262 subflow->remote_key); 263 mptcp_finish_connect(sk); 264 265 if (skb) { 266 pr_debug("synack seq=%u", TCP_SKB_CB(skb)->seq); 267 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 268 } 269 } else if (subflow->mp_join) { 270 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", 271 subflow, subflow->thmac, 272 subflow->remote_nonce); 273 if (!subflow_thmac_valid(subflow)) { 274 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); 275 subflow->mp_join = 0; 276 goto do_reset; 277 } 278 279 subflow_generate_hmac(subflow->local_key, subflow->remote_key, 280 subflow->local_nonce, 281 subflow->remote_nonce, 282 subflow->hmac); 283 284 if (skb) 285 subflow->ssn_offset = TCP_SKB_CB(skb)->seq; 286 287 if (!mptcp_finish_join(sk)) 288 goto do_reset; 289 290 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); 291 } else { 292 do_reset: 293 tcp_send_active_reset(sk, GFP_ATOMIC); 294 tcp_done(sk); 295 } 296 } 297 298 static struct request_sock_ops subflow_request_sock_ops; 299 static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops; 300 301 static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) 302 { 303 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 304 305 pr_debug("subflow=%p", subflow); 306 307 /* Never answer to SYNs sent to broadcast or multicast */ 308 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 309 goto drop; 310 311 return tcp_conn_request(&subflow_request_sock_ops, 312 &subflow_request_sock_ipv4_ops, 313 sk, skb); 314 drop: 315 tcp_listendrop(sk); 316 return 0; 317 } 318 319 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 320 static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops; 321 static struct inet_connection_sock_af_ops subflow_v6_specific; 322 static struct inet_connection_sock_af_ops subflow_v6m_specific; 323 324 static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) 325 { 326 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 327 328 pr_debug("subflow=%p", subflow); 329 330 if (skb->protocol == htons(ETH_P_IP)) 331 return subflow_v4_conn_request(sk, skb); 332 333 if (!ipv6_unicast_destination(skb)) 334 goto drop; 335 336 return tcp_conn_request(&subflow_request_sock_ops, 337 &subflow_request_sock_ipv6_ops, sk, skb); 338 339 drop: 340 tcp_listendrop(sk); 341 return 0; /* don't send reset */ 342 } 343 #endif 344 345 /* validate hmac received in third ACK */ 346 static bool subflow_hmac_valid(const struct request_sock *req, 347 const struct mptcp_options_received *mp_opt) 348 { 349 const struct mptcp_subflow_request_sock *subflow_req; 350 u8 hmac[MPTCPOPT_HMAC_LEN]; 351 struct mptcp_sock *msk; 352 bool ret; 353 354 subflow_req = mptcp_subflow_rsk(req); 355 msk = mptcp_token_get_sock(subflow_req->token); 356 if (!msk) 357 return false; 358 359 subflow_generate_hmac(msk->remote_key, msk->local_key, 360 subflow_req->remote_nonce, 361 subflow_req->local_nonce, hmac); 362 363 ret = true; 364 if (crypto_memneq(hmac, mp_opt->hmac, sizeof(hmac))) 365 ret = false; 366 367 sock_put((struct sock *)msk); 368 return ret; 369 } 370 371 static void mptcp_sock_destruct(struct sock *sk) 372 { 373 /* if new mptcp socket isn't accepted, it is free'd 374 * from the tcp listener sockets request queue, linked 375 * from req->sk. The tcp socket is released. 376 * This calls the ULP release function which will 377 * also remove the mptcp socket, via 378 * sock_put(ctx->conn). 379 * 380 * Problem is that the mptcp socket will not be in 381 * SYN_RECV state and doesn't have SOCK_DEAD flag. 382 * Both result in warnings from inet_sock_destruct. 383 */ 384 385 if (sk->sk_state == TCP_SYN_RECV) { 386 sk->sk_state = TCP_CLOSE; 387 WARN_ON_ONCE(sk->sk_socket); 388 sock_orphan(sk); 389 } 390 391 inet_sock_destruct(sk); 392 } 393 394 static void mptcp_force_close(struct sock *sk) 395 { 396 inet_sk_state_store(sk, TCP_CLOSE); 397 sk_common_release(sk); 398 } 399 400 static void subflow_ulp_fallback(struct sock *sk, 401 struct mptcp_subflow_context *old_ctx) 402 { 403 struct inet_connection_sock *icsk = inet_csk(sk); 404 405 mptcp_subflow_tcp_fallback(sk, old_ctx); 406 icsk->icsk_ulp_ops = NULL; 407 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 408 tcp_sk(sk)->is_mptcp = 0; 409 } 410 411 static struct sock *subflow_syn_recv_sock(const struct sock *sk, 412 struct sk_buff *skb, 413 struct request_sock *req, 414 struct dst_entry *dst, 415 struct request_sock *req_unhash, 416 bool *own_req) 417 { 418 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); 419 struct mptcp_subflow_request_sock *subflow_req; 420 struct mptcp_options_received mp_opt; 421 bool fallback_is_fatal = false; 422 struct sock *new_msk = NULL; 423 bool fallback = false; 424 struct sock *child; 425 426 pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); 427 428 /* we need later a valid 'mp_capable' value even when options are not 429 * parsed 430 */ 431 mp_opt.mp_capable = 0; 432 if (tcp_rsk(req)->is_mptcp == 0) 433 goto create_child; 434 435 /* if the sk is MP_CAPABLE, we try to fetch the client key */ 436 subflow_req = mptcp_subflow_rsk(req); 437 if (subflow_req->mp_capable) { 438 if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) { 439 /* here we can receive and accept an in-window, 440 * out-of-order pkt, which will not carry the MP_CAPABLE 441 * opt even on mptcp enabled paths 442 */ 443 goto create_msk; 444 } 445 446 mptcp_get_options(skb, &mp_opt); 447 if (!mp_opt.mp_capable) { 448 fallback = true; 449 goto create_child; 450 } 451 452 create_msk: 453 new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req); 454 if (!new_msk) 455 fallback = true; 456 } else if (subflow_req->mp_join) { 457 fallback_is_fatal = true; 458 mptcp_get_options(skb, &mp_opt); 459 if (!mp_opt.mp_join || 460 !subflow_hmac_valid(req, &mp_opt)) { 461 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); 462 return NULL; 463 } 464 } 465 466 create_child: 467 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, 468 req_unhash, own_req); 469 470 if (child && *own_req) { 471 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); 472 473 /* we need to fallback on ctx allocation failure and on pre-reqs 474 * checking above. In the latter scenario we additionally need 475 * to reset the context to non MPTCP status. 476 */ 477 if (!ctx || fallback) { 478 if (fallback_is_fatal) 479 goto close_child; 480 481 if (ctx) { 482 subflow_ulp_fallback(child, ctx); 483 kfree_rcu(ctx, rcu); 484 } 485 goto out; 486 } 487 488 if (ctx->mp_capable) { 489 /* new mpc subflow takes ownership of the newly 490 * created mptcp socket 491 */ 492 new_msk->sk_destruct = mptcp_sock_destruct; 493 mptcp_pm_new_connection(mptcp_sk(new_msk), 1); 494 ctx->conn = new_msk; 495 new_msk = NULL; 496 497 /* with OoO packets we can reach here without ingress 498 * mpc option 499 */ 500 ctx->remote_key = mp_opt.sndr_key; 501 ctx->fully_established = mp_opt.mp_capable; 502 ctx->can_ack = mp_opt.mp_capable; 503 } else if (ctx->mp_join) { 504 struct mptcp_sock *owner; 505 506 owner = mptcp_token_get_sock(ctx->token); 507 if (!owner) 508 goto close_child; 509 510 ctx->conn = (struct sock *)owner; 511 if (!mptcp_finish_join(child)) 512 goto close_child; 513 514 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); 515 } 516 } 517 518 out: 519 /* dispose of the left over mptcp master, if any */ 520 if (unlikely(new_msk)) 521 mptcp_force_close(new_msk); 522 523 /* check for expected invariant - should never trigger, just help 524 * catching eariler subtle bugs 525 */ 526 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && 527 (!mptcp_subflow_ctx(child) || 528 !mptcp_subflow_ctx(child)->conn)); 529 return child; 530 531 close_child: 532 tcp_send_active_reset(child, GFP_ATOMIC); 533 inet_csk_prepare_forced_close(child); 534 tcp_done(child); 535 return NULL; 536 } 537 538 static struct inet_connection_sock_af_ops subflow_specific; 539 540 enum mapping_status { 541 MAPPING_OK, 542 MAPPING_INVALID, 543 MAPPING_EMPTY, 544 MAPPING_DATA_FIN 545 }; 546 547 static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq) 548 { 549 if ((u32)seq == (u32)old_seq) 550 return old_seq; 551 552 /* Assume map covers data not mapped yet. */ 553 return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32)); 554 } 555 556 static void warn_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) 557 { 558 WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d", 559 ssn, subflow->map_subflow_seq, subflow->map_data_len); 560 } 561 562 static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) 563 { 564 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 565 unsigned int skb_consumed; 566 567 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; 568 if (WARN_ON_ONCE(skb_consumed >= skb->len)) 569 return true; 570 571 return skb->len - skb_consumed <= subflow->map_data_len - 572 mptcp_subflow_get_map_offset(subflow); 573 } 574 575 static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) 576 { 577 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 578 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; 579 580 if (unlikely(before(ssn, subflow->map_subflow_seq))) { 581 /* Mapping covers data later in the subflow stream, 582 * currently unsupported. 583 */ 584 warn_bad_map(subflow, ssn); 585 return false; 586 } 587 if (unlikely(!before(ssn, subflow->map_subflow_seq + 588 subflow->map_data_len))) { 589 /* Mapping does covers past subflow data, invalid */ 590 warn_bad_map(subflow, ssn + skb->len); 591 return false; 592 } 593 return true; 594 } 595 596 static enum mapping_status get_mapping_status(struct sock *ssk) 597 { 598 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 599 struct mptcp_ext *mpext; 600 struct sk_buff *skb; 601 u16 data_len; 602 u64 map_seq; 603 604 skb = skb_peek(&ssk->sk_receive_queue); 605 if (!skb) 606 return MAPPING_EMPTY; 607 608 mpext = mptcp_get_ext(skb); 609 if (!mpext || !mpext->use_map) { 610 if (!subflow->map_valid && !skb->len) { 611 /* the TCP stack deliver 0 len FIN pkt to the receive 612 * queue, that is the only 0len pkts ever expected here, 613 * and we can admit no mapping only for 0 len pkts 614 */ 615 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) 616 WARN_ONCE(1, "0len seq %d:%d flags %x", 617 TCP_SKB_CB(skb)->seq, 618 TCP_SKB_CB(skb)->end_seq, 619 TCP_SKB_CB(skb)->tcp_flags); 620 sk_eat_skb(ssk, skb); 621 return MAPPING_EMPTY; 622 } 623 624 if (!subflow->map_valid) 625 return MAPPING_INVALID; 626 627 goto validate_seq; 628 } 629 630 pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d", 631 mpext->data_seq, mpext->dsn64, mpext->subflow_seq, 632 mpext->data_len, mpext->data_fin); 633 634 data_len = mpext->data_len; 635 if (data_len == 0) { 636 pr_err("Infinite mapping not handled"); 637 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); 638 return MAPPING_INVALID; 639 } 640 641 if (mpext->data_fin == 1) { 642 if (data_len == 1) { 643 pr_debug("DATA_FIN with no payload"); 644 if (subflow->map_valid) { 645 /* A DATA_FIN might arrive in a DSS 646 * option before the previous mapping 647 * has been fully consumed. Continue 648 * handling the existing mapping. 649 */ 650 skb_ext_del(skb, SKB_EXT_MPTCP); 651 return MAPPING_OK; 652 } else { 653 return MAPPING_DATA_FIN; 654 } 655 } 656 657 /* Adjust for DATA_FIN using 1 byte of sequence space */ 658 data_len--; 659 } 660 661 if (!mpext->dsn64) { 662 map_seq = expand_seq(subflow->map_seq, subflow->map_data_len, 663 mpext->data_seq); 664 pr_debug("expanded seq=%llu", subflow->map_seq); 665 } else { 666 map_seq = mpext->data_seq; 667 } 668 669 if (subflow->map_valid) { 670 /* Allow replacing only with an identical map */ 671 if (subflow->map_seq == map_seq && 672 subflow->map_subflow_seq == mpext->subflow_seq && 673 subflow->map_data_len == data_len) { 674 skb_ext_del(skb, SKB_EXT_MPTCP); 675 return MAPPING_OK; 676 } 677 678 /* If this skb data are fully covered by the current mapping, 679 * the new map would need caching, which is not supported 680 */ 681 if (skb_is_fully_mapped(ssk, skb)) { 682 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); 683 return MAPPING_INVALID; 684 } 685 686 /* will validate the next map after consuming the current one */ 687 return MAPPING_OK; 688 } 689 690 subflow->map_seq = map_seq; 691 subflow->map_subflow_seq = mpext->subflow_seq; 692 subflow->map_data_len = data_len; 693 subflow->map_valid = 1; 694 subflow->mpc_map = mpext->mpc_map; 695 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u", 696 subflow->map_seq, subflow->map_subflow_seq, 697 subflow->map_data_len); 698 699 validate_seq: 700 /* we revalidate valid mapping on new skb, because we must ensure 701 * the current skb is completely covered by the available mapping 702 */ 703 if (!validate_mapping(ssk, skb)) 704 return MAPPING_INVALID; 705 706 skb_ext_del(skb, SKB_EXT_MPTCP); 707 return MAPPING_OK; 708 } 709 710 static int subflow_read_actor(read_descriptor_t *desc, 711 struct sk_buff *skb, 712 unsigned int offset, size_t len) 713 { 714 size_t copy_len = min(desc->count, len); 715 716 desc->count -= copy_len; 717 718 pr_debug("flushed %zu bytes, %zu left", copy_len, desc->count); 719 return copy_len; 720 } 721 722 static bool subflow_check_data_avail(struct sock *ssk) 723 { 724 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 725 enum mapping_status status; 726 struct mptcp_sock *msk; 727 struct sk_buff *skb; 728 729 pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk, 730 subflow->data_avail, skb_peek(&ssk->sk_receive_queue)); 731 if (subflow->data_avail) 732 return true; 733 734 msk = mptcp_sk(subflow->conn); 735 for (;;) { 736 u32 map_remaining; 737 size_t delta; 738 u64 ack_seq; 739 u64 old_ack; 740 741 status = get_mapping_status(ssk); 742 pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status); 743 if (status == MAPPING_INVALID) { 744 ssk->sk_err = EBADMSG; 745 goto fatal; 746 } 747 748 if (status != MAPPING_OK) 749 return false; 750 751 skb = skb_peek(&ssk->sk_receive_queue); 752 if (WARN_ON_ONCE(!skb)) 753 return false; 754 755 /* if msk lacks the remote key, this subflow must provide an 756 * MP_CAPABLE-based mapping 757 */ 758 if (unlikely(!READ_ONCE(msk->can_ack))) { 759 if (!subflow->mpc_map) { 760 ssk->sk_err = EBADMSG; 761 goto fatal; 762 } 763 WRITE_ONCE(msk->remote_key, subflow->remote_key); 764 WRITE_ONCE(msk->ack_seq, subflow->map_seq); 765 WRITE_ONCE(msk->can_ack, true); 766 } 767 768 old_ack = READ_ONCE(msk->ack_seq); 769 ack_seq = mptcp_subflow_get_mapped_dsn(subflow); 770 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, 771 ack_seq); 772 if (ack_seq == old_ack) 773 break; 774 775 /* only accept in-sequence mapping. Old values are spurious 776 * retransmission; we can hit "future" values on active backup 777 * subflow switch, we relay on retransmissions to get 778 * in-sequence data. 779 * Cuncurrent subflows support will require subflow data 780 * reordering 781 */ 782 map_remaining = subflow->map_data_len - 783 mptcp_subflow_get_map_offset(subflow); 784 if (before64(ack_seq, old_ack)) 785 delta = min_t(size_t, old_ack - ack_seq, map_remaining); 786 else 787 delta = min_t(size_t, ack_seq - old_ack, map_remaining); 788 789 /* discard mapped data */ 790 pr_debug("discarding %zu bytes, current map len=%d", delta, 791 map_remaining); 792 if (delta) { 793 read_descriptor_t desc = { 794 .count = delta, 795 }; 796 int ret; 797 798 ret = tcp_read_sock(ssk, &desc, subflow_read_actor); 799 if (ret < 0) { 800 ssk->sk_err = -ret; 801 goto fatal; 802 } 803 if (ret < delta) 804 return false; 805 if (delta == map_remaining) 806 subflow->map_valid = 0; 807 } 808 } 809 return true; 810 811 fatal: 812 /* fatal protocol error, close the socket */ 813 /* This barrier is coupled with smp_rmb() in tcp_poll() */ 814 smp_wmb(); 815 ssk->sk_error_report(ssk); 816 tcp_set_state(ssk, TCP_CLOSE); 817 tcp_send_active_reset(ssk, GFP_ATOMIC); 818 return false; 819 } 820 821 bool mptcp_subflow_data_available(struct sock *sk) 822 { 823 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 824 struct sk_buff *skb; 825 826 /* check if current mapping is still valid */ 827 if (subflow->map_valid && 828 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { 829 subflow->map_valid = 0; 830 subflow->data_avail = 0; 831 832 pr_debug("Done with mapping: seq=%u data_len=%u", 833 subflow->map_subflow_seq, 834 subflow->map_data_len); 835 } 836 837 if (!subflow_check_data_avail(sk)) { 838 subflow->data_avail = 0; 839 return false; 840 } 841 842 skb = skb_peek(&sk->sk_receive_queue); 843 subflow->data_avail = skb && 844 before(tcp_sk(sk)->copied_seq, TCP_SKB_CB(skb)->end_seq); 845 return subflow->data_avail; 846 } 847 848 static void subflow_data_ready(struct sock *sk) 849 { 850 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 851 struct sock *parent = subflow->conn; 852 853 if (!subflow->mp_capable && !subflow->mp_join) { 854 subflow->tcp_data_ready(sk); 855 856 parent->sk_data_ready(parent); 857 return; 858 } 859 860 if (mptcp_subflow_data_available(sk)) 861 mptcp_data_ready(parent, sk); 862 } 863 864 static void subflow_write_space(struct sock *sk) 865 { 866 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 867 struct sock *parent = subflow->conn; 868 869 sk_stream_write_space(sk); 870 if (sk_stream_is_writeable(sk)) { 871 set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags); 872 smp_mb__after_atomic(); 873 /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */ 874 sk_stream_write_space(parent); 875 } 876 } 877 878 static struct inet_connection_sock_af_ops * 879 subflow_default_af_ops(struct sock *sk) 880 { 881 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 882 if (sk->sk_family == AF_INET6) 883 return &subflow_v6_specific; 884 #endif 885 return &subflow_specific; 886 } 887 888 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 889 void mptcpv6_handle_mapped(struct sock *sk, bool mapped) 890 { 891 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 892 struct inet_connection_sock *icsk = inet_csk(sk); 893 struct inet_connection_sock_af_ops *target; 894 895 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); 896 897 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", 898 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); 899 900 if (likely(icsk->icsk_af_ops == target)) 901 return; 902 903 subflow->icsk_af_ops = icsk->icsk_af_ops; 904 icsk->icsk_af_ops = target; 905 } 906 #endif 907 908 static void mptcp_info2sockaddr(const struct mptcp_addr_info *info, 909 struct sockaddr_storage *addr) 910 { 911 memset(addr, 0, sizeof(*addr)); 912 addr->ss_family = info->family; 913 if (addr->ss_family == AF_INET) { 914 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; 915 916 in_addr->sin_addr = info->addr; 917 in_addr->sin_port = info->port; 918 } 919 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 920 else if (addr->ss_family == AF_INET6) { 921 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; 922 923 in6_addr->sin6_addr = info->addr6; 924 in6_addr->sin6_port = info->port; 925 } 926 #endif 927 } 928 929 int __mptcp_subflow_connect(struct sock *sk, int ifindex, 930 const struct mptcp_addr_info *loc, 931 const struct mptcp_addr_info *remote) 932 { 933 struct mptcp_sock *msk = mptcp_sk(sk); 934 struct mptcp_subflow_context *subflow; 935 struct sockaddr_storage addr; 936 struct socket *sf; 937 u32 remote_token; 938 int addrlen; 939 int err; 940 941 if (sk->sk_state != TCP_ESTABLISHED) 942 return -ENOTCONN; 943 944 err = mptcp_subflow_create_socket(sk, &sf); 945 if (err) 946 return err; 947 948 subflow = mptcp_subflow_ctx(sf->sk); 949 subflow->remote_key = msk->remote_key; 950 subflow->local_key = msk->local_key; 951 subflow->token = msk->token; 952 mptcp_info2sockaddr(loc, &addr); 953 954 addrlen = sizeof(struct sockaddr_in); 955 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 956 if (loc->family == AF_INET6) 957 addrlen = sizeof(struct sockaddr_in6); 958 #endif 959 sf->sk->sk_bound_dev_if = ifindex; 960 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); 961 if (err) 962 goto failed; 963 964 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); 965 pr_debug("msk=%p remote_token=%u", msk, remote_token); 966 subflow->remote_token = remote_token; 967 subflow->local_id = loc->id; 968 subflow->request_join = 1; 969 subflow->request_bkup = 1; 970 mptcp_info2sockaddr(remote, &addr); 971 972 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); 973 if (err && err != -EINPROGRESS) 974 goto failed; 975 976 spin_lock_bh(&msk->join_list_lock); 977 list_add_tail(&subflow->node, &msk->join_list); 978 spin_unlock_bh(&msk->join_list_lock); 979 980 return err; 981 982 failed: 983 sock_release(sf); 984 return err; 985 } 986 987 int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock) 988 { 989 struct mptcp_subflow_context *subflow; 990 struct net *net = sock_net(sk); 991 struct socket *sf; 992 int err; 993 994 err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP, 995 &sf); 996 if (err) 997 return err; 998 999 lock_sock(sf->sk); 1000 1001 /* kernel sockets do not by default acquire net ref, but TCP timer 1002 * needs it. 1003 */ 1004 sf->sk->sk_net_refcnt = 1; 1005 get_net(net); 1006 #ifdef CONFIG_PROC_FS 1007 this_cpu_add(*net->core.sock_inuse, 1); 1008 #endif 1009 err = tcp_set_ulp(sf->sk, "mptcp"); 1010 release_sock(sf->sk); 1011 1012 if (err) 1013 return err; 1014 1015 subflow = mptcp_subflow_ctx(sf->sk); 1016 pr_debug("subflow=%p", subflow); 1017 1018 *new_sock = sf; 1019 sock_hold(sk); 1020 subflow->conn = sk; 1021 1022 return 0; 1023 } 1024 1025 static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, 1026 gfp_t priority) 1027 { 1028 struct inet_connection_sock *icsk = inet_csk(sk); 1029 struct mptcp_subflow_context *ctx; 1030 1031 ctx = kzalloc(sizeof(*ctx), priority); 1032 if (!ctx) 1033 return NULL; 1034 1035 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 1036 INIT_LIST_HEAD(&ctx->node); 1037 1038 pr_debug("subflow=%p", ctx); 1039 1040 ctx->tcp_sock = sk; 1041 1042 return ctx; 1043 } 1044 1045 static void __subflow_state_change(struct sock *sk) 1046 { 1047 struct socket_wq *wq; 1048 1049 rcu_read_lock(); 1050 wq = rcu_dereference(sk->sk_wq); 1051 if (skwq_has_sleeper(wq)) 1052 wake_up_interruptible_all(&wq->wait); 1053 rcu_read_unlock(); 1054 } 1055 1056 static bool subflow_is_done(const struct sock *sk) 1057 { 1058 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; 1059 } 1060 1061 static void subflow_state_change(struct sock *sk) 1062 { 1063 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1064 struct sock *parent = subflow->conn; 1065 1066 __subflow_state_change(sk); 1067 1068 /* as recvmsg() does not acquire the subflow socket for ssk selection 1069 * a fin packet carrying a DSS can be unnoticed if we don't trigger 1070 * the data available machinery here. 1071 */ 1072 if (subflow->mp_capable && mptcp_subflow_data_available(sk)) 1073 mptcp_data_ready(parent, sk); 1074 1075 if (!(parent->sk_shutdown & RCV_SHUTDOWN) && 1076 !subflow->rx_eof && subflow_is_done(sk)) { 1077 subflow->rx_eof = 1; 1078 mptcp_subflow_eof(parent); 1079 } 1080 } 1081 1082 static int subflow_ulp_init(struct sock *sk) 1083 { 1084 struct inet_connection_sock *icsk = inet_csk(sk); 1085 struct mptcp_subflow_context *ctx; 1086 struct tcp_sock *tp = tcp_sk(sk); 1087 int err = 0; 1088 1089 /* disallow attaching ULP to a socket unless it has been 1090 * created with sock_create_kern() 1091 */ 1092 if (!sk->sk_kern_sock) { 1093 err = -EOPNOTSUPP; 1094 goto out; 1095 } 1096 1097 ctx = subflow_create_ctx(sk, GFP_KERNEL); 1098 if (!ctx) { 1099 err = -ENOMEM; 1100 goto out; 1101 } 1102 1103 pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); 1104 1105 tp->is_mptcp = 1; 1106 ctx->icsk_af_ops = icsk->icsk_af_ops; 1107 icsk->icsk_af_ops = subflow_default_af_ops(sk); 1108 ctx->tcp_data_ready = sk->sk_data_ready; 1109 ctx->tcp_state_change = sk->sk_state_change; 1110 ctx->tcp_write_space = sk->sk_write_space; 1111 sk->sk_data_ready = subflow_data_ready; 1112 sk->sk_write_space = subflow_write_space; 1113 sk->sk_state_change = subflow_state_change; 1114 out: 1115 return err; 1116 } 1117 1118 static void subflow_ulp_release(struct sock *sk) 1119 { 1120 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk); 1121 1122 if (!ctx) 1123 return; 1124 1125 if (ctx->conn) 1126 sock_put(ctx->conn); 1127 1128 kfree_rcu(ctx, rcu); 1129 } 1130 1131 static void subflow_ulp_clone(const struct request_sock *req, 1132 struct sock *newsk, 1133 const gfp_t priority) 1134 { 1135 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1136 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); 1137 struct mptcp_subflow_context *new_ctx; 1138 1139 if (!tcp_rsk(req)->is_mptcp || 1140 (!subflow_req->mp_capable && !subflow_req->mp_join)) { 1141 subflow_ulp_fallback(newsk, old_ctx); 1142 return; 1143 } 1144 1145 new_ctx = subflow_create_ctx(newsk, priority); 1146 if (!new_ctx) { 1147 subflow_ulp_fallback(newsk, old_ctx); 1148 return; 1149 } 1150 1151 new_ctx->conn_finished = 1; 1152 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; 1153 new_ctx->tcp_data_ready = old_ctx->tcp_data_ready; 1154 new_ctx->tcp_state_change = old_ctx->tcp_state_change; 1155 new_ctx->tcp_write_space = old_ctx->tcp_write_space; 1156 new_ctx->rel_write_seq = 1; 1157 new_ctx->tcp_sock = newsk; 1158 1159 if (subflow_req->mp_capable) { 1160 /* see comments in subflow_syn_recv_sock(), MPTCP connection 1161 * is fully established only after we receive the remote key 1162 */ 1163 new_ctx->mp_capable = 1; 1164 new_ctx->local_key = subflow_req->local_key; 1165 new_ctx->token = subflow_req->token; 1166 new_ctx->ssn_offset = subflow_req->ssn_offset; 1167 new_ctx->idsn = subflow_req->idsn; 1168 } else if (subflow_req->mp_join) { 1169 new_ctx->ssn_offset = subflow_req->ssn_offset; 1170 new_ctx->mp_join = 1; 1171 new_ctx->fully_established = 1; 1172 new_ctx->backup = subflow_req->backup; 1173 new_ctx->local_id = subflow_req->local_id; 1174 new_ctx->token = subflow_req->token; 1175 new_ctx->thmac = subflow_req->thmac; 1176 } 1177 } 1178 1179 static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { 1180 .name = "mptcp", 1181 .owner = THIS_MODULE, 1182 .init = subflow_ulp_init, 1183 .release = subflow_ulp_release, 1184 .clone = subflow_ulp_clone, 1185 }; 1186 1187 static int subflow_ops_init(struct request_sock_ops *subflow_ops) 1188 { 1189 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); 1190 subflow_ops->slab_name = "request_sock_subflow"; 1191 1192 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, 1193 subflow_ops->obj_size, 0, 1194 SLAB_ACCOUNT | 1195 SLAB_TYPESAFE_BY_RCU, 1196 NULL); 1197 if (!subflow_ops->slab) 1198 return -ENOMEM; 1199 1200 subflow_ops->destructor = subflow_req_destructor; 1201 1202 return 0; 1203 } 1204 1205 void mptcp_subflow_init(void) 1206 { 1207 subflow_request_sock_ops = tcp_request_sock_ops; 1208 if (subflow_ops_init(&subflow_request_sock_ops) != 0) 1209 panic("MPTCP: failed to init subflow request sock ops\n"); 1210 1211 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; 1212 subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req; 1213 1214 subflow_specific = ipv4_specific; 1215 subflow_specific.conn_request = subflow_v4_conn_request; 1216 subflow_specific.syn_recv_sock = subflow_syn_recv_sock; 1217 subflow_specific.sk_rx_dst_set = subflow_finish_connect; 1218 subflow_specific.rebuild_header = subflow_rebuild_header; 1219 1220 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1221 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; 1222 subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req; 1223 1224 subflow_v6_specific = ipv6_specific; 1225 subflow_v6_specific.conn_request = subflow_v6_conn_request; 1226 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; 1227 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; 1228 subflow_v6_specific.rebuild_header = subflow_rebuild_header; 1229 1230 subflow_v6m_specific = subflow_v6_specific; 1231 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; 1232 subflow_v6m_specific.send_check = ipv4_specific.send_check; 1233 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; 1234 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; 1235 subflow_v6m_specific.net_frag_header_len = 0; 1236 #endif 1237 1238 mptcp_diag_subflow_init(&subflow_ulp_ops); 1239 1240 if (tcp_register_ulp(&subflow_ulp_ops) != 0) 1241 panic("MPTCP: failed to register subflows to ULP\n"); 1242 } 1243