1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2019, Intel Corporation. 5 */ 6 #define pr_fmt(fmt) "MPTCP: " fmt 7 8 #include <linux/rculist.h> 9 #include <linux/spinlock.h> 10 #include "protocol.h" 11 #include "mib.h" 12 13 #define ADD_ADDR_RETRANS_MAX 3 14 15 struct mptcp_pm_add_addr { 16 struct list_head list; 17 struct mptcp_addr_info addr; 18 u8 retrans_times; 19 bool timer_done; 20 struct timer_list timer; 21 struct mptcp_sock *sock; 22 struct rcu_head rcu; 23 }; 24 25 static DEFINE_SPINLOCK(mptcp_pm_list_lock); 26 static LIST_HEAD(mptcp_pm_list); 27 28 /* path manager helpers */ 29 30 /* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses, 31 * otherwise allow any matching local/remote pair 32 */ 33 bool mptcp_pm_addr_families_match(const struct sock *sk, 34 const struct mptcp_addr_info *loc, 35 const struct mptcp_addr_info *rem) 36 { 37 bool mptcp_is_v4 = sk->sk_family == AF_INET; 38 39 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 40 bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(&loc->addr6); 41 bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(&rem->addr6); 42 43 if (mptcp_is_v4) 44 return loc_is_v4 && rem_is_v4; 45 46 if (ipv6_only_sock(sk)) 47 return !loc_is_v4 && !rem_is_v4; 48 49 return loc_is_v4 == rem_is_v4; 50 #else 51 return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET; 52 #endif 53 } 54 55 bool mptcp_addresses_equal(const struct mptcp_addr_info *a, 56 const struct mptcp_addr_info *b, bool use_port) 57 { 58 bool addr_equals = false; 59 60 if (a->family == b->family) { 61 if (a->family == AF_INET) 62 addr_equals = a->addr.s_addr == b->addr.s_addr; 63 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 64 else 65 addr_equals = ipv6_addr_equal(&a->addr6, &b->addr6); 66 } else if (a->family == AF_INET) { 67 if (ipv6_addr_v4mapped(&b->addr6)) 68 addr_equals = a->addr.s_addr == b->addr6.s6_addr32[3]; 69 } else if (b->family == AF_INET) { 70 if (ipv6_addr_v4mapped(&a->addr6)) 71 addr_equals = a->addr6.s6_addr32[3] == b->addr.s_addr; 72 #endif 73 } 74 75 if (!addr_equals) 76 return false; 77 if (!use_port) 78 return true; 79 80 return a->port == b->port; 81 } 82 83 void mptcp_local_address(const struct sock_common *skc, 84 struct mptcp_addr_info *addr) 85 { 86 addr->family = skc->skc_family; 87 addr->port = htons(skc->skc_num); 88 if (addr->family == AF_INET) 89 addr->addr.s_addr = skc->skc_rcv_saddr; 90 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 91 else if (addr->family == AF_INET6) 92 addr->addr6 = skc->skc_v6_rcv_saddr; 93 #endif 94 } 95 96 void mptcp_remote_address(const struct sock_common *skc, 97 struct mptcp_addr_info *addr) 98 { 99 addr->family = skc->skc_family; 100 addr->port = skc->skc_dport; 101 if (addr->family == AF_INET) 102 addr->addr.s_addr = skc->skc_daddr; 103 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 104 else if (addr->family == AF_INET6) 105 addr->addr6 = skc->skc_v6_daddr; 106 #endif 107 } 108 109 static bool mptcp_pm_is_init_remote_addr(struct mptcp_sock *msk, 110 const struct mptcp_addr_info *remote) 111 { 112 struct mptcp_addr_info mpc_remote; 113 114 mptcp_remote_address((struct sock_common *)msk, &mpc_remote); 115 return mptcp_addresses_equal(&mpc_remote, remote, remote->port); 116 } 117 118 bool mptcp_pm_has_subflow_saddr(const struct mptcp_sock *msk, 119 const struct mptcp_addr_info *saddr) 120 { 121 struct mptcp_subflow_context *subflow; 122 struct mptcp_addr_info cur; 123 struct sock_common *skc; 124 125 mptcp_for_each_subflow(msk, subflow) { 126 skc = (struct sock_common *)mptcp_subflow_tcp_sock(subflow); 127 128 mptcp_local_address(skc, &cur); 129 if (mptcp_addresses_equal(&cur, saddr, saddr->port)) 130 return true; 131 } 132 133 return false; 134 } 135 136 static struct mptcp_pm_add_addr * 137 mptcp_pm_announced_lookup(const struct mptcp_sock *msk, 138 const struct mptcp_addr_info *addr) 139 { 140 struct mptcp_pm_add_addr *entry; 141 142 lockdep_assert_held(&msk->pm.lock); 143 144 list_for_each_entry(entry, &msk->pm.anno_list, list) { 145 if (mptcp_addresses_equal(&entry->addr, addr, true)) 146 return entry; 147 } 148 149 return NULL; 150 } 151 152 static struct mptcp_pm_add_addr * 153 mptcp_pm_announced_del_timer(struct mptcp_sock *msk, 154 const struct mptcp_addr_info *addr, bool check_id) 155 { 156 struct sock *sk = (struct sock *)msk; 157 struct mptcp_pm_add_addr *entry; 158 bool stop_timer = false; 159 160 rcu_read_lock(); 161 162 spin_lock_bh(&msk->pm.lock); 163 entry = mptcp_pm_announced_lookup(msk, addr); 164 if (entry && (!check_id || entry->addr.id == addr->id)) { 165 entry->retrans_times = ADD_ADDR_RETRANS_MAX; 166 stop_timer = true; 167 } 168 if (!check_id && entry) 169 list_del(&entry->list); 170 spin_unlock_bh(&msk->pm.lock); 171 172 /* Note: entry might have been removed by another thread. 173 * We hold rcu_read_lock() to ensure it is not freed under us. 174 */ 175 if (stop_timer) { 176 if (check_id) 177 sk_stop_timer(sk, &entry->timer); 178 else 179 sk_stop_timer_sync(sk, &entry->timer); 180 } 181 182 rcu_read_unlock(); 183 return entry; 184 } 185 186 bool mptcp_pm_announced_remove(struct mptcp_sock *msk, 187 const struct mptcp_addr_info *addr) 188 { 189 struct mptcp_pm_add_addr *entry; 190 bool ret; 191 192 entry = mptcp_pm_announced_del_timer(msk, addr, false); 193 ret = entry; 194 kfree_rcu(entry, rcu); 195 196 return ret; 197 } 198 199 bool mptcp_pm_announced_has_ssk(struct mptcp_sock *msk, const struct sock *ssk) 200 { 201 struct mptcp_pm_add_addr *entry; 202 struct mptcp_addr_info saddr; 203 bool ret = false; 204 205 mptcp_local_address((struct sock_common *)ssk, &saddr); 206 207 spin_lock_bh(&msk->pm.lock); 208 list_for_each_entry(entry, &msk->pm.anno_list, list) { 209 if (mptcp_addresses_equal(&entry->addr, &saddr, true)) { 210 ret = true; 211 goto out; 212 } 213 } 214 215 out: 216 spin_unlock_bh(&msk->pm.lock); 217 return ret; 218 } 219 220 static void __mptcp_pm_send_ack(struct mptcp_sock *msk, 221 struct mptcp_subflow_context *subflow, 222 bool prio, bool backup) 223 { 224 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 225 bool slow; 226 227 pr_debug("send ack for %s\n", 228 prio ? "mp_prio" : 229 (mptcp_pm_should_add_signal(msk) ? "add_addr" : "rm_addr")); 230 231 slow = lock_sock_fast(ssk); 232 if (prio) { 233 subflow->send_mp_prio = 1; 234 subflow->request_bkup = backup; 235 } 236 237 __mptcp_subflow_send_ack(ssk); 238 unlock_sock_fast(ssk, slow); 239 } 240 241 void mptcp_pm_send_ack(struct mptcp_sock *msk, 242 struct mptcp_subflow_context *subflow, 243 bool prio, bool backup) 244 { 245 spin_unlock_bh(&msk->pm.lock); 246 __mptcp_pm_send_ack(msk, subflow, prio, backup); 247 spin_lock_bh(&msk->pm.lock); 248 } 249 250 static bool subflow_in_rm_list(const struct mptcp_subflow_context *subflow, 251 const struct mptcp_rm_list *rm_list) 252 { 253 u8 i, id = subflow_get_local_id(subflow); 254 255 for (i = 0; i < rm_list->nr; i++) { 256 if (rm_list->ids[i] == id) 257 return true; 258 } 259 260 return false; 261 } 262 263 static void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk); 264 static void 265 mptcp_pm_addr_send_ack_avoid_list(struct mptcp_sock *msk, 266 const struct mptcp_rm_list *rm_list) 267 { 268 struct mptcp_subflow_context *subflow, *stale = NULL, *same_id = NULL; 269 270 msk_owned_by_me(msk); 271 lockdep_assert_held(&msk->pm.lock); 272 273 if (!mptcp_pm_should_add_signal(msk) && 274 !mptcp_pm_should_rm_signal(msk)) 275 return; 276 277 mptcp_for_each_subflow(msk, subflow) { 278 if (!__mptcp_subflow_active(subflow)) 279 continue; 280 281 if (unlikely(subflow->stale)) { 282 if (!stale) 283 stale = subflow; 284 } else if (unlikely(rm_list && 285 subflow_in_rm_list(subflow, rm_list))) { 286 if (!same_id) 287 same_id = subflow; 288 } else { 289 goto send_ack; 290 } 291 } 292 293 if (same_id) 294 subflow = same_id; 295 else if (stale) 296 subflow = stale; 297 else 298 return; 299 300 send_ack: 301 mptcp_pm_send_ack(msk, subflow, false, false); 302 } 303 304 void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) 305 { 306 mptcp_pm_addr_send_ack_avoid_list(msk, NULL); 307 } 308 309 int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, 310 struct mptcp_addr_info *addr, 311 struct mptcp_addr_info *rem, 312 u8 bkup) 313 { 314 struct mptcp_subflow_context *subflow; 315 316 pr_debug("bkup=%d\n", bkup); 317 318 mptcp_for_each_subflow(msk, subflow) { 319 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 320 struct mptcp_addr_info local, remote; 321 322 if (!__mptcp_subflow_active(subflow)) 323 continue; 324 325 mptcp_local_address((struct sock_common *)ssk, &local); 326 if (!mptcp_addresses_equal(&local, addr, addr->port)) 327 continue; 328 329 if (rem && rem->family != AF_UNSPEC) { 330 mptcp_remote_address((struct sock_common *)ssk, &remote); 331 if (!mptcp_addresses_equal(&remote, rem, rem->port)) 332 continue; 333 } 334 335 __mptcp_pm_send_ack(msk, subflow, true, bkup); 336 return 0; 337 } 338 339 return -EINVAL; 340 } 341 342 static unsigned int mptcp_adjust_add_addr_timeout(struct mptcp_sock *msk) 343 { 344 const struct net *net = sock_net((struct sock *)msk); 345 unsigned int rto = mptcp_get_add_addr_timeout(net); 346 struct mptcp_subflow_context *subflow; 347 unsigned int max = 0, max_stale = 0; 348 349 if (!rto) 350 return 0; 351 352 mptcp_for_each_subflow(msk, subflow) { 353 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 354 struct inet_connection_sock *icsk = inet_csk(ssk); 355 356 if (!__mptcp_subflow_active(subflow)) 357 continue; 358 359 if (unlikely(subflow->stale)) { 360 if (icsk->icsk_rto > max_stale) 361 max_stale = icsk->icsk_rto; 362 } else if (icsk->icsk_rto > max) { 363 max = icsk->icsk_rto; 364 } 365 } 366 367 if (max) 368 return min(max, rto); 369 370 if (max_stale) 371 return min(max_stale, rto); 372 373 return rto; 374 } 375 376 static void mptcp_pm_add_addr_timer(struct timer_list *timer) 377 { 378 struct mptcp_pm_add_addr *entry = timer_container_of(entry, timer, 379 timer); 380 struct mptcp_sock *msk = entry->sock; 381 struct sock *sk = (struct sock *)msk; 382 unsigned int timeout = 0; 383 bool retransmit; 384 385 pr_debug("msk=%p\n", msk); 386 387 bh_lock_sock(sk); 388 if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 389 goto out; 390 391 if (sock_owned_by_user(sk)) { 392 /* Try again later. */ 393 timeout = HZ / 20; 394 goto out; 395 } 396 397 timeout = mptcp_adjust_add_addr_timeout(msk); 398 if (!timeout || mptcp_pm_should_add_signal_addr(msk)) 399 goto out; 400 401 spin_lock_bh(&msk->pm.lock); 402 403 /* The cancel path (mptcp_pm_announced_del_timer()) can race with this 404 * callback. Once cancel updates retrans_times to MAX, suppress further 405 * retransmissions here. If this callback acquires pm.lock first, one 406 * final transmit attempt is still possible. 407 */ 408 if (entry->retrans_times < ADD_ADDR_RETRANS_MAX && 409 !mptcp_pm_should_add_signal_addr(msk)) { 410 pr_debug("retransmit ADD_ADDR id=%d\n", entry->addr.id); 411 mptcp_pm_announce_addr(msk, &entry->addr, false); 412 mptcp_pm_add_addr_send_ack(msk); 413 entry->retrans_times++; 414 } 415 416 retransmit = entry->retrans_times < ADD_ADDR_RETRANS_MAX; 417 if (retransmit) 418 timeout <<= entry->retrans_times; 419 else 420 timeout = 0; 421 422 spin_unlock_bh(&msk->pm.lock); 423 424 if (!retransmit) 425 mptcp_pm_subflow_established(msk); 426 427 out: 428 if (timeout) 429 sk_reset_timer(sk, timer, jiffies + timeout); 430 else 431 /* if sock_put calls sk_free: avoid waiting for this timer */ 432 entry->timer_done = true; 433 bh_unlock_sock(sk); 434 sock_put(sk); 435 } 436 437 bool mptcp_pm_announced_alloc(struct mptcp_sock *msk, 438 const struct mptcp_addr_info *addr) 439 { 440 struct mptcp_pm_add_addr *add_entry = NULL; 441 struct sock *sk = (struct sock *)msk; 442 unsigned int timeout; 443 444 lockdep_assert_held(&msk->pm.lock); 445 446 if (msk->pm.status & BIT(MPTCP_PM_DESTROYING)) 447 return false; 448 449 add_entry = mptcp_pm_announced_lookup(msk, addr); 450 if (add_entry) { 451 if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk))) 452 return false; 453 454 goto reset_timer; 455 } 456 457 add_entry = kmalloc_obj(*add_entry, GFP_ATOMIC); 458 if (!add_entry) 459 return false; 460 461 list_add(&add_entry->list, &msk->pm.anno_list); 462 463 add_entry->addr = *addr; 464 add_entry->sock = msk; 465 add_entry->retrans_times = 0; 466 467 timer_setup(&add_entry->timer, mptcp_pm_add_addr_timer, 0); 468 reset_timer: 469 add_entry->timer_done = false; 470 timeout = mptcp_adjust_add_addr_timeout(msk); 471 if (timeout) 472 sk_reset_timer(sk, &add_entry->timer, jiffies + timeout); 473 474 return true; 475 } 476 477 static void mptcp_pm_free_announced_list(struct mptcp_sock *msk) 478 { 479 struct mptcp_pm_add_addr *entry, *tmp; 480 struct sock *sk = (struct sock *)msk; 481 LIST_HEAD(free_list); 482 483 pr_debug("msk=%p\n", msk); 484 485 spin_lock_bh(&msk->pm.lock); 486 list_splice_init(&msk->pm.anno_list, &free_list); 487 spin_unlock_bh(&msk->pm.lock); 488 489 list_for_each_entry_safe(entry, tmp, &free_list, list) { 490 if (!entry->timer_done) 491 sk_stop_timer_sync(sk, &entry->timer); 492 kfree_rcu(entry, rcu); 493 } 494 } 495 496 /* path manager command handlers */ 497 498 int mptcp_pm_announce_addr(struct mptcp_sock *msk, 499 const struct mptcp_addr_info *addr, 500 bool echo) 501 { 502 u8 add_addr = READ_ONCE(msk->pm.addr_signal); 503 504 pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo); 505 506 lockdep_assert_held(&msk->pm.lock); 507 508 if (add_addr & 509 (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) { 510 MPTCP_INC_STATS(sock_net((struct sock *)msk), 511 echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP); 512 return -EINVAL; 513 } 514 515 if (echo) { 516 msk->pm.remote = *addr; 517 add_addr |= BIT(MPTCP_ADD_ADDR_ECHO); 518 } else { 519 msk->pm.local = *addr; 520 add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL); 521 } 522 WRITE_ONCE(msk->pm.addr_signal, add_addr); 523 return 0; 524 } 525 526 int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list) 527 { 528 u8 rm_addr = READ_ONCE(msk->pm.addr_signal); 529 530 pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr); 531 532 if (rm_addr) { 533 MPTCP_ADD_STATS(sock_net((struct sock *)msk), 534 MPTCP_MIB_RMADDRTXDROP, rm_list->nr); 535 return -EINVAL; 536 } 537 538 msk->pm.rm_list_tx = *rm_list; 539 rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL); 540 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 541 mptcp_pm_addr_send_ack_avoid_list(msk, rm_list); 542 return 0; 543 } 544 545 /* path manager event handlers */ 546 547 void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side) 548 { 549 struct mptcp_pm_data *pm = &msk->pm; 550 551 pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side); 552 553 WRITE_ONCE(pm->server_side, server_side); 554 mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC); 555 } 556 557 bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk) 558 { 559 struct mptcp_pm_data *pm = &msk->pm; 560 unsigned int limit_extra_subflows; 561 int ret = 0; 562 563 if (mptcp_pm_is_userspace(msk)) { 564 if (mptcp_userspace_pm_active(msk)) { 565 spin_lock_bh(&pm->lock); 566 pm->extra_subflows++; 567 spin_unlock_bh(&pm->lock); 568 return true; 569 } 570 return false; 571 } 572 573 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); 574 575 pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk, 576 pm->extra_subflows, limit_extra_subflows, 577 READ_ONCE(pm->accept_subflow)); 578 579 /* try to avoid acquiring the lock below */ 580 if (!READ_ONCE(pm->accept_subflow)) 581 return false; 582 583 spin_lock_bh(&pm->lock); 584 if (READ_ONCE(pm->accept_subflow)) { 585 ret = pm->extra_subflows < limit_extra_subflows; 586 if (ret && ++pm->extra_subflows == limit_extra_subflows) 587 WRITE_ONCE(pm->accept_subflow, false); 588 } 589 spin_unlock_bh(&pm->lock); 590 591 return ret; 592 } 593 594 /* return true if the new status bit is currently cleared, that is, this event 595 * can be server, eventually by an already scheduled work 596 */ 597 static bool mptcp_pm_schedule_work(struct mptcp_sock *msk, 598 enum mptcp_pm_status new_status) 599 { 600 pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status, 601 BIT(new_status)); 602 if (msk->pm.status & BIT(new_status)) 603 return false; 604 605 msk->pm.status |= BIT(new_status); 606 mptcp_schedule_work((struct sock *)msk); 607 return true; 608 } 609 610 void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk) 611 { 612 struct mptcp_pm_data *pm = &msk->pm; 613 bool announce = false; 614 615 pr_debug("msk=%p\n", msk); 616 617 spin_lock_bh(&pm->lock); 618 619 /* mptcp_pm_fully_established() can be invoked by multiple 620 * racing paths - accept() and check_fully_established() 621 * be sure to serve this event only once. 622 */ 623 if (READ_ONCE(pm->work_pending) && 624 !(pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED))) 625 mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED); 626 627 if ((pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0) 628 announce = true; 629 630 pm->status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED); 631 spin_unlock_bh(&pm->lock); 632 633 if (announce) 634 mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC); 635 } 636 637 void mptcp_pm_connection_closed(struct mptcp_sock *msk) 638 { 639 pr_debug("msk=%p\n", msk); 640 641 if (msk->token) 642 mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 643 } 644 645 void mptcp_pm_subflow_established(struct mptcp_sock *msk) 646 { 647 struct mptcp_pm_data *pm = &msk->pm; 648 649 pr_debug("msk=%p\n", msk); 650 651 if (!READ_ONCE(pm->work_pending)) 652 return; 653 654 spin_lock_bh(&pm->lock); 655 656 if (READ_ONCE(pm->work_pending)) 657 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 658 659 spin_unlock_bh(&pm->lock); 660 } 661 662 void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, 663 const struct mptcp_subflow_context *subflow) 664 { 665 struct sock *sk = (struct sock *)msk; 666 struct mptcp_pm_data *pm = &msk->pm; 667 bool update_subflows; 668 669 update_subflows = subflow->request_join || subflow->mp_join; 670 if (mptcp_pm_is_userspace(msk)) { 671 if (update_subflows) { 672 spin_lock_bh(&pm->lock); 673 if (!WARN_ON_ONCE(pm->extra_subflows == 0)) 674 pm->extra_subflows--; 675 spin_unlock_bh(&pm->lock); 676 } 677 return; 678 } 679 680 if (!READ_ONCE(pm->work_pending) && !update_subflows) 681 return; 682 683 spin_lock_bh(&pm->lock); 684 if (update_subflows) 685 __mptcp_pm_close_subflow(msk); 686 687 /* Even if this subflow is not really established, tell the PM to try 688 * to pick the next ones, if possible. 689 */ 690 if (mptcp_is_fully_established(sk) && 691 mptcp_pm_nl_check_work_pending(msk)) 692 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 693 694 spin_unlock_bh(&pm->lock); 695 } 696 697 void mptcp_pm_add_addr_received(const struct sock *ssk, 698 const struct mptcp_addr_info *addr) 699 { 700 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 701 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 702 struct mptcp_pm_data *pm = &msk->pm; 703 704 pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id, 705 READ_ONCE(pm->accept_addr)); 706 707 mptcp_event_addr_announced(ssk, addr); 708 709 spin_lock_bh(&pm->lock); 710 711 if (mptcp_pm_is_userspace(msk)) { 712 if (mptcp_userspace_pm_active(msk)) { 713 mptcp_pm_announce_addr(msk, addr, true); 714 mptcp_pm_add_addr_send_ack(msk); 715 } else { 716 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); 717 } 718 /* - id0 should not have a different address 719 * - special case for C-flag: linked to fill_local_addresses_vec() 720 */ 721 } else if ((addr->id == 0 && !mptcp_pm_is_init_remote_addr(msk, addr)) || 722 (addr->id > 0 && !READ_ONCE(pm->accept_addr) && 723 !mptcp_pm_add_addr_c_flag_case(msk))) { 724 mptcp_pm_announce_addr(msk, addr, true); 725 mptcp_pm_add_addr_send_ack(msk); 726 } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) { 727 pm->remote = *addr; 728 } else { 729 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); 730 } 731 732 spin_unlock_bh(&pm->lock); 733 } 734 735 void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, 736 const struct mptcp_addr_info *addr) 737 { 738 struct mptcp_pm_data *pm = &msk->pm; 739 struct mptcp_pm_add_addr *entry; 740 741 pr_debug("msk=%p\n", msk); 742 743 entry = mptcp_pm_announced_del_timer(msk, addr, true); 744 745 if (!entry || !READ_ONCE(pm->work_pending)) 746 return; 747 748 spin_lock_bh(&pm->lock); 749 750 if (READ_ONCE(pm->work_pending)) 751 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 752 753 spin_unlock_bh(&pm->lock); 754 } 755 756 /* To be called while pm->lock is held */ 757 static void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk) 758 { 759 if (!mptcp_pm_should_add_signal(msk)) 760 return; 761 762 mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK); 763 } 764 765 static void mptcp_pm_rm_addr_or_subflow(struct mptcp_sock *msk, 766 const struct mptcp_rm_list *rm_list, 767 enum linux_mptcp_mib_field rm_type) 768 { 769 struct mptcp_subflow_context *subflow, *tmp; 770 struct sock *sk = (struct sock *)msk; 771 u8 i; 772 773 pr_debug("%s rm_list_nr %d\n", 774 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", rm_list->nr); 775 776 msk_owned_by_me(msk); 777 778 if (sk->sk_state == TCP_LISTEN) 779 return; 780 781 if (!rm_list->nr) 782 return; 783 784 if (list_empty(&msk->conn_list)) 785 return; 786 787 for (i = 0; i < rm_list->nr; i++) { 788 u8 rm_id = rm_list->ids[i]; 789 bool removed = false; 790 791 mptcp_for_each_subflow_safe(msk, subflow, tmp) { 792 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 793 u8 remote_id = READ_ONCE(subflow->remote_id); 794 int how = RCV_SHUTDOWN | SEND_SHUTDOWN; 795 u8 id = subflow_get_local_id(subflow); 796 797 if ((1 << inet_sk_state_load(ssk)) & 798 (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING | TCPF_CLOSE)) 799 continue; 800 if (rm_type == MPTCP_MIB_RMADDR && remote_id != rm_id) 801 continue; 802 if (rm_type == MPTCP_MIB_RMSUBFLOW && id != rm_id) 803 continue; 804 805 pr_debug(" -> %s rm_list_ids[%d]=%u local_id=%u remote_id=%u mpc_id=%u\n", 806 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", 807 i, rm_id, id, remote_id, msk->mpc_endpoint_id); 808 spin_unlock_bh(&msk->pm.lock); 809 mptcp_subflow_shutdown(sk, ssk, how); 810 removed |= subflow->request_join; 811 812 /* the following takes care of updating the subflows counter */ 813 mptcp_close_ssk(sk, ssk, subflow); 814 spin_lock_bh(&msk->pm.lock); 815 816 if (rm_type == MPTCP_MIB_RMSUBFLOW) 817 __MPTCP_INC_STATS(sock_net(sk), rm_type); 818 } 819 820 if (rm_type == MPTCP_MIB_RMADDR) { 821 __MPTCP_INC_STATS(sock_net(sk), rm_type); 822 if (removed && mptcp_pm_is_kernel(msk)) 823 mptcp_pm_nl_rm_addr(msk, rm_id); 824 } 825 } 826 } 827 828 static void mptcp_pm_rm_addr_recv(struct mptcp_sock *msk) 829 { 830 mptcp_pm_rm_addr_or_subflow(msk, &msk->pm.rm_list_rx, MPTCP_MIB_RMADDR); 831 } 832 833 void mptcp_pm_rm_subflow(struct mptcp_sock *msk, 834 const struct mptcp_rm_list *rm_list) 835 { 836 mptcp_pm_rm_addr_or_subflow(msk, rm_list, MPTCP_MIB_RMSUBFLOW); 837 } 838 839 void mptcp_pm_rm_addr_received(struct mptcp_sock *msk, 840 const struct mptcp_rm_list *rm_list) 841 { 842 struct mptcp_pm_data *pm = &msk->pm; 843 u8 i; 844 845 pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr); 846 847 for (i = 0; i < rm_list->nr; i++) 848 mptcp_event_addr_removed(msk, rm_list->ids[i]); 849 850 spin_lock_bh(&pm->lock); 851 if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED)) 852 pm->rm_list_rx = *rm_list; 853 else 854 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP); 855 spin_unlock_bh(&pm->lock); 856 } 857 858 void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup) 859 { 860 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 861 struct sock *sk = subflow->conn; 862 struct mptcp_sock *msk; 863 864 pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup); 865 msk = mptcp_sk(sk); 866 if (subflow->backup != bkup) 867 subflow->backup = bkup; 868 869 mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC); 870 } 871 872 void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq) 873 { 874 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 875 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 876 877 pr_debug("fail_seq=%llu\n", fail_seq); 878 879 /* After accepting the fail, we can't create any other subflows */ 880 spin_lock_bh(&msk->fallback_lock); 881 if (!msk->allow_infinite_fallback) { 882 spin_unlock_bh(&msk->fallback_lock); 883 return; 884 } 885 msk->allow_subflows = false; 886 spin_unlock_bh(&msk->fallback_lock); 887 888 if (!subflow->fail_tout) { 889 pr_debug("send MP_FAIL response and infinite map\n"); 890 891 subflow->send_mp_fail = 1; 892 subflow->send_infinite_map = 1; 893 tcp_send_ack(sk); 894 } else { 895 pr_debug("MP_FAIL response received\n"); 896 WRITE_ONCE(subflow->fail_tout, 0); 897 } 898 } 899 900 static int mptcp_add_addr_len(int family, bool echo, bool port) 901 { 902 int len = TCPOLEN_MPTCP_ADD_ADDR_BASE; 903 904 if (family == AF_INET6) 905 len = TCPOLEN_MPTCP_ADD_ADDR6_BASE; 906 if (!echo) 907 len += MPTCPOPT_THMAC_LEN; 908 /* account for 2 trailing 'nop' options */ 909 if (port) 910 len += TCPOLEN_MPTCP_PORT_LEN + TCPOLEN_MPTCP_PORT_ALIGN; 911 912 return len; 913 } 914 915 bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, int *size, int remaining, 916 struct mptcp_addr_info *addr, bool *echo, 917 bool *drop_ts) 918 { 919 bool skip_add_addr = false; 920 bool ret = false; 921 u8 add_addr; 922 int len = 0; 923 u8 family; 924 bool port; 925 926 spin_lock_bh(&msk->pm.lock); 927 928 /* double check after the lock is acquired */ 929 if (!mptcp_pm_should_add_signal(msk)) 930 goto out_unlock; 931 932 /* always drop every other options for pure ack ADD_ADDR; this is a 933 * plain dup-ack from TCP perspective. The other MPTCP-relevant info, 934 * if any, will be carried by the 'original' TCP ack 935 */ 936 len -= *size; 937 938 *echo = mptcp_pm_should_add_signal_echo(msk); 939 if (*echo) { 940 *addr = msk->pm.remote; 941 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO); 942 port = !!msk->pm.remote.port; 943 family = msk->pm.remote.family; 944 } else { 945 *addr = msk->pm.local; 946 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL); 947 port = !!msk->pm.local.port; 948 family = msk->pm.local.family; 949 } 950 951 len += mptcp_add_addr_len(family, *echo, port); 952 if (len > remaining) { 953 struct net *net = sock_net((struct sock *)msk); 954 955 if (*drop_ts && mptcp_add_addr_v6_port_drop_ts(net)) { 956 /* OK without TCP Timestamps? */ 957 len -= TCPOLEN_TSTAMP_ALIGNED; 958 if (len <= remaining) 959 goto enough_space; 960 } 961 962 if (*echo) { 963 MPTCP_INC_STATS(net, MPTCP_MIB_ECHOADDTXDROP); 964 } else { 965 skip_add_addr = true; 966 MPTCP_INC_STATS(net, MPTCP_MIB_ADDADDRTXDROP); 967 } 968 goto drop_signal_mark; 969 } 970 971 *drop_ts = false; 972 973 enough_space: 974 ret = true; 975 *size = len; 976 977 drop_signal_mark: 978 WRITE_ONCE(msk->pm.addr_signal, add_addr); 979 980 out_unlock: 981 spin_unlock_bh(&msk->pm.lock); 982 983 /* On pure-ACK option-space exhaustion, stop retrying this ADD_ADDR: 984 * clear the signal bit, cancel the matching retransmission timer, and 985 * let the PM state machine progress. 986 */ 987 if (skip_add_addr) { 988 mptcp_pm_announced_del_timer(msk, addr, true); 989 mptcp_pm_subflow_established(msk); 990 } 991 return ret; 992 } 993 994 static int mptcp_rm_addr_len(const struct mptcp_rm_list *rm_list) 995 { 996 if (rm_list->nr == 0 || rm_list->nr > MPTCP_RM_IDS_MAX) 997 return -EINVAL; 998 999 return TCPOLEN_MPTCP_RM_ADDR_BASE + roundup(rm_list->nr - 1, 4) + 1; 1000 } 1001 1002 bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining, 1003 struct mptcp_rm_list *rm_list, int *size) 1004 { 1005 int ret = false, len; 1006 u8 rm_addr; 1007 1008 spin_lock_bh(&msk->pm.lock); 1009 1010 /* double check after the lock is acquired */ 1011 if (!mptcp_pm_should_rm_signal(msk)) 1012 goto out_unlock; 1013 1014 rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL); 1015 len = mptcp_rm_addr_len(&msk->pm.rm_list_tx); 1016 if (len < 0) { 1017 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 1018 goto out_unlock; 1019 } 1020 if (remaining < len) 1021 goto out_unlock; 1022 1023 *size = len; 1024 *rm_list = msk->pm.rm_list_tx; 1025 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 1026 ret = true; 1027 1028 out_unlock: 1029 spin_unlock_bh(&msk->pm.lock); 1030 return ret; 1031 } 1032 1033 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc) 1034 { 1035 struct mptcp_pm_addr_entry skc_local = { 0 }; 1036 struct mptcp_addr_info msk_local; 1037 1038 if (WARN_ON_ONCE(!msk)) 1039 return -1; 1040 1041 /* The 0 ID mapping is defined by the first subflow, copied into the msk 1042 * addr 1043 */ 1044 mptcp_local_address((struct sock_common *)msk, &msk_local); 1045 mptcp_local_address((struct sock_common *)skc, &skc_local.addr); 1046 if (mptcp_addresses_equal(&msk_local, &skc_local.addr, false)) 1047 return 0; 1048 1049 skc_local.addr.id = 0; 1050 skc_local.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT; 1051 1052 if (mptcp_pm_is_userspace(msk)) 1053 return mptcp_userspace_pm_get_local_id(msk, &skc_local); 1054 return mptcp_pm_nl_get_local_id(msk, &skc_local); 1055 } 1056 1057 bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc) 1058 { 1059 struct mptcp_addr_info skc_local; 1060 1061 mptcp_local_address((struct sock_common *)skc, &skc_local); 1062 1063 if (mptcp_pm_is_userspace(msk)) 1064 return mptcp_userspace_pm_is_backup(msk, &skc_local); 1065 1066 return mptcp_pm_nl_is_backup(msk, &skc_local); 1067 } 1068 1069 static void 1070 mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) 1071 { 1072 struct mptcp_subflow_context *iter, *subflow = mptcp_subflow_ctx(ssk); 1073 struct sock *sk = (struct sock *)msk; 1074 unsigned int active_max_loss_cnt; 1075 struct net *net = sock_net(sk); 1076 unsigned int stale_loss_cnt; 1077 bool slow; 1078 1079 stale_loss_cnt = mptcp_stale_loss_cnt(net); 1080 if (subflow->stale || !stale_loss_cnt || subflow->stale_count <= stale_loss_cnt) 1081 return; 1082 1083 /* look for another available subflow not in loss state */ 1084 active_max_loss_cnt = max_t(int, stale_loss_cnt - 1, 1); 1085 mptcp_for_each_subflow(msk, iter) { 1086 if (iter != subflow && mptcp_subflow_active(iter) && 1087 iter->stale_count < active_max_loss_cnt) { 1088 /* we have some alternatives, try to mark this subflow as idle ...*/ 1089 slow = lock_sock_fast(ssk); 1090 if (!tcp_rtx_and_write_queues_empty(ssk)) { 1091 subflow->stale = 1; 1092 __mptcp_retransmit_pending_data(sk); 1093 MPTCP_INC_STATS(net, MPTCP_MIB_SUBFLOWSTALE); 1094 } 1095 unlock_sock_fast(ssk, slow); 1096 1097 /* always try to push the pending data regardless of re-injections: 1098 * we can possibly use backup subflows now, and subflow selection 1099 * is cheap under the msk socket lock 1100 */ 1101 __mptcp_push_pending(sk, 0); 1102 return; 1103 } 1104 } 1105 } 1106 1107 void mptcp_pm_chk_stale(const struct mptcp_sock *msk) 1108 { 1109 struct mptcp_subflow_context *subflow; 1110 1111 mptcp_for_each_subflow(msk, subflow) { 1112 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1113 u32 rcv_tstamp; 1114 1115 if (!__mptcp_subflow_active(subflow)) 1116 continue; 1117 1118 /* No data outstanding at TCP level? not stale */ 1119 if (tcp_rtx_and_write_queues_empty(ssk)) 1120 continue; 1121 1122 /* keep track of rtx periods with no progress */ 1123 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); 1124 if (!subflow->stale_count) { 1125 subflow->stale_rcv_tstamp = rcv_tstamp; 1126 subflow->stale_count++; 1127 } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { 1128 if (subflow->stale_count < U8_MAX) 1129 subflow->stale_count++; 1130 mptcp_pm_subflow_chk_stale(msk, ssk); 1131 } else { 1132 subflow->stale_count = 0; 1133 mptcp_subflow_set_active(subflow); 1134 } 1135 } 1136 } 1137 1138 void mptcp_pm_worker(struct mptcp_sock *msk) 1139 { 1140 struct mptcp_pm_data *pm = &msk->pm; 1141 1142 msk_owned_by_me(msk); 1143 1144 if (!(pm->status & MPTCP_PM_WORK_MASK)) 1145 return; 1146 1147 spin_lock_bh(&msk->pm.lock); 1148 1149 pr_debug("msk=%p status=%x\n", msk, pm->status); 1150 if (pm->status & BIT(MPTCP_PM_ADD_ADDR_SEND_ACK)) { 1151 pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_SEND_ACK); 1152 mptcp_pm_addr_send_ack(msk); 1153 } 1154 if (pm->status & BIT(MPTCP_PM_RM_ADDR_RECEIVED)) { 1155 pm->status &= ~BIT(MPTCP_PM_RM_ADDR_RECEIVED); 1156 mptcp_pm_rm_addr_recv(msk); 1157 } 1158 __mptcp_pm_kernel_worker(msk); 1159 1160 spin_unlock_bh(&msk->pm.lock); 1161 } 1162 1163 void mptcp_pm_destroy(struct mptcp_sock *msk) 1164 { 1165 spin_lock_bh(&msk->pm.lock); 1166 msk->pm.status |= BIT(MPTCP_PM_DESTROYING); 1167 spin_unlock_bh(&msk->pm.lock); 1168 1169 mptcp_pm_free_announced_list(msk); 1170 1171 /* Free the userspace local address list unconditionally: the socket 1172 * can be reused (mptcp_disconnect()) and re-selected to a different PM 1173 */ 1174 mptcp_userspace_pm_free_local_addr_list(msk); 1175 } 1176 1177 void mptcp_pm_data_reset(struct mptcp_sock *msk) 1178 { 1179 u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk)); 1180 struct mptcp_pm_data *pm = &msk->pm; 1181 1182 memset(&pm->reset, 0, sizeof(pm->reset)); 1183 pm->rm_list_tx.nr = 0; 1184 pm->rm_list_rx.nr = 0; 1185 WRITE_ONCE(pm->pm_type, pm_type); 1186 1187 if (pm_type == MPTCP_PM_TYPE_KERNEL) { 1188 bool subflows_allowed = !!mptcp_pm_get_limit_extra_subflows(msk); 1189 1190 /* pm->work_pending must be only be set to 'true' when 1191 * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL 1192 */ 1193 WRITE_ONCE(pm->work_pending, 1194 (!!mptcp_pm_get_endp_subflow_max(msk) && 1195 subflows_allowed) || 1196 !!mptcp_pm_get_endp_signal_max(msk)); 1197 WRITE_ONCE(pm->accept_addr, 1198 !!mptcp_pm_get_limit_add_addr_accepted(msk) && 1199 subflows_allowed); 1200 WRITE_ONCE(pm->accept_subflow, subflows_allowed); 1201 1202 bitmap_fill(pm->id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); 1203 } 1204 } 1205 1206 void mptcp_pm_data_init(struct mptcp_sock *msk) 1207 { 1208 spin_lock_init(&msk->pm.lock); 1209 INIT_LIST_HEAD(&msk->pm.anno_list); 1210 INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list); 1211 mptcp_pm_data_reset(msk); 1212 } 1213 1214 void __init mptcp_pm_init(void) 1215 { 1216 mptcp_pm_kernel_register(); 1217 mptcp_pm_userspace_register(); 1218 mptcp_pm_nl_init(); 1219 } 1220 1221 /* Must be called with rcu read lock held */ 1222 struct mptcp_pm_ops *mptcp_pm_find(const char *name) 1223 { 1224 struct mptcp_pm_ops *pm_ops; 1225 1226 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) { 1227 if (!strcmp(pm_ops->name, name)) 1228 return pm_ops; 1229 } 1230 1231 return NULL; 1232 } 1233 1234 int mptcp_pm_validate(struct mptcp_pm_ops *pm_ops) 1235 { 1236 return 0; 1237 } 1238 1239 int mptcp_pm_register(struct mptcp_pm_ops *pm_ops) 1240 { 1241 int ret; 1242 1243 ret = mptcp_pm_validate(pm_ops); 1244 if (ret) 1245 return ret; 1246 1247 spin_lock(&mptcp_pm_list_lock); 1248 if (mptcp_pm_find(pm_ops->name)) { 1249 spin_unlock(&mptcp_pm_list_lock); 1250 return -EEXIST; 1251 } 1252 list_add_tail_rcu(&pm_ops->list, &mptcp_pm_list); 1253 spin_unlock(&mptcp_pm_list_lock); 1254 1255 pr_debug("%s registered\n", pm_ops->name); 1256 return 0; 1257 } 1258 1259 void mptcp_pm_unregister(struct mptcp_pm_ops *pm_ops) 1260 { 1261 /* skip unregistering the default path manager */ 1262 if (WARN_ON_ONCE(pm_ops == &mptcp_pm_kernel)) 1263 return; 1264 1265 spin_lock(&mptcp_pm_list_lock); 1266 list_del_rcu(&pm_ops->list); 1267 spin_unlock(&mptcp_pm_list_lock); 1268 } 1269 1270 /* Build string with list of available path manager values. 1271 * Similar to tcp_get_available_congestion_control() 1272 */ 1273 void mptcp_pm_get_available(char *buf, size_t maxlen) 1274 { 1275 struct mptcp_pm_ops *pm_ops; 1276 size_t offs = 0; 1277 1278 rcu_read_lock(); 1279 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) { 1280 offs += snprintf(buf + offs, maxlen - offs, "%s%s", 1281 offs == 0 ? "" : " ", pm_ops->name); 1282 1283 if (WARN_ON_ONCE(offs >= maxlen)) 1284 break; 1285 } 1286 rcu_read_unlock(); 1287 } 1288