1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2019, Intel Corporation. 5 */ 6 #define pr_fmt(fmt) "MPTCP: " fmt 7 8 #include <linux/rculist.h> 9 #include <linux/spinlock.h> 10 #include "protocol.h" 11 #include "mib.h" 12 13 #define ADD_ADDR_RETRANS_MAX 3 14 15 struct mptcp_pm_add_entry { 16 struct list_head list; 17 struct mptcp_addr_info addr; 18 u8 retrans_times; 19 struct timer_list add_timer; 20 struct mptcp_sock *sock; 21 struct rcu_head rcu; 22 }; 23 24 static DEFINE_SPINLOCK(mptcp_pm_list_lock); 25 static LIST_HEAD(mptcp_pm_list); 26 27 /* path manager helpers */ 28 29 /* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses, 30 * otherwise allow any matching local/remote pair 31 */ 32 bool mptcp_pm_addr_families_match(const struct sock *sk, 33 const struct mptcp_addr_info *loc, 34 const struct mptcp_addr_info *rem) 35 { 36 bool mptcp_is_v4 = sk->sk_family == AF_INET; 37 38 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 39 bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(&loc->addr6); 40 bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(&rem->addr6); 41 42 if (mptcp_is_v4) 43 return loc_is_v4 && rem_is_v4; 44 45 if (ipv6_only_sock(sk)) 46 return !loc_is_v4 && !rem_is_v4; 47 48 return loc_is_v4 == rem_is_v4; 49 #else 50 return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET; 51 #endif 52 } 53 54 bool mptcp_addresses_equal(const struct mptcp_addr_info *a, 55 const struct mptcp_addr_info *b, bool use_port) 56 { 57 bool addr_equals = false; 58 59 if (a->family == b->family) { 60 if (a->family == AF_INET) 61 addr_equals = a->addr.s_addr == b->addr.s_addr; 62 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 63 else 64 addr_equals = ipv6_addr_equal(&a->addr6, &b->addr6); 65 } else if (a->family == AF_INET) { 66 if (ipv6_addr_v4mapped(&b->addr6)) 67 addr_equals = a->addr.s_addr == b->addr6.s6_addr32[3]; 68 } else if (b->family == AF_INET) { 69 if (ipv6_addr_v4mapped(&a->addr6)) 70 addr_equals = a->addr6.s6_addr32[3] == b->addr.s_addr; 71 #endif 72 } 73 74 if (!addr_equals) 75 return false; 76 if (!use_port) 77 return true; 78 79 return a->port == b->port; 80 } 81 82 void mptcp_local_address(const struct sock_common *skc, 83 struct mptcp_addr_info *addr) 84 { 85 addr->family = skc->skc_family; 86 addr->port = htons(skc->skc_num); 87 if (addr->family == AF_INET) 88 addr->addr.s_addr = skc->skc_rcv_saddr; 89 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 90 else if (addr->family == AF_INET6) 91 addr->addr6 = skc->skc_v6_rcv_saddr; 92 #endif 93 } 94 95 void mptcp_remote_address(const struct sock_common *skc, 96 struct mptcp_addr_info *addr) 97 { 98 addr->family = skc->skc_family; 99 addr->port = skc->skc_dport; 100 if (addr->family == AF_INET) 101 addr->addr.s_addr = skc->skc_daddr; 102 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 103 else if (addr->family == AF_INET6) 104 addr->addr6 = skc->skc_v6_daddr; 105 #endif 106 } 107 108 static bool mptcp_pm_is_init_remote_addr(struct mptcp_sock *msk, 109 const struct mptcp_addr_info *remote) 110 { 111 struct mptcp_addr_info mpc_remote; 112 113 mptcp_remote_address((struct sock_common *)msk, &mpc_remote); 114 return mptcp_addresses_equal(&mpc_remote, remote, remote->port); 115 } 116 117 bool mptcp_lookup_subflow_by_saddr(const struct list_head *list, 118 const struct mptcp_addr_info *saddr) 119 { 120 struct mptcp_subflow_context *subflow; 121 struct mptcp_addr_info cur; 122 struct sock_common *skc; 123 124 list_for_each_entry(subflow, list, node) { 125 skc = (struct sock_common *)mptcp_subflow_tcp_sock(subflow); 126 127 mptcp_local_address(skc, &cur); 128 if (mptcp_addresses_equal(&cur, saddr, saddr->port)) 129 return true; 130 } 131 132 return false; 133 } 134 135 static struct mptcp_pm_add_entry * 136 mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, 137 const struct mptcp_addr_info *addr) 138 { 139 struct mptcp_pm_add_entry *entry; 140 141 lockdep_assert_held(&msk->pm.lock); 142 143 list_for_each_entry(entry, &msk->pm.anno_list, list) { 144 if (mptcp_addresses_equal(&entry->addr, addr, true)) 145 return entry; 146 } 147 148 return NULL; 149 } 150 151 bool mptcp_remove_anno_list_by_saddr(struct mptcp_sock *msk, 152 const struct mptcp_addr_info *addr) 153 { 154 struct mptcp_pm_add_entry *entry; 155 bool ret; 156 157 entry = mptcp_pm_del_add_timer(msk, addr, false); 158 ret = entry; 159 kfree_rcu(entry, rcu); 160 161 return ret; 162 } 163 164 bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk) 165 { 166 struct mptcp_pm_add_entry *entry; 167 struct mptcp_addr_info saddr; 168 bool ret = false; 169 170 mptcp_local_address((struct sock_common *)sk, &saddr); 171 172 spin_lock_bh(&msk->pm.lock); 173 list_for_each_entry(entry, &msk->pm.anno_list, list) { 174 if (mptcp_addresses_equal(&entry->addr, &saddr, true)) { 175 ret = true; 176 goto out; 177 } 178 } 179 180 out: 181 spin_unlock_bh(&msk->pm.lock); 182 return ret; 183 } 184 185 static void __mptcp_pm_send_ack(struct mptcp_sock *msk, 186 struct mptcp_subflow_context *subflow, 187 bool prio, bool backup) 188 { 189 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 190 bool slow; 191 192 pr_debug("send ack for %s\n", 193 prio ? "mp_prio" : 194 (mptcp_pm_should_add_signal(msk) ? "add_addr" : "rm_addr")); 195 196 slow = lock_sock_fast(ssk); 197 if (prio) { 198 subflow->send_mp_prio = 1; 199 subflow->request_bkup = backup; 200 } 201 202 __mptcp_subflow_send_ack(ssk); 203 unlock_sock_fast(ssk, slow); 204 } 205 206 void mptcp_pm_send_ack(struct mptcp_sock *msk, 207 struct mptcp_subflow_context *subflow, 208 bool prio, bool backup) 209 { 210 spin_unlock_bh(&msk->pm.lock); 211 __mptcp_pm_send_ack(msk, subflow, prio, backup); 212 spin_lock_bh(&msk->pm.lock); 213 } 214 215 void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) 216 { 217 struct mptcp_subflow_context *subflow, *alt = NULL; 218 219 msk_owned_by_me(msk); 220 lockdep_assert_held(&msk->pm.lock); 221 222 if (!mptcp_pm_should_add_signal(msk) && 223 !mptcp_pm_should_rm_signal(msk)) 224 return; 225 226 mptcp_for_each_subflow(msk, subflow) { 227 if (__mptcp_subflow_active(subflow)) { 228 if (!subflow->stale) { 229 mptcp_pm_send_ack(msk, subflow, false, false); 230 return; 231 } 232 233 if (!alt) 234 alt = subflow; 235 } 236 } 237 238 if (alt) 239 mptcp_pm_send_ack(msk, alt, false, false); 240 } 241 242 int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, 243 struct mptcp_addr_info *addr, 244 struct mptcp_addr_info *rem, 245 u8 bkup) 246 { 247 struct mptcp_subflow_context *subflow; 248 249 pr_debug("bkup=%d\n", bkup); 250 251 mptcp_for_each_subflow(msk, subflow) { 252 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 253 struct mptcp_addr_info local, remote; 254 255 mptcp_local_address((struct sock_common *)ssk, &local); 256 if (!mptcp_addresses_equal(&local, addr, addr->port)) 257 continue; 258 259 if (rem && rem->family != AF_UNSPEC) { 260 mptcp_remote_address((struct sock_common *)ssk, &remote); 261 if (!mptcp_addresses_equal(&remote, rem, rem->port)) 262 continue; 263 } 264 265 __mptcp_pm_send_ack(msk, subflow, true, bkup); 266 return 0; 267 } 268 269 return -EINVAL; 270 } 271 272 static unsigned int mptcp_adjust_add_addr_timeout(struct mptcp_sock *msk) 273 { 274 const struct net *net = sock_net((struct sock *)msk); 275 unsigned int rto = mptcp_get_add_addr_timeout(net); 276 struct mptcp_subflow_context *subflow; 277 unsigned int max = 0; 278 279 mptcp_for_each_subflow(msk, subflow) { 280 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 281 struct inet_connection_sock *icsk = inet_csk(ssk); 282 283 if (icsk->icsk_rto > max) 284 max = icsk->icsk_rto; 285 } 286 287 if (max && max < rto) 288 rto = max; 289 290 return rto; 291 } 292 293 static void mptcp_pm_add_timer(struct timer_list *timer) 294 { 295 struct mptcp_pm_add_entry *entry = timer_container_of(entry, timer, 296 add_timer); 297 struct mptcp_sock *msk = entry->sock; 298 struct sock *sk = (struct sock *)msk; 299 unsigned int timeout; 300 301 pr_debug("msk=%p\n", msk); 302 303 if (!msk) 304 return; 305 306 if (inet_sk_state_load(sk) == TCP_CLOSE) 307 return; 308 309 if (!entry->addr.id) 310 return; 311 312 if (mptcp_pm_should_add_signal_addr(msk)) { 313 sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8); 314 goto out; 315 } 316 317 timeout = mptcp_adjust_add_addr_timeout(msk); 318 if (!timeout) 319 goto out; 320 321 spin_lock_bh(&msk->pm.lock); 322 323 if (!mptcp_pm_should_add_signal_addr(msk)) { 324 pr_debug("retransmit ADD_ADDR id=%d\n", entry->addr.id); 325 mptcp_pm_announce_addr(msk, &entry->addr, false); 326 mptcp_pm_add_addr_send_ack(msk); 327 entry->retrans_times++; 328 } 329 330 if (entry->retrans_times < ADD_ADDR_RETRANS_MAX) 331 sk_reset_timer(sk, timer, 332 jiffies + (timeout << entry->retrans_times)); 333 334 spin_unlock_bh(&msk->pm.lock); 335 336 if (entry->retrans_times == ADD_ADDR_RETRANS_MAX) 337 mptcp_pm_subflow_established(msk); 338 339 out: 340 __sock_put(sk); 341 } 342 343 struct mptcp_pm_add_entry * 344 mptcp_pm_del_add_timer(struct mptcp_sock *msk, 345 const struct mptcp_addr_info *addr, bool check_id) 346 { 347 struct mptcp_pm_add_entry *entry; 348 struct sock *sk = (struct sock *)msk; 349 bool stop_timer = false; 350 351 rcu_read_lock(); 352 353 spin_lock_bh(&msk->pm.lock); 354 entry = mptcp_lookup_anno_list_by_saddr(msk, addr); 355 if (entry && (!check_id || entry->addr.id == addr->id)) { 356 entry->retrans_times = ADD_ADDR_RETRANS_MAX; 357 stop_timer = true; 358 } 359 if (!check_id && entry) 360 list_del(&entry->list); 361 spin_unlock_bh(&msk->pm.lock); 362 363 /* Note: entry might have been removed by another thread. 364 * We hold rcu_read_lock() to ensure it is not freed under us. 365 */ 366 if (stop_timer) 367 sk_stop_timer_sync(sk, &entry->add_timer); 368 369 rcu_read_unlock(); 370 return entry; 371 } 372 373 bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, 374 const struct mptcp_addr_info *addr) 375 { 376 struct mptcp_pm_add_entry *add_entry = NULL; 377 struct sock *sk = (struct sock *)msk; 378 unsigned int timeout; 379 380 lockdep_assert_held(&msk->pm.lock); 381 382 add_entry = mptcp_lookup_anno_list_by_saddr(msk, addr); 383 384 if (add_entry) { 385 if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk))) 386 return false; 387 388 goto reset_timer; 389 } 390 391 add_entry = kmalloc(sizeof(*add_entry), GFP_ATOMIC); 392 if (!add_entry) 393 return false; 394 395 list_add(&add_entry->list, &msk->pm.anno_list); 396 397 add_entry->addr = *addr; 398 add_entry->sock = msk; 399 add_entry->retrans_times = 0; 400 401 timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0); 402 reset_timer: 403 timeout = mptcp_adjust_add_addr_timeout(msk); 404 if (timeout) 405 sk_reset_timer(sk, &add_entry->add_timer, jiffies + timeout); 406 407 return true; 408 } 409 410 static void mptcp_pm_free_anno_list(struct mptcp_sock *msk) 411 { 412 struct mptcp_pm_add_entry *entry, *tmp; 413 struct sock *sk = (struct sock *)msk; 414 LIST_HEAD(free_list); 415 416 pr_debug("msk=%p\n", msk); 417 418 spin_lock_bh(&msk->pm.lock); 419 list_splice_init(&msk->pm.anno_list, &free_list); 420 spin_unlock_bh(&msk->pm.lock); 421 422 list_for_each_entry_safe(entry, tmp, &free_list, list) { 423 sk_stop_timer_sync(sk, &entry->add_timer); 424 kfree_rcu(entry, rcu); 425 } 426 } 427 428 /* path manager command handlers */ 429 430 int mptcp_pm_announce_addr(struct mptcp_sock *msk, 431 const struct mptcp_addr_info *addr, 432 bool echo) 433 { 434 u8 add_addr = READ_ONCE(msk->pm.addr_signal); 435 436 pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo); 437 438 lockdep_assert_held(&msk->pm.lock); 439 440 if (add_addr & 441 (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) { 442 MPTCP_INC_STATS(sock_net((struct sock *)msk), 443 echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP); 444 return -EINVAL; 445 } 446 447 if (echo) { 448 msk->pm.remote = *addr; 449 add_addr |= BIT(MPTCP_ADD_ADDR_ECHO); 450 } else { 451 msk->pm.local = *addr; 452 add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL); 453 } 454 WRITE_ONCE(msk->pm.addr_signal, add_addr); 455 return 0; 456 } 457 458 int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list) 459 { 460 u8 rm_addr = READ_ONCE(msk->pm.addr_signal); 461 462 pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr); 463 464 if (rm_addr) { 465 MPTCP_ADD_STATS(sock_net((struct sock *)msk), 466 MPTCP_MIB_RMADDRTXDROP, rm_list->nr); 467 return -EINVAL; 468 } 469 470 msk->pm.rm_list_tx = *rm_list; 471 rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL); 472 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 473 mptcp_pm_addr_send_ack(msk); 474 return 0; 475 } 476 477 /* path manager event handlers */ 478 479 void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side) 480 { 481 struct mptcp_pm_data *pm = &msk->pm; 482 483 pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side); 484 485 WRITE_ONCE(pm->server_side, server_side); 486 mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC); 487 } 488 489 bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk) 490 { 491 struct mptcp_pm_data *pm = &msk->pm; 492 unsigned int limit_extra_subflows; 493 int ret = 0; 494 495 if (mptcp_pm_is_userspace(msk)) { 496 if (mptcp_userspace_pm_active(msk)) { 497 spin_lock_bh(&pm->lock); 498 pm->extra_subflows++; 499 spin_unlock_bh(&pm->lock); 500 return true; 501 } 502 return false; 503 } 504 505 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk); 506 507 pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk, 508 pm->extra_subflows, limit_extra_subflows, 509 READ_ONCE(pm->accept_subflow)); 510 511 /* try to avoid acquiring the lock below */ 512 if (!READ_ONCE(pm->accept_subflow)) 513 return false; 514 515 spin_lock_bh(&pm->lock); 516 if (READ_ONCE(pm->accept_subflow)) { 517 ret = pm->extra_subflows < limit_extra_subflows; 518 if (ret && ++pm->extra_subflows == limit_extra_subflows) 519 WRITE_ONCE(pm->accept_subflow, false); 520 } 521 spin_unlock_bh(&pm->lock); 522 523 return ret; 524 } 525 526 /* return true if the new status bit is currently cleared, that is, this event 527 * can be server, eventually by an already scheduled work 528 */ 529 static bool mptcp_pm_schedule_work(struct mptcp_sock *msk, 530 enum mptcp_pm_status new_status) 531 { 532 pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status, 533 BIT(new_status)); 534 if (msk->pm.status & BIT(new_status)) 535 return false; 536 537 msk->pm.status |= BIT(new_status); 538 mptcp_schedule_work((struct sock *)msk); 539 return true; 540 } 541 542 void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk) 543 { 544 struct mptcp_pm_data *pm = &msk->pm; 545 bool announce = false; 546 547 pr_debug("msk=%p\n", msk); 548 549 spin_lock_bh(&pm->lock); 550 551 /* mptcp_pm_fully_established() can be invoked by multiple 552 * racing paths - accept() and check_fully_established() 553 * be sure to serve this event only once. 554 */ 555 if (READ_ONCE(pm->work_pending) && 556 !(pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED))) 557 mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED); 558 559 if ((pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0) 560 announce = true; 561 562 pm->status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED); 563 spin_unlock_bh(&pm->lock); 564 565 if (announce) 566 mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC); 567 } 568 569 void mptcp_pm_connection_closed(struct mptcp_sock *msk) 570 { 571 pr_debug("msk=%p\n", msk); 572 573 if (msk->token) 574 mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 575 } 576 577 void mptcp_pm_subflow_established(struct mptcp_sock *msk) 578 { 579 struct mptcp_pm_data *pm = &msk->pm; 580 581 pr_debug("msk=%p\n", msk); 582 583 if (!READ_ONCE(pm->work_pending)) 584 return; 585 586 spin_lock_bh(&pm->lock); 587 588 if (READ_ONCE(pm->work_pending)) 589 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 590 591 spin_unlock_bh(&pm->lock); 592 } 593 594 void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, 595 const struct mptcp_subflow_context *subflow) 596 { 597 struct sock *sk = (struct sock *)msk; 598 struct mptcp_pm_data *pm = &msk->pm; 599 bool update_subflows; 600 601 update_subflows = subflow->request_join || subflow->mp_join; 602 if (mptcp_pm_is_userspace(msk)) { 603 if (update_subflows) { 604 spin_lock_bh(&pm->lock); 605 pm->extra_subflows--; 606 spin_unlock_bh(&pm->lock); 607 } 608 return; 609 } 610 611 if (!READ_ONCE(pm->work_pending) && !update_subflows) 612 return; 613 614 spin_lock_bh(&pm->lock); 615 if (update_subflows) 616 __mptcp_pm_close_subflow(msk); 617 618 /* Even if this subflow is not really established, tell the PM to try 619 * to pick the next ones, if possible. 620 */ 621 if (mptcp_is_fully_established(sk) && 622 mptcp_pm_nl_check_work_pending(msk)) 623 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 624 625 spin_unlock_bh(&pm->lock); 626 } 627 628 void mptcp_pm_add_addr_received(const struct sock *ssk, 629 const struct mptcp_addr_info *addr) 630 { 631 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 632 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 633 struct mptcp_pm_data *pm = &msk->pm; 634 635 pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id, 636 READ_ONCE(pm->accept_addr)); 637 638 mptcp_event_addr_announced(ssk, addr); 639 640 spin_lock_bh(&pm->lock); 641 642 if (mptcp_pm_is_userspace(msk)) { 643 if (mptcp_userspace_pm_active(msk)) { 644 mptcp_pm_announce_addr(msk, addr, true); 645 mptcp_pm_add_addr_send_ack(msk); 646 } else { 647 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); 648 } 649 /* - id0 should not have a different address 650 * - special case for C-flag: linked to fill_local_addresses_vec() 651 */ 652 } else if ((addr->id == 0 && !mptcp_pm_is_init_remote_addr(msk, addr)) || 653 (addr->id > 0 && !READ_ONCE(pm->accept_addr) && 654 !mptcp_pm_add_addr_c_flag_case(msk))) { 655 mptcp_pm_announce_addr(msk, addr, true); 656 mptcp_pm_add_addr_send_ack(msk); 657 } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) { 658 pm->remote = *addr; 659 } else { 660 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); 661 } 662 663 spin_unlock_bh(&pm->lock); 664 } 665 666 void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, 667 const struct mptcp_addr_info *addr) 668 { 669 struct mptcp_pm_data *pm = &msk->pm; 670 671 pr_debug("msk=%p\n", msk); 672 673 if (!READ_ONCE(pm->work_pending)) 674 return; 675 676 spin_lock_bh(&pm->lock); 677 678 if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending)) 679 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 680 681 spin_unlock_bh(&pm->lock); 682 } 683 684 void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk) 685 { 686 if (!mptcp_pm_should_add_signal(msk)) 687 return; 688 689 mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK); 690 } 691 692 static void mptcp_pm_rm_addr_or_subflow(struct mptcp_sock *msk, 693 const struct mptcp_rm_list *rm_list, 694 enum linux_mptcp_mib_field rm_type) 695 { 696 struct mptcp_subflow_context *subflow, *tmp; 697 struct sock *sk = (struct sock *)msk; 698 u8 i; 699 700 pr_debug("%s rm_list_nr %d\n", 701 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", rm_list->nr); 702 703 msk_owned_by_me(msk); 704 705 if (sk->sk_state == TCP_LISTEN) 706 return; 707 708 if (!rm_list->nr) 709 return; 710 711 if (list_empty(&msk->conn_list)) 712 return; 713 714 for (i = 0; i < rm_list->nr; i++) { 715 u8 rm_id = rm_list->ids[i]; 716 bool removed = false; 717 718 mptcp_for_each_subflow_safe(msk, subflow, tmp) { 719 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 720 u8 remote_id = READ_ONCE(subflow->remote_id); 721 int how = RCV_SHUTDOWN | SEND_SHUTDOWN; 722 u8 id = subflow_get_local_id(subflow); 723 724 if ((1 << inet_sk_state_load(ssk)) & 725 (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING | TCPF_CLOSE)) 726 continue; 727 if (rm_type == MPTCP_MIB_RMADDR && remote_id != rm_id) 728 continue; 729 if (rm_type == MPTCP_MIB_RMSUBFLOW && id != rm_id) 730 continue; 731 732 pr_debug(" -> %s rm_list_ids[%d]=%u local_id=%u remote_id=%u mpc_id=%u\n", 733 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", 734 i, rm_id, id, remote_id, msk->mpc_endpoint_id); 735 spin_unlock_bh(&msk->pm.lock); 736 mptcp_subflow_shutdown(sk, ssk, how); 737 removed |= subflow->request_join; 738 739 /* the following takes care of updating the subflows counter */ 740 mptcp_close_ssk(sk, ssk, subflow); 741 spin_lock_bh(&msk->pm.lock); 742 743 if (rm_type == MPTCP_MIB_RMSUBFLOW) 744 __MPTCP_INC_STATS(sock_net(sk), rm_type); 745 } 746 747 if (rm_type == MPTCP_MIB_RMADDR) { 748 __MPTCP_INC_STATS(sock_net(sk), rm_type); 749 if (removed && mptcp_pm_is_kernel(msk)) 750 mptcp_pm_nl_rm_addr(msk, rm_id); 751 } 752 } 753 } 754 755 static void mptcp_pm_rm_addr_recv(struct mptcp_sock *msk) 756 { 757 mptcp_pm_rm_addr_or_subflow(msk, &msk->pm.rm_list_rx, MPTCP_MIB_RMADDR); 758 } 759 760 void mptcp_pm_rm_subflow(struct mptcp_sock *msk, 761 const struct mptcp_rm_list *rm_list) 762 { 763 mptcp_pm_rm_addr_or_subflow(msk, rm_list, MPTCP_MIB_RMSUBFLOW); 764 } 765 766 void mptcp_pm_rm_addr_received(struct mptcp_sock *msk, 767 const struct mptcp_rm_list *rm_list) 768 { 769 struct mptcp_pm_data *pm = &msk->pm; 770 u8 i; 771 772 pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr); 773 774 for (i = 0; i < rm_list->nr; i++) 775 mptcp_event_addr_removed(msk, rm_list->ids[i]); 776 777 spin_lock_bh(&pm->lock); 778 if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED)) 779 pm->rm_list_rx = *rm_list; 780 else 781 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP); 782 spin_unlock_bh(&pm->lock); 783 } 784 785 void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup) 786 { 787 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 788 struct sock *sk = subflow->conn; 789 struct mptcp_sock *msk; 790 791 pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup); 792 msk = mptcp_sk(sk); 793 if (subflow->backup != bkup) 794 subflow->backup = bkup; 795 796 mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC); 797 } 798 799 void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq) 800 { 801 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 802 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 803 804 pr_debug("fail_seq=%llu\n", fail_seq); 805 806 /* After accepting the fail, we can't create any other subflows */ 807 spin_lock_bh(&msk->fallback_lock); 808 if (!msk->allow_infinite_fallback) { 809 spin_unlock_bh(&msk->fallback_lock); 810 return; 811 } 812 msk->allow_subflows = false; 813 spin_unlock_bh(&msk->fallback_lock); 814 815 if (!subflow->fail_tout) { 816 pr_debug("send MP_FAIL response and infinite map\n"); 817 818 subflow->send_mp_fail = 1; 819 subflow->send_infinite_map = 1; 820 tcp_send_ack(sk); 821 } else { 822 pr_debug("MP_FAIL response received\n"); 823 WRITE_ONCE(subflow->fail_tout, 0); 824 } 825 } 826 827 bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb, 828 unsigned int opt_size, unsigned int remaining, 829 struct mptcp_addr_info *addr, bool *echo, 830 bool *drop_other_suboptions) 831 { 832 int ret = false; 833 u8 add_addr; 834 u8 family; 835 bool port; 836 837 spin_lock_bh(&msk->pm.lock); 838 839 /* double check after the lock is acquired */ 840 if (!mptcp_pm_should_add_signal(msk)) 841 goto out_unlock; 842 843 /* always drop every other options for pure ack ADD_ADDR; this is a 844 * plain dup-ack from TCP perspective. The other MPTCP-relevant info, 845 * if any, will be carried by the 'original' TCP ack 846 */ 847 if (skb && skb_is_tcp_pure_ack(skb)) { 848 remaining += opt_size; 849 *drop_other_suboptions = true; 850 } 851 852 *echo = mptcp_pm_should_add_signal_echo(msk); 853 port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port); 854 855 family = *echo ? msk->pm.remote.family : msk->pm.local.family; 856 if (remaining < mptcp_add_addr_len(family, *echo, port)) 857 goto out_unlock; 858 859 if (*echo) { 860 *addr = msk->pm.remote; 861 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO); 862 } else { 863 *addr = msk->pm.local; 864 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL); 865 } 866 WRITE_ONCE(msk->pm.addr_signal, add_addr); 867 ret = true; 868 869 out_unlock: 870 spin_unlock_bh(&msk->pm.lock); 871 return ret; 872 } 873 874 bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining, 875 struct mptcp_rm_list *rm_list) 876 { 877 int ret = false, len; 878 u8 rm_addr; 879 880 spin_lock_bh(&msk->pm.lock); 881 882 /* double check after the lock is acquired */ 883 if (!mptcp_pm_should_rm_signal(msk)) 884 goto out_unlock; 885 886 rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL); 887 len = mptcp_rm_addr_len(&msk->pm.rm_list_tx); 888 if (len < 0) { 889 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 890 goto out_unlock; 891 } 892 if (remaining < len) 893 goto out_unlock; 894 895 *rm_list = msk->pm.rm_list_tx; 896 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 897 ret = true; 898 899 out_unlock: 900 spin_unlock_bh(&msk->pm.lock); 901 return ret; 902 } 903 904 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc) 905 { 906 struct mptcp_pm_addr_entry skc_local = { 0 }; 907 struct mptcp_addr_info msk_local; 908 909 if (WARN_ON_ONCE(!msk)) 910 return -1; 911 912 /* The 0 ID mapping is defined by the first subflow, copied into the msk 913 * addr 914 */ 915 mptcp_local_address((struct sock_common *)msk, &msk_local); 916 mptcp_local_address((struct sock_common *)skc, &skc_local.addr); 917 if (mptcp_addresses_equal(&msk_local, &skc_local.addr, false)) 918 return 0; 919 920 skc_local.addr.id = 0; 921 skc_local.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT; 922 923 if (mptcp_pm_is_userspace(msk)) 924 return mptcp_userspace_pm_get_local_id(msk, &skc_local); 925 return mptcp_pm_nl_get_local_id(msk, &skc_local); 926 } 927 928 bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc) 929 { 930 struct mptcp_addr_info skc_local; 931 932 mptcp_local_address((struct sock_common *)skc, &skc_local); 933 934 if (mptcp_pm_is_userspace(msk)) 935 return mptcp_userspace_pm_is_backup(msk, &skc_local); 936 937 return mptcp_pm_nl_is_backup(msk, &skc_local); 938 } 939 940 static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) 941 { 942 struct mptcp_subflow_context *iter, *subflow = mptcp_subflow_ctx(ssk); 943 struct sock *sk = (struct sock *)msk; 944 unsigned int active_max_loss_cnt; 945 struct net *net = sock_net(sk); 946 unsigned int stale_loss_cnt; 947 bool slow; 948 949 stale_loss_cnt = mptcp_stale_loss_cnt(net); 950 if (subflow->stale || !stale_loss_cnt || subflow->stale_count <= stale_loss_cnt) 951 return; 952 953 /* look for another available subflow not in loss state */ 954 active_max_loss_cnt = max_t(int, stale_loss_cnt - 1, 1); 955 mptcp_for_each_subflow(msk, iter) { 956 if (iter != subflow && mptcp_subflow_active(iter) && 957 iter->stale_count < active_max_loss_cnt) { 958 /* we have some alternatives, try to mark this subflow as idle ...*/ 959 slow = lock_sock_fast(ssk); 960 if (!tcp_rtx_and_write_queues_empty(ssk)) { 961 subflow->stale = 1; 962 __mptcp_retransmit_pending_data(sk); 963 MPTCP_INC_STATS(net, MPTCP_MIB_SUBFLOWSTALE); 964 } 965 unlock_sock_fast(ssk, slow); 966 967 /* always try to push the pending data regardless of re-injections: 968 * we can possibly use backup subflows now, and subflow selection 969 * is cheap under the msk socket lock 970 */ 971 __mptcp_push_pending(sk, 0); 972 return; 973 } 974 } 975 } 976 977 void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) 978 { 979 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 980 u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); 981 982 /* keep track of rtx periods with no progress */ 983 if (!subflow->stale_count) { 984 subflow->stale_rcv_tstamp = rcv_tstamp; 985 subflow->stale_count++; 986 } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { 987 if (subflow->stale_count < U8_MAX) 988 subflow->stale_count++; 989 mptcp_pm_subflows_chk_stale(msk, ssk); 990 } else { 991 subflow->stale_count = 0; 992 mptcp_subflow_set_active(subflow); 993 } 994 } 995 996 void mptcp_pm_worker(struct mptcp_sock *msk) 997 { 998 struct mptcp_pm_data *pm = &msk->pm; 999 1000 msk_owned_by_me(msk); 1001 1002 if (!(pm->status & MPTCP_PM_WORK_MASK)) 1003 return; 1004 1005 spin_lock_bh(&msk->pm.lock); 1006 1007 pr_debug("msk=%p status=%x\n", msk, pm->status); 1008 if (pm->status & BIT(MPTCP_PM_ADD_ADDR_SEND_ACK)) { 1009 pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_SEND_ACK); 1010 mptcp_pm_addr_send_ack(msk); 1011 } 1012 if (pm->status & BIT(MPTCP_PM_RM_ADDR_RECEIVED)) { 1013 pm->status &= ~BIT(MPTCP_PM_RM_ADDR_RECEIVED); 1014 mptcp_pm_rm_addr_recv(msk); 1015 } 1016 __mptcp_pm_kernel_worker(msk); 1017 1018 spin_unlock_bh(&msk->pm.lock); 1019 } 1020 1021 void mptcp_pm_destroy(struct mptcp_sock *msk) 1022 { 1023 mptcp_pm_free_anno_list(msk); 1024 1025 if (mptcp_pm_is_userspace(msk)) 1026 mptcp_userspace_pm_free_local_addr_list(msk); 1027 } 1028 1029 void mptcp_pm_data_reset(struct mptcp_sock *msk) 1030 { 1031 u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk)); 1032 struct mptcp_pm_data *pm = &msk->pm; 1033 1034 memset(&pm->reset, 0, sizeof(pm->reset)); 1035 pm->rm_list_tx.nr = 0; 1036 pm->rm_list_rx.nr = 0; 1037 WRITE_ONCE(pm->pm_type, pm_type); 1038 1039 if (pm_type == MPTCP_PM_TYPE_KERNEL) { 1040 bool subflows_allowed = !!mptcp_pm_get_limit_extra_subflows(msk); 1041 1042 /* pm->work_pending must be only be set to 'true' when 1043 * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL 1044 */ 1045 WRITE_ONCE(pm->work_pending, 1046 (!!mptcp_pm_get_endp_subflow_max(msk) && 1047 subflows_allowed) || 1048 !!mptcp_pm_get_endp_signal_max(msk)); 1049 WRITE_ONCE(pm->accept_addr, 1050 !!mptcp_pm_get_limit_add_addr_accepted(msk) && 1051 subflows_allowed); 1052 WRITE_ONCE(pm->accept_subflow, subflows_allowed); 1053 1054 bitmap_fill(pm->id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); 1055 } 1056 } 1057 1058 void mptcp_pm_data_init(struct mptcp_sock *msk) 1059 { 1060 spin_lock_init(&msk->pm.lock); 1061 INIT_LIST_HEAD(&msk->pm.anno_list); 1062 INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list); 1063 mptcp_pm_data_reset(msk); 1064 } 1065 1066 void __init mptcp_pm_init(void) 1067 { 1068 mptcp_pm_kernel_register(); 1069 mptcp_pm_userspace_register(); 1070 mptcp_pm_nl_init(); 1071 } 1072 1073 /* Must be called with rcu read lock held */ 1074 struct mptcp_pm_ops *mptcp_pm_find(const char *name) 1075 { 1076 struct mptcp_pm_ops *pm_ops; 1077 1078 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) { 1079 if (!strcmp(pm_ops->name, name)) 1080 return pm_ops; 1081 } 1082 1083 return NULL; 1084 } 1085 1086 int mptcp_pm_validate(struct mptcp_pm_ops *pm_ops) 1087 { 1088 return 0; 1089 } 1090 1091 int mptcp_pm_register(struct mptcp_pm_ops *pm_ops) 1092 { 1093 int ret; 1094 1095 ret = mptcp_pm_validate(pm_ops); 1096 if (ret) 1097 return ret; 1098 1099 spin_lock(&mptcp_pm_list_lock); 1100 if (mptcp_pm_find(pm_ops->name)) { 1101 spin_unlock(&mptcp_pm_list_lock); 1102 return -EEXIST; 1103 } 1104 list_add_tail_rcu(&pm_ops->list, &mptcp_pm_list); 1105 spin_unlock(&mptcp_pm_list_lock); 1106 1107 pr_debug("%s registered\n", pm_ops->name); 1108 return 0; 1109 } 1110 1111 void mptcp_pm_unregister(struct mptcp_pm_ops *pm_ops) 1112 { 1113 /* skip unregistering the default path manager */ 1114 if (WARN_ON_ONCE(pm_ops == &mptcp_pm_kernel)) 1115 return; 1116 1117 spin_lock(&mptcp_pm_list_lock); 1118 list_del_rcu(&pm_ops->list); 1119 spin_unlock(&mptcp_pm_list_lock); 1120 } 1121 1122 /* Build string with list of available path manager values. 1123 * Similar to tcp_get_available_congestion_control() 1124 */ 1125 void mptcp_pm_get_available(char *buf, size_t maxlen) 1126 { 1127 struct mptcp_pm_ops *pm_ops; 1128 size_t offs = 0; 1129 1130 rcu_read_lock(); 1131 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) { 1132 offs += snprintf(buf + offs, maxlen - offs, "%s%s", 1133 offs == 0 ? "" : " ", pm_ops->name); 1134 1135 if (WARN_ON_ONCE(offs >= maxlen)) 1136 break; 1137 } 1138 rcu_read_unlock(); 1139 } 1140