1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2019, Intel Corporation. 5 */ 6 #define pr_fmt(fmt) "MPTCP: " fmt 7 8 #include <linux/rculist.h> 9 #include <linux/spinlock.h> 10 #include "protocol.h" 11 #include "mib.h" 12 13 #define ADD_ADDR_RETRANS_MAX 3 14 15 struct mptcp_pm_add_entry { 16 struct list_head list; 17 struct mptcp_addr_info addr; 18 u8 retrans_times; 19 struct timer_list add_timer; 20 struct mptcp_sock *sock; 21 }; 22 23 static DEFINE_SPINLOCK(mptcp_pm_list_lock); 24 static LIST_HEAD(mptcp_pm_list); 25 26 /* path manager helpers */ 27 28 /* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses, 29 * otherwise allow any matching local/remote pair 30 */ 31 bool mptcp_pm_addr_families_match(const struct sock *sk, 32 const struct mptcp_addr_info *loc, 33 const struct mptcp_addr_info *rem) 34 { 35 bool mptcp_is_v4 = sk->sk_family == AF_INET; 36 37 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 38 bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(&loc->addr6); 39 bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(&rem->addr6); 40 41 if (mptcp_is_v4) 42 return loc_is_v4 && rem_is_v4; 43 44 if (ipv6_only_sock(sk)) 45 return !loc_is_v4 && !rem_is_v4; 46 47 return loc_is_v4 == rem_is_v4; 48 #else 49 return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET; 50 #endif 51 } 52 53 bool mptcp_addresses_equal(const struct mptcp_addr_info *a, 54 const struct mptcp_addr_info *b, bool use_port) 55 { 56 bool addr_equals = false; 57 58 if (a->family == b->family) { 59 if (a->family == AF_INET) 60 addr_equals = a->addr.s_addr == b->addr.s_addr; 61 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 62 else 63 addr_equals = ipv6_addr_equal(&a->addr6, &b->addr6); 64 } else if (a->family == AF_INET) { 65 if (ipv6_addr_v4mapped(&b->addr6)) 66 addr_equals = a->addr.s_addr == b->addr6.s6_addr32[3]; 67 } else if (b->family == AF_INET) { 68 if (ipv6_addr_v4mapped(&a->addr6)) 69 addr_equals = a->addr6.s6_addr32[3] == b->addr.s_addr; 70 #endif 71 } 72 73 if (!addr_equals) 74 return false; 75 if (!use_port) 76 return true; 77 78 return a->port == b->port; 79 } 80 81 void mptcp_local_address(const struct sock_common *skc, 82 struct mptcp_addr_info *addr) 83 { 84 addr->family = skc->skc_family; 85 addr->port = htons(skc->skc_num); 86 if (addr->family == AF_INET) 87 addr->addr.s_addr = skc->skc_rcv_saddr; 88 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 89 else if (addr->family == AF_INET6) 90 addr->addr6 = skc->skc_v6_rcv_saddr; 91 #endif 92 } 93 94 void mptcp_remote_address(const struct sock_common *skc, 95 struct mptcp_addr_info *addr) 96 { 97 addr->family = skc->skc_family; 98 addr->port = skc->skc_dport; 99 if (addr->family == AF_INET) 100 addr->addr.s_addr = skc->skc_daddr; 101 #if IS_ENABLED(CONFIG_MPTCP_IPV6) 102 else if (addr->family == AF_INET6) 103 addr->addr6 = skc->skc_v6_daddr; 104 #endif 105 } 106 107 static bool mptcp_pm_is_init_remote_addr(struct mptcp_sock *msk, 108 const struct mptcp_addr_info *remote) 109 { 110 struct mptcp_addr_info mpc_remote; 111 112 mptcp_remote_address((struct sock_common *)msk, &mpc_remote); 113 return mptcp_addresses_equal(&mpc_remote, remote, remote->port); 114 } 115 116 bool mptcp_lookup_subflow_by_saddr(const struct list_head *list, 117 const struct mptcp_addr_info *saddr) 118 { 119 struct mptcp_subflow_context *subflow; 120 struct mptcp_addr_info cur; 121 struct sock_common *skc; 122 123 list_for_each_entry(subflow, list, node) { 124 skc = (struct sock_common *)mptcp_subflow_tcp_sock(subflow); 125 126 mptcp_local_address(skc, &cur); 127 if (mptcp_addresses_equal(&cur, saddr, saddr->port)) 128 return true; 129 } 130 131 return false; 132 } 133 134 static struct mptcp_pm_add_entry * 135 mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, 136 const struct mptcp_addr_info *addr) 137 { 138 struct mptcp_pm_add_entry *entry; 139 140 lockdep_assert_held(&msk->pm.lock); 141 142 list_for_each_entry(entry, &msk->pm.anno_list, list) { 143 if (mptcp_addresses_equal(&entry->addr, addr, true)) 144 return entry; 145 } 146 147 return NULL; 148 } 149 150 bool mptcp_remove_anno_list_by_saddr(struct mptcp_sock *msk, 151 const struct mptcp_addr_info *addr) 152 { 153 struct mptcp_pm_add_entry *entry; 154 155 entry = mptcp_pm_del_add_timer(msk, addr, false); 156 kfree(entry); 157 return entry; 158 } 159 160 bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk) 161 { 162 struct mptcp_pm_add_entry *entry; 163 struct mptcp_addr_info saddr; 164 bool ret = false; 165 166 mptcp_local_address((struct sock_common *)sk, &saddr); 167 168 spin_lock_bh(&msk->pm.lock); 169 list_for_each_entry(entry, &msk->pm.anno_list, list) { 170 if (mptcp_addresses_equal(&entry->addr, &saddr, true)) { 171 ret = true; 172 goto out; 173 } 174 } 175 176 out: 177 spin_unlock_bh(&msk->pm.lock); 178 return ret; 179 } 180 181 static void __mptcp_pm_send_ack(struct mptcp_sock *msk, 182 struct mptcp_subflow_context *subflow, 183 bool prio, bool backup) 184 { 185 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 186 bool slow; 187 188 pr_debug("send ack for %s\n", 189 prio ? "mp_prio" : 190 (mptcp_pm_should_add_signal(msk) ? "add_addr" : "rm_addr")); 191 192 slow = lock_sock_fast(ssk); 193 if (prio) { 194 subflow->send_mp_prio = 1; 195 subflow->request_bkup = backup; 196 } 197 198 __mptcp_subflow_send_ack(ssk); 199 unlock_sock_fast(ssk, slow); 200 } 201 202 void mptcp_pm_send_ack(struct mptcp_sock *msk, 203 struct mptcp_subflow_context *subflow, 204 bool prio, bool backup) 205 { 206 spin_unlock_bh(&msk->pm.lock); 207 __mptcp_pm_send_ack(msk, subflow, prio, backup); 208 spin_lock_bh(&msk->pm.lock); 209 } 210 211 void mptcp_pm_addr_send_ack(struct mptcp_sock *msk) 212 { 213 struct mptcp_subflow_context *subflow, *alt = NULL; 214 215 msk_owned_by_me(msk); 216 lockdep_assert_held(&msk->pm.lock); 217 218 if (!mptcp_pm_should_add_signal(msk) && 219 !mptcp_pm_should_rm_signal(msk)) 220 return; 221 222 mptcp_for_each_subflow(msk, subflow) { 223 if (__mptcp_subflow_active(subflow)) { 224 if (!subflow->stale) { 225 mptcp_pm_send_ack(msk, subflow, false, false); 226 return; 227 } 228 229 if (!alt) 230 alt = subflow; 231 } 232 } 233 234 if (alt) 235 mptcp_pm_send_ack(msk, alt, false, false); 236 } 237 238 int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, 239 struct mptcp_addr_info *addr, 240 struct mptcp_addr_info *rem, 241 u8 bkup) 242 { 243 struct mptcp_subflow_context *subflow; 244 245 pr_debug("bkup=%d\n", bkup); 246 247 mptcp_for_each_subflow(msk, subflow) { 248 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 249 struct mptcp_addr_info local, remote; 250 251 mptcp_local_address((struct sock_common *)ssk, &local); 252 if (!mptcp_addresses_equal(&local, addr, addr->port)) 253 continue; 254 255 if (rem && rem->family != AF_UNSPEC) { 256 mptcp_remote_address((struct sock_common *)ssk, &remote); 257 if (!mptcp_addresses_equal(&remote, rem, rem->port)) 258 continue; 259 } 260 261 __mptcp_pm_send_ack(msk, subflow, true, bkup); 262 return 0; 263 } 264 265 return -EINVAL; 266 } 267 268 static void mptcp_pm_add_timer(struct timer_list *timer) 269 { 270 struct mptcp_pm_add_entry *entry = from_timer(entry, timer, add_timer); 271 struct mptcp_sock *msk = entry->sock; 272 struct sock *sk = (struct sock *)msk; 273 274 pr_debug("msk=%p\n", msk); 275 276 if (!msk) 277 return; 278 279 if (inet_sk_state_load(sk) == TCP_CLOSE) 280 return; 281 282 if (!entry->addr.id) 283 return; 284 285 if (mptcp_pm_should_add_signal_addr(msk)) { 286 sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8); 287 goto out; 288 } 289 290 spin_lock_bh(&msk->pm.lock); 291 292 if (!mptcp_pm_should_add_signal_addr(msk)) { 293 pr_debug("retransmit ADD_ADDR id=%d\n", entry->addr.id); 294 mptcp_pm_announce_addr(msk, &entry->addr, false); 295 mptcp_pm_add_addr_send_ack(msk); 296 entry->retrans_times++; 297 } 298 299 if (entry->retrans_times < ADD_ADDR_RETRANS_MAX) 300 sk_reset_timer(sk, timer, 301 jiffies + mptcp_get_add_addr_timeout(sock_net(sk))); 302 303 spin_unlock_bh(&msk->pm.lock); 304 305 if (entry->retrans_times == ADD_ADDR_RETRANS_MAX) 306 mptcp_pm_subflow_established(msk); 307 308 out: 309 __sock_put(sk); 310 } 311 312 struct mptcp_pm_add_entry * 313 mptcp_pm_del_add_timer(struct mptcp_sock *msk, 314 const struct mptcp_addr_info *addr, bool check_id) 315 { 316 struct mptcp_pm_add_entry *entry; 317 struct sock *sk = (struct sock *)msk; 318 struct timer_list *add_timer = NULL; 319 320 spin_lock_bh(&msk->pm.lock); 321 entry = mptcp_lookup_anno_list_by_saddr(msk, addr); 322 if (entry && (!check_id || entry->addr.id == addr->id)) { 323 entry->retrans_times = ADD_ADDR_RETRANS_MAX; 324 add_timer = &entry->add_timer; 325 } 326 if (!check_id && entry) 327 list_del(&entry->list); 328 spin_unlock_bh(&msk->pm.lock); 329 330 /* no lock, because sk_stop_timer_sync() is calling del_timer_sync() */ 331 if (add_timer) 332 sk_stop_timer_sync(sk, add_timer); 333 334 return entry; 335 } 336 337 bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, 338 const struct mptcp_addr_info *addr) 339 { 340 struct mptcp_pm_add_entry *add_entry = NULL; 341 struct sock *sk = (struct sock *)msk; 342 struct net *net = sock_net(sk); 343 344 lockdep_assert_held(&msk->pm.lock); 345 346 add_entry = mptcp_lookup_anno_list_by_saddr(msk, addr); 347 348 if (add_entry) { 349 if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk))) 350 return false; 351 352 sk_reset_timer(sk, &add_entry->add_timer, 353 jiffies + mptcp_get_add_addr_timeout(net)); 354 return true; 355 } 356 357 add_entry = kmalloc(sizeof(*add_entry), GFP_ATOMIC); 358 if (!add_entry) 359 return false; 360 361 list_add(&add_entry->list, &msk->pm.anno_list); 362 363 add_entry->addr = *addr; 364 add_entry->sock = msk; 365 add_entry->retrans_times = 0; 366 367 timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0); 368 sk_reset_timer(sk, &add_entry->add_timer, 369 jiffies + mptcp_get_add_addr_timeout(net)); 370 371 return true; 372 } 373 374 static void mptcp_pm_free_anno_list(struct mptcp_sock *msk) 375 { 376 struct mptcp_pm_add_entry *entry, *tmp; 377 struct sock *sk = (struct sock *)msk; 378 LIST_HEAD(free_list); 379 380 pr_debug("msk=%p\n", msk); 381 382 spin_lock_bh(&msk->pm.lock); 383 list_splice_init(&msk->pm.anno_list, &free_list); 384 spin_unlock_bh(&msk->pm.lock); 385 386 list_for_each_entry_safe(entry, tmp, &free_list, list) { 387 sk_stop_timer_sync(sk, &entry->add_timer); 388 kfree(entry); 389 } 390 } 391 392 /* path manager command handlers */ 393 394 int mptcp_pm_announce_addr(struct mptcp_sock *msk, 395 const struct mptcp_addr_info *addr, 396 bool echo) 397 { 398 u8 add_addr = READ_ONCE(msk->pm.addr_signal); 399 400 pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo); 401 402 lockdep_assert_held(&msk->pm.lock); 403 404 if (add_addr & 405 (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) { 406 MPTCP_INC_STATS(sock_net((struct sock *)msk), 407 echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP); 408 return -EINVAL; 409 } 410 411 if (echo) { 412 msk->pm.remote = *addr; 413 add_addr |= BIT(MPTCP_ADD_ADDR_ECHO); 414 } else { 415 msk->pm.local = *addr; 416 add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL); 417 } 418 WRITE_ONCE(msk->pm.addr_signal, add_addr); 419 return 0; 420 } 421 422 int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list) 423 { 424 u8 rm_addr = READ_ONCE(msk->pm.addr_signal); 425 426 pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr); 427 428 if (rm_addr) { 429 MPTCP_ADD_STATS(sock_net((struct sock *)msk), 430 MPTCP_MIB_RMADDRTXDROP, rm_list->nr); 431 return -EINVAL; 432 } 433 434 msk->pm.rm_list_tx = *rm_list; 435 rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL); 436 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 437 mptcp_pm_addr_send_ack(msk); 438 return 0; 439 } 440 441 /* path manager event handlers */ 442 443 void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side) 444 { 445 struct mptcp_pm_data *pm = &msk->pm; 446 447 pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side); 448 449 WRITE_ONCE(pm->server_side, server_side); 450 mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC); 451 } 452 453 bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk) 454 { 455 struct mptcp_pm_data *pm = &msk->pm; 456 unsigned int subflows_max; 457 int ret = 0; 458 459 if (mptcp_pm_is_userspace(msk)) { 460 if (mptcp_userspace_pm_active(msk)) { 461 spin_lock_bh(&pm->lock); 462 pm->subflows++; 463 spin_unlock_bh(&pm->lock); 464 return true; 465 } 466 return false; 467 } 468 469 subflows_max = mptcp_pm_get_subflows_max(msk); 470 471 pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk, pm->subflows, 472 subflows_max, READ_ONCE(pm->accept_subflow)); 473 474 /* try to avoid acquiring the lock below */ 475 if (!READ_ONCE(pm->accept_subflow)) 476 return false; 477 478 spin_lock_bh(&pm->lock); 479 if (READ_ONCE(pm->accept_subflow)) { 480 ret = pm->subflows < subflows_max; 481 if (ret && ++pm->subflows == subflows_max) 482 WRITE_ONCE(pm->accept_subflow, false); 483 } 484 spin_unlock_bh(&pm->lock); 485 486 return ret; 487 } 488 489 /* return true if the new status bit is currently cleared, that is, this event 490 * can be server, eventually by an already scheduled work 491 */ 492 static bool mptcp_pm_schedule_work(struct mptcp_sock *msk, 493 enum mptcp_pm_status new_status) 494 { 495 pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status, 496 BIT(new_status)); 497 if (msk->pm.status & BIT(new_status)) 498 return false; 499 500 msk->pm.status |= BIT(new_status); 501 mptcp_schedule_work((struct sock *)msk); 502 return true; 503 } 504 505 void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk) 506 { 507 struct mptcp_pm_data *pm = &msk->pm; 508 bool announce = false; 509 510 pr_debug("msk=%p\n", msk); 511 512 spin_lock_bh(&pm->lock); 513 514 /* mptcp_pm_fully_established() can be invoked by multiple 515 * racing paths - accept() and check_fully_established() 516 * be sure to serve this event only once. 517 */ 518 if (READ_ONCE(pm->work_pending) && 519 !(pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED))) 520 mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED); 521 522 if ((pm->status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0) 523 announce = true; 524 525 pm->status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED); 526 spin_unlock_bh(&pm->lock); 527 528 if (announce) 529 mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC); 530 } 531 532 void mptcp_pm_connection_closed(struct mptcp_sock *msk) 533 { 534 pr_debug("msk=%p\n", msk); 535 536 if (msk->token) 537 mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 538 } 539 540 void mptcp_pm_subflow_established(struct mptcp_sock *msk) 541 { 542 struct mptcp_pm_data *pm = &msk->pm; 543 544 pr_debug("msk=%p\n", msk); 545 546 if (!READ_ONCE(pm->work_pending)) 547 return; 548 549 spin_lock_bh(&pm->lock); 550 551 if (READ_ONCE(pm->work_pending)) 552 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 553 554 spin_unlock_bh(&pm->lock); 555 } 556 557 void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, 558 const struct mptcp_subflow_context *subflow) 559 { 560 struct mptcp_pm_data *pm = &msk->pm; 561 bool update_subflows; 562 563 update_subflows = subflow->request_join || subflow->mp_join; 564 if (mptcp_pm_is_userspace(msk)) { 565 if (update_subflows) { 566 spin_lock_bh(&pm->lock); 567 pm->subflows--; 568 spin_unlock_bh(&pm->lock); 569 } 570 return; 571 } 572 573 if (!READ_ONCE(pm->work_pending) && !update_subflows) 574 return; 575 576 spin_lock_bh(&pm->lock); 577 if (update_subflows) 578 __mptcp_pm_close_subflow(msk); 579 580 /* Even if this subflow is not really established, tell the PM to try 581 * to pick the next ones, if possible. 582 */ 583 if (mptcp_pm_nl_check_work_pending(msk)) 584 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 585 586 spin_unlock_bh(&pm->lock); 587 } 588 589 void mptcp_pm_add_addr_received(const struct sock *ssk, 590 const struct mptcp_addr_info *addr) 591 { 592 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 593 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 594 struct mptcp_pm_data *pm = &msk->pm; 595 596 pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id, 597 READ_ONCE(pm->accept_addr)); 598 599 mptcp_event_addr_announced(ssk, addr); 600 601 spin_lock_bh(&pm->lock); 602 603 if (mptcp_pm_is_userspace(msk)) { 604 if (mptcp_userspace_pm_active(msk)) { 605 mptcp_pm_announce_addr(msk, addr, true); 606 mptcp_pm_add_addr_send_ack(msk); 607 } else { 608 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); 609 } 610 /* id0 should not have a different address */ 611 } else if ((addr->id == 0 && !mptcp_pm_is_init_remote_addr(msk, addr)) || 612 (addr->id > 0 && !READ_ONCE(pm->accept_addr))) { 613 mptcp_pm_announce_addr(msk, addr, true); 614 mptcp_pm_add_addr_send_ack(msk); 615 } else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) { 616 pm->remote = *addr; 617 } else { 618 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP); 619 } 620 621 spin_unlock_bh(&pm->lock); 622 } 623 624 void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, 625 const struct mptcp_addr_info *addr) 626 { 627 struct mptcp_pm_data *pm = &msk->pm; 628 629 pr_debug("msk=%p\n", msk); 630 631 if (!READ_ONCE(pm->work_pending)) 632 return; 633 634 spin_lock_bh(&pm->lock); 635 636 if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending)) 637 mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); 638 639 spin_unlock_bh(&pm->lock); 640 } 641 642 void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk) 643 { 644 if (!mptcp_pm_should_add_signal(msk)) 645 return; 646 647 mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK); 648 } 649 650 static void mptcp_pm_rm_addr_or_subflow(struct mptcp_sock *msk, 651 const struct mptcp_rm_list *rm_list, 652 enum linux_mptcp_mib_field rm_type) 653 { 654 struct mptcp_subflow_context *subflow, *tmp; 655 struct sock *sk = (struct sock *)msk; 656 u8 i; 657 658 pr_debug("%s rm_list_nr %d\n", 659 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", rm_list->nr); 660 661 msk_owned_by_me(msk); 662 663 if (sk->sk_state == TCP_LISTEN) 664 return; 665 666 if (!rm_list->nr) 667 return; 668 669 if (list_empty(&msk->conn_list)) 670 return; 671 672 for (i = 0; i < rm_list->nr; i++) { 673 u8 rm_id = rm_list->ids[i]; 674 bool removed = false; 675 676 mptcp_for_each_subflow_safe(msk, subflow, tmp) { 677 struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 678 u8 remote_id = READ_ONCE(subflow->remote_id); 679 int how = RCV_SHUTDOWN | SEND_SHUTDOWN; 680 u8 id = subflow_get_local_id(subflow); 681 682 if ((1 << inet_sk_state_load(ssk)) & 683 (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSING | TCPF_CLOSE)) 684 continue; 685 if (rm_type == MPTCP_MIB_RMADDR && remote_id != rm_id) 686 continue; 687 if (rm_type == MPTCP_MIB_RMSUBFLOW && id != rm_id) 688 continue; 689 690 pr_debug(" -> %s rm_list_ids[%d]=%u local_id=%u remote_id=%u mpc_id=%u\n", 691 rm_type == MPTCP_MIB_RMADDR ? "address" : "subflow", 692 i, rm_id, id, remote_id, msk->mpc_endpoint_id); 693 spin_unlock_bh(&msk->pm.lock); 694 mptcp_subflow_shutdown(sk, ssk, how); 695 removed |= subflow->request_join; 696 697 /* the following takes care of updating the subflows counter */ 698 mptcp_close_ssk(sk, ssk, subflow); 699 spin_lock_bh(&msk->pm.lock); 700 701 if (rm_type == MPTCP_MIB_RMSUBFLOW) 702 __MPTCP_INC_STATS(sock_net(sk), rm_type); 703 } 704 705 if (rm_type == MPTCP_MIB_RMADDR) { 706 __MPTCP_INC_STATS(sock_net(sk), rm_type); 707 if (removed && mptcp_pm_is_kernel(msk)) 708 mptcp_pm_nl_rm_addr(msk, rm_id); 709 } 710 } 711 } 712 713 static void mptcp_pm_rm_addr_recv(struct mptcp_sock *msk) 714 { 715 mptcp_pm_rm_addr_or_subflow(msk, &msk->pm.rm_list_rx, MPTCP_MIB_RMADDR); 716 } 717 718 void mptcp_pm_rm_subflow(struct mptcp_sock *msk, 719 const struct mptcp_rm_list *rm_list) 720 { 721 mptcp_pm_rm_addr_or_subflow(msk, rm_list, MPTCP_MIB_RMSUBFLOW); 722 } 723 724 void mptcp_pm_rm_addr_received(struct mptcp_sock *msk, 725 const struct mptcp_rm_list *rm_list) 726 { 727 struct mptcp_pm_data *pm = &msk->pm; 728 u8 i; 729 730 pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr); 731 732 for (i = 0; i < rm_list->nr; i++) 733 mptcp_event_addr_removed(msk, rm_list->ids[i]); 734 735 spin_lock_bh(&pm->lock); 736 if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED)) 737 pm->rm_list_rx = *rm_list; 738 else 739 __MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP); 740 spin_unlock_bh(&pm->lock); 741 } 742 743 void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup) 744 { 745 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 746 struct sock *sk = subflow->conn; 747 struct mptcp_sock *msk; 748 749 pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup); 750 msk = mptcp_sk(sk); 751 if (subflow->backup != bkup) 752 subflow->backup = bkup; 753 754 mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC); 755 } 756 757 void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq) 758 { 759 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 760 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 761 762 pr_debug("fail_seq=%llu\n", fail_seq); 763 764 if (!READ_ONCE(msk->allow_infinite_fallback)) 765 return; 766 767 if (!subflow->fail_tout) { 768 pr_debug("send MP_FAIL response and infinite map\n"); 769 770 subflow->send_mp_fail = 1; 771 subflow->send_infinite_map = 1; 772 tcp_send_ack(sk); 773 } else { 774 pr_debug("MP_FAIL response received\n"); 775 WRITE_ONCE(subflow->fail_tout, 0); 776 } 777 } 778 779 bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb, 780 unsigned int opt_size, unsigned int remaining, 781 struct mptcp_addr_info *addr, bool *echo, 782 bool *drop_other_suboptions) 783 { 784 int ret = false; 785 u8 add_addr; 786 u8 family; 787 bool port; 788 789 spin_lock_bh(&msk->pm.lock); 790 791 /* double check after the lock is acquired */ 792 if (!mptcp_pm_should_add_signal(msk)) 793 goto out_unlock; 794 795 /* always drop every other options for pure ack ADD_ADDR; this is a 796 * plain dup-ack from TCP perspective. The other MPTCP-relevant info, 797 * if any, will be carried by the 'original' TCP ack 798 */ 799 if (skb && skb_is_tcp_pure_ack(skb)) { 800 remaining += opt_size; 801 *drop_other_suboptions = true; 802 } 803 804 *echo = mptcp_pm_should_add_signal_echo(msk); 805 port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port); 806 807 family = *echo ? msk->pm.remote.family : msk->pm.local.family; 808 if (remaining < mptcp_add_addr_len(family, *echo, port)) 809 goto out_unlock; 810 811 if (*echo) { 812 *addr = msk->pm.remote; 813 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO); 814 } else { 815 *addr = msk->pm.local; 816 add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL); 817 } 818 WRITE_ONCE(msk->pm.addr_signal, add_addr); 819 ret = true; 820 821 out_unlock: 822 spin_unlock_bh(&msk->pm.lock); 823 return ret; 824 } 825 826 bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining, 827 struct mptcp_rm_list *rm_list) 828 { 829 int ret = false, len; 830 u8 rm_addr; 831 832 spin_lock_bh(&msk->pm.lock); 833 834 /* double check after the lock is acquired */ 835 if (!mptcp_pm_should_rm_signal(msk)) 836 goto out_unlock; 837 838 rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL); 839 len = mptcp_rm_addr_len(&msk->pm.rm_list_tx); 840 if (len < 0) { 841 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 842 goto out_unlock; 843 } 844 if (remaining < len) 845 goto out_unlock; 846 847 *rm_list = msk->pm.rm_list_tx; 848 WRITE_ONCE(msk->pm.addr_signal, rm_addr); 849 ret = true; 850 851 out_unlock: 852 spin_unlock_bh(&msk->pm.lock); 853 return ret; 854 } 855 856 int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc) 857 { 858 struct mptcp_pm_addr_entry skc_local = { 0 }; 859 struct mptcp_addr_info msk_local; 860 861 if (WARN_ON_ONCE(!msk)) 862 return -1; 863 864 /* The 0 ID mapping is defined by the first subflow, copied into the msk 865 * addr 866 */ 867 mptcp_local_address((struct sock_common *)msk, &msk_local); 868 mptcp_local_address((struct sock_common *)skc, &skc_local.addr); 869 if (mptcp_addresses_equal(&msk_local, &skc_local.addr, false)) 870 return 0; 871 872 skc_local.addr.id = 0; 873 skc_local.flags = MPTCP_PM_ADDR_FLAG_IMPLICIT; 874 875 if (mptcp_pm_is_userspace(msk)) 876 return mptcp_userspace_pm_get_local_id(msk, &skc_local); 877 return mptcp_pm_nl_get_local_id(msk, &skc_local); 878 } 879 880 bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc) 881 { 882 struct mptcp_addr_info skc_local; 883 884 mptcp_local_address((struct sock_common *)skc, &skc_local); 885 886 if (mptcp_pm_is_userspace(msk)) 887 return mptcp_userspace_pm_is_backup(msk, &skc_local); 888 889 return mptcp_pm_nl_is_backup(msk, &skc_local); 890 } 891 892 static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) 893 { 894 struct mptcp_subflow_context *iter, *subflow = mptcp_subflow_ctx(ssk); 895 struct sock *sk = (struct sock *)msk; 896 unsigned int active_max_loss_cnt; 897 struct net *net = sock_net(sk); 898 unsigned int stale_loss_cnt; 899 bool slow; 900 901 stale_loss_cnt = mptcp_stale_loss_cnt(net); 902 if (subflow->stale || !stale_loss_cnt || subflow->stale_count <= stale_loss_cnt) 903 return; 904 905 /* look for another available subflow not in loss state */ 906 active_max_loss_cnt = max_t(int, stale_loss_cnt - 1, 1); 907 mptcp_for_each_subflow(msk, iter) { 908 if (iter != subflow && mptcp_subflow_active(iter) && 909 iter->stale_count < active_max_loss_cnt) { 910 /* we have some alternatives, try to mark this subflow as idle ...*/ 911 slow = lock_sock_fast(ssk); 912 if (!tcp_rtx_and_write_queues_empty(ssk)) { 913 subflow->stale = 1; 914 __mptcp_retransmit_pending_data(sk); 915 MPTCP_INC_STATS(net, MPTCP_MIB_SUBFLOWSTALE); 916 } 917 unlock_sock_fast(ssk, slow); 918 919 /* always try to push the pending data regardless of re-injections: 920 * we can possibly use backup subflows now, and subflow selection 921 * is cheap under the msk socket lock 922 */ 923 __mptcp_push_pending(sk, 0); 924 return; 925 } 926 } 927 } 928 929 void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) 930 { 931 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 932 u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); 933 934 /* keep track of rtx periods with no progress */ 935 if (!subflow->stale_count) { 936 subflow->stale_rcv_tstamp = rcv_tstamp; 937 subflow->stale_count++; 938 } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { 939 if (subflow->stale_count < U8_MAX) 940 subflow->stale_count++; 941 mptcp_pm_subflows_chk_stale(msk, ssk); 942 } else { 943 subflow->stale_count = 0; 944 mptcp_subflow_set_active(subflow); 945 } 946 } 947 948 void mptcp_pm_worker(struct mptcp_sock *msk) 949 { 950 struct mptcp_pm_data *pm = &msk->pm; 951 952 msk_owned_by_me(msk); 953 954 if (!(pm->status & MPTCP_PM_WORK_MASK)) 955 return; 956 957 spin_lock_bh(&msk->pm.lock); 958 959 pr_debug("msk=%p status=%x\n", msk, pm->status); 960 if (pm->status & BIT(MPTCP_PM_ADD_ADDR_SEND_ACK)) { 961 pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_SEND_ACK); 962 mptcp_pm_addr_send_ack(msk); 963 } 964 if (pm->status & BIT(MPTCP_PM_RM_ADDR_RECEIVED)) { 965 pm->status &= ~BIT(MPTCP_PM_RM_ADDR_RECEIVED); 966 mptcp_pm_rm_addr_recv(msk); 967 } 968 __mptcp_pm_kernel_worker(msk); 969 970 spin_unlock_bh(&msk->pm.lock); 971 } 972 973 void mptcp_pm_destroy(struct mptcp_sock *msk) 974 { 975 mptcp_pm_free_anno_list(msk); 976 977 if (mptcp_pm_is_userspace(msk)) 978 mptcp_userspace_pm_free_local_addr_list(msk); 979 } 980 981 void mptcp_pm_data_reset(struct mptcp_sock *msk) 982 { 983 u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk)); 984 struct mptcp_pm_data *pm = &msk->pm; 985 986 memset(&pm->reset, 0, sizeof(pm->reset)); 987 pm->rm_list_tx.nr = 0; 988 pm->rm_list_rx.nr = 0; 989 WRITE_ONCE(pm->pm_type, pm_type); 990 991 if (pm_type == MPTCP_PM_TYPE_KERNEL) { 992 bool subflows_allowed = !!mptcp_pm_get_subflows_max(msk); 993 994 /* pm->work_pending must be only be set to 'true' when 995 * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL 996 */ 997 WRITE_ONCE(pm->work_pending, 998 (!!mptcp_pm_get_local_addr_max(msk) && 999 subflows_allowed) || 1000 !!mptcp_pm_get_add_addr_signal_max(msk)); 1001 WRITE_ONCE(pm->accept_addr, 1002 !!mptcp_pm_get_add_addr_accept_max(msk) && 1003 subflows_allowed); 1004 WRITE_ONCE(pm->accept_subflow, subflows_allowed); 1005 1006 bitmap_fill(pm->id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); 1007 } 1008 } 1009 1010 void mptcp_pm_data_init(struct mptcp_sock *msk) 1011 { 1012 spin_lock_init(&msk->pm.lock); 1013 INIT_LIST_HEAD(&msk->pm.anno_list); 1014 INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list); 1015 mptcp_pm_data_reset(msk); 1016 } 1017 1018 void __init mptcp_pm_init(void) 1019 { 1020 mptcp_pm_kernel_register(); 1021 mptcp_pm_userspace_register(); 1022 mptcp_pm_nl_init(); 1023 } 1024 1025 /* Must be called with rcu read lock held */ 1026 struct mptcp_pm_ops *mptcp_pm_find(const char *name) 1027 { 1028 struct mptcp_pm_ops *pm_ops; 1029 1030 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) { 1031 if (!strcmp(pm_ops->name, name)) 1032 return pm_ops; 1033 } 1034 1035 return NULL; 1036 } 1037 1038 int mptcp_pm_validate(struct mptcp_pm_ops *pm_ops) 1039 { 1040 return 0; 1041 } 1042 1043 int mptcp_pm_register(struct mptcp_pm_ops *pm_ops) 1044 { 1045 int ret; 1046 1047 ret = mptcp_pm_validate(pm_ops); 1048 if (ret) 1049 return ret; 1050 1051 spin_lock(&mptcp_pm_list_lock); 1052 if (mptcp_pm_find(pm_ops->name)) { 1053 spin_unlock(&mptcp_pm_list_lock); 1054 return -EEXIST; 1055 } 1056 list_add_tail_rcu(&pm_ops->list, &mptcp_pm_list); 1057 spin_unlock(&mptcp_pm_list_lock); 1058 1059 pr_debug("%s registered\n", pm_ops->name); 1060 return 0; 1061 } 1062 1063 void mptcp_pm_unregister(struct mptcp_pm_ops *pm_ops) 1064 { 1065 /* skip unregistering the default path manager */ 1066 if (WARN_ON_ONCE(pm_ops == &mptcp_pm_kernel)) 1067 return; 1068 1069 spin_lock(&mptcp_pm_list_lock); 1070 list_del_rcu(&pm_ops->list); 1071 spin_unlock(&mptcp_pm_list_lock); 1072 } 1073 1074 /* Build string with list of available path manager values. 1075 * Similar to tcp_get_available_congestion_control() 1076 */ 1077 void mptcp_pm_get_available(char *buf, size_t maxlen) 1078 { 1079 struct mptcp_pm_ops *pm_ops; 1080 size_t offs = 0; 1081 1082 rcu_read_lock(); 1083 list_for_each_entry_rcu(pm_ops, &mptcp_pm_list, list) { 1084 offs += snprintf(buf + offs, maxlen - offs, "%s%s", 1085 offs == 0 ? "" : " ", pm_ops->name); 1086 1087 if (WARN_ON_ONCE(offs >= maxlen)) 1088 break; 1089 } 1090 rcu_read_unlock(); 1091 } 1092