1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Support for INET connection oriented protocols. 8 * 9 * Authors: See the TCP sources 10 */ 11 12 #include <linux/module.h> 13 #include <linux/jhash.h> 14 15 #include <net/inet_connection_sock.h> 16 #include <net/inet_hashtables.h> 17 #include <net/inet_timewait_sock.h> 18 #include <net/ip.h> 19 #include <net/route.h> 20 #include <net/tcp_states.h> 21 #include <net/xfrm.h> 22 #include <net/tcp.h> 23 #include <net/tcp_ecn.h> 24 #include <net/mptcp.h> 25 #include <net/sock_reuseport.h> 26 #include <net/addrconf.h> 27 28 #if IS_ENABLED(CONFIG_IPV6) 29 /* match_sk*_wildcard == true: IPV6_ADDR_ANY equals to any IPv6 addresses 30 * if IPv6 only, and any IPv4 addresses 31 * if not IPv6 only 32 * match_sk*_wildcard == false: addresses must be exactly the same, i.e. 33 * IPV6_ADDR_ANY only equals to IPV6_ADDR_ANY, 34 * and 0.0.0.0 equals to 0.0.0.0 only 35 */ 36 static bool ipv6_rcv_saddr_equal(const struct in6_addr *sk1_rcv_saddr6, 37 const struct in6_addr *sk2_rcv_saddr6, 38 __be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr, 39 bool sk1_ipv6only, bool sk2_ipv6only, 40 bool match_sk1_wildcard, 41 bool match_sk2_wildcard) 42 { 43 int addr_type = ipv6_addr_type(sk1_rcv_saddr6); 44 int addr_type2 = sk2_rcv_saddr6 ? ipv6_addr_type(sk2_rcv_saddr6) : IPV6_ADDR_MAPPED; 45 46 /* if both are mapped, treat as IPv4 */ 47 if (addr_type == IPV6_ADDR_MAPPED && addr_type2 == IPV6_ADDR_MAPPED) { 48 if (!sk2_ipv6only) { 49 if (sk1_rcv_saddr == sk2_rcv_saddr) 50 return true; 51 return (match_sk1_wildcard && !sk1_rcv_saddr) || 52 (match_sk2_wildcard && !sk2_rcv_saddr); 53 } 54 return false; 55 } 56 57 if (addr_type == IPV6_ADDR_ANY && addr_type2 == IPV6_ADDR_ANY) 58 return true; 59 60 if (addr_type2 == IPV6_ADDR_ANY && match_sk2_wildcard && 61 !(sk2_ipv6only && addr_type == IPV6_ADDR_MAPPED)) 62 return true; 63 64 if (addr_type == IPV6_ADDR_ANY && match_sk1_wildcard && 65 !(sk1_ipv6only && addr_type2 == IPV6_ADDR_MAPPED)) 66 return true; 67 68 if (sk2_rcv_saddr6 && 69 ipv6_addr_equal(sk1_rcv_saddr6, sk2_rcv_saddr6)) 70 return true; 71 72 return false; 73 } 74 #endif 75 76 /* match_sk*_wildcard == true: 0.0.0.0 equals to any IPv4 addresses 77 * match_sk*_wildcard == false: addresses must be exactly the same, i.e. 78 * 0.0.0.0 only equals to 0.0.0.0 79 */ 80 static bool ipv4_rcv_saddr_equal(__be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr, 81 bool sk2_ipv6only, bool match_sk1_wildcard, 82 bool match_sk2_wildcard) 83 { 84 if (!sk2_ipv6only) { 85 if (sk1_rcv_saddr == sk2_rcv_saddr) 86 return true; 87 return (match_sk1_wildcard && !sk1_rcv_saddr) || 88 (match_sk2_wildcard && !sk2_rcv_saddr); 89 } 90 return false; 91 } 92 93 bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2, 94 bool match_wildcard) 95 { 96 #if IS_ENABLED(CONFIG_IPV6) 97 if (sk->sk_family == AF_INET6) 98 return ipv6_rcv_saddr_equal(&sk->sk_v6_rcv_saddr, 99 inet6_rcv_saddr(sk2), 100 sk->sk_rcv_saddr, 101 sk2->sk_rcv_saddr, 102 ipv6_only_sock(sk), 103 ipv6_only_sock(sk2), 104 match_wildcard, 105 match_wildcard); 106 #endif 107 return ipv4_rcv_saddr_equal(sk->sk_rcv_saddr, sk2->sk_rcv_saddr, 108 ipv6_only_sock(sk2), match_wildcard, 109 match_wildcard); 110 } 111 112 bool inet_rcv_saddr_any(const struct sock *sk) 113 { 114 #if IS_ENABLED(CONFIG_IPV6) 115 if (sk->sk_family == AF_INET6) 116 return ipv6_addr_any(&sk->sk_v6_rcv_saddr); 117 #endif 118 return !sk->sk_rcv_saddr; 119 } 120 121 /** 122 * inet_sk_get_local_port_range - fetch ephemeral ports range 123 * @sk: socket 124 * @low: pointer to low port 125 * @high: pointer to high port 126 * 127 * Fetch netns port range (/proc/sys/net/ipv4/ip_local_port_range) 128 * Range can be overridden if socket got IP_LOCAL_PORT_RANGE option. 129 * Returns true if IP_LOCAL_PORT_RANGE was set on this socket. 130 */ 131 bool inet_sk_get_local_port_range(const struct sock *sk, int *low, int *high) 132 { 133 int lo, hi, sk_lo, sk_hi; 134 bool local_range = false; 135 u32 sk_range; 136 137 inet_get_local_port_range(sock_net(sk), &lo, &hi); 138 139 sk_range = READ_ONCE(inet_sk(sk)->local_port_range); 140 if (unlikely(sk_range)) { 141 sk_lo = sk_range & 0xffff; 142 sk_hi = sk_range >> 16; 143 144 if (lo <= sk_lo && sk_lo <= hi) 145 lo = sk_lo; 146 if (lo <= sk_hi && sk_hi <= hi) 147 hi = sk_hi; 148 local_range = true; 149 } 150 151 *low = lo; 152 *high = hi; 153 return local_range; 154 } 155 EXPORT_SYMBOL(inet_sk_get_local_port_range); 156 157 static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2, 158 kuid_t uid, bool relax, 159 bool reuseport_cb_ok, bool reuseport_ok) 160 { 161 int bound_dev_if2; 162 163 if (sk == sk2) 164 return false; 165 166 bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if); 167 168 if (!sk->sk_bound_dev_if || !bound_dev_if2 || 169 sk->sk_bound_dev_if == bound_dev_if2) { 170 if (sk->sk_reuse && sk2->sk_reuse && 171 sk2->sk_state != TCP_LISTEN) { 172 if (!relax || (!reuseport_ok && sk->sk_reuseport && 173 sk2->sk_reuseport && reuseport_cb_ok && 174 (sk2->sk_state == TCP_TIME_WAIT || 175 uid_eq(uid, sk_uid(sk2))))) 176 return true; 177 } else if (!reuseport_ok || !sk->sk_reuseport || 178 !sk2->sk_reuseport || !reuseport_cb_ok || 179 (sk2->sk_state != TCP_TIME_WAIT && 180 !uid_eq(uid, sk_uid(sk2)))) { 181 return true; 182 } 183 } 184 return false; 185 } 186 187 static bool __inet_bhash2_conflict(const struct sock *sk, struct sock *sk2, 188 kuid_t uid, bool relax, 189 bool reuseport_cb_ok, bool reuseport_ok) 190 { 191 if (ipv6_only_sock(sk2)) { 192 if (sk->sk_family == AF_INET) 193 return false; 194 195 #if IS_ENABLED(CONFIG_IPV6) 196 if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) 197 return false; 198 #endif 199 } 200 201 return inet_bind_conflict(sk, sk2, uid, relax, 202 reuseport_cb_ok, reuseport_ok); 203 } 204 205 static bool inet_bhash2_conflict(const struct sock *sk, 206 const struct inet_bind2_bucket *tb2, 207 kuid_t uid, 208 bool relax, bool reuseport_cb_ok, 209 bool reuseport_ok) 210 { 211 struct sock *sk2; 212 213 sk_for_each_bound(sk2, &tb2->owners) { 214 if (__inet_bhash2_conflict(sk, sk2, uid, relax, 215 reuseport_cb_ok, reuseport_ok)) 216 return true; 217 } 218 219 return false; 220 } 221 222 #define sk_for_each_bound_bhash(__sk, __tb2, __tb) \ 223 hlist_for_each_entry(__tb2, &(__tb)->bhash2, bhash_node) \ 224 sk_for_each_bound((__sk), &(__tb2)->owners) 225 226 /* This should be called only when the tb and tb2 hashbuckets' locks are held */ 227 static int inet_csk_bind_conflict(const struct sock *sk, 228 const struct inet_bind_bucket *tb, 229 const struct inet_bind2_bucket *tb2, /* may be null */ 230 bool relax, bool reuseport_ok) 231 { 232 struct sock_reuseport *reuseport_cb; 233 kuid_t uid = sk_uid(sk); 234 bool reuseport_cb_ok; 235 struct sock *sk2; 236 237 rcu_read_lock(); 238 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb); 239 /* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */ 240 reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks); 241 rcu_read_unlock(); 242 243 /* Conflicts with an existing IPV6_ADDR_ANY (if ipv6) or INADDR_ANY (if 244 * ipv4) should have been checked already. We need to do these two 245 * checks separately because their spinlocks have to be acquired/released 246 * independently of each other, to prevent possible deadlocks 247 */ 248 if (inet_use_hash2_on_bind(sk)) 249 return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, 250 reuseport_cb_ok, reuseport_ok); 251 252 /* Unlike other sk lookup places we do not check 253 * for sk_net here, since _all_ the socks listed 254 * in tb->owners and tb2->owners list belong 255 * to the same net - the one this bucket belongs to. 256 */ 257 sk_for_each_bound_bhash(sk2, tb2, tb) { 258 if (!inet_bind_conflict(sk, sk2, uid, relax, reuseport_cb_ok, reuseport_ok)) 259 continue; 260 261 if (inet_rcv_saddr_equal(sk, sk2, true)) 262 return true; 263 } 264 265 return false; 266 } 267 268 /* Determine if there is a bind conflict with an existing IPV6_ADDR_ANY (if ipv6) or 269 * INADDR_ANY (if ipv4) socket. 270 * 271 * Caller must hold bhash hashbucket lock with local bh disabled, to protect 272 * against concurrent binds on the port for addr any 273 */ 274 static bool inet_bhash2_addr_any_conflict(const struct sock *sk, int port, int l3mdev, 275 bool relax, bool reuseport_ok) 276 { 277 const struct net *net = sock_net(sk); 278 struct sock_reuseport *reuseport_cb; 279 struct inet_bind_hashbucket *head2; 280 struct inet_bind2_bucket *tb2; 281 kuid_t uid = sk_uid(sk); 282 bool conflict = false; 283 bool reuseport_cb_ok; 284 285 rcu_read_lock(); 286 reuseport_cb = rcu_dereference(sk->sk_reuseport_cb); 287 /* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */ 288 reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks); 289 rcu_read_unlock(); 290 291 head2 = inet_bhash2_addr_any_hashbucket(sk, net, port); 292 293 spin_lock(&head2->lock); 294 295 inet_bind_bucket_for_each(tb2, &head2->chain) { 296 if (!inet_bind2_bucket_match_addr_any(tb2, net, port, l3mdev, sk)) 297 continue; 298 299 if (!inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, reuseport_ok)) 300 continue; 301 302 conflict = true; 303 break; 304 } 305 306 spin_unlock(&head2->lock); 307 308 return conflict; 309 } 310 311 /* 312 * Find an open port number for the socket. Returns with the 313 * inet_bind_hashbucket locks held if successful. 314 */ 315 static struct inet_bind_hashbucket * 316 inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret, 317 struct inet_bind2_bucket **tb2_ret, 318 struct inet_bind_hashbucket **head2_ret, int *port_ret) 319 { 320 struct inet_hashinfo *hinfo = tcp_get_hashinfo(sk); 321 int i, low, high, attempt_half, port, l3mdev; 322 struct inet_bind_hashbucket *head, *head2; 323 struct net *net = sock_net(sk); 324 struct inet_bind2_bucket *tb2; 325 struct inet_bind_bucket *tb; 326 u32 remaining, offset; 327 bool relax = false; 328 329 l3mdev = inet_sk_bound_l3mdev(sk); 330 ports_exhausted: 331 attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0; 332 other_half_scan: 333 inet_sk_get_local_port_range(sk, &low, &high); 334 high++; /* [32768, 60999] -> [32768, 61000[ */ 335 if (high - low < 4) 336 attempt_half = 0; 337 if (attempt_half) { 338 int half = low + (((high - low) >> 2) << 1); 339 340 if (attempt_half == 1) 341 high = half; 342 else 343 low = half; 344 } 345 remaining = high - low; 346 if (likely(remaining > 1)) 347 remaining &= ~1U; 348 349 offset = get_random_u32_below(remaining); 350 /* __inet_hash_connect() favors ports having @low parity 351 * We do the opposite to not pollute connect() users. 352 */ 353 offset |= 1U; 354 355 other_parity_scan: 356 port = low + offset; 357 for (i = 0; i < remaining; i += 2, port += 2) { 358 if (unlikely(port >= high)) 359 port -= remaining; 360 if (inet_is_local_reserved_port(net, port)) 361 continue; 362 head = &hinfo->bhash[inet_bhashfn(net, port, 363 hinfo->bhash_size)]; 364 spin_lock_bh(&head->lock); 365 if (inet_use_hash2_on_bind(sk)) { 366 if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false)) 367 goto next_port; 368 } 369 370 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); 371 spin_lock(&head2->lock); 372 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 373 inet_bind_bucket_for_each(tb, &head->chain) 374 if (inet_bind_bucket_match(tb, net, port, l3mdev)) { 375 if (!inet_csk_bind_conflict(sk, tb, tb2, 376 relax, false)) 377 goto success; 378 spin_unlock(&head2->lock); 379 goto next_port; 380 } 381 tb = NULL; 382 goto success; 383 next_port: 384 spin_unlock_bh(&head->lock); 385 cond_resched(); 386 } 387 388 offset--; 389 if (!(offset & 1)) 390 goto other_parity_scan; 391 392 if (attempt_half == 1) { 393 /* OK we now try the upper half of the range */ 394 attempt_half = 2; 395 goto other_half_scan; 396 } 397 398 if (READ_ONCE(net->ipv4.sysctl_ip_autobind_reuse) && !relax) { 399 /* We still have a chance to connect to different destinations */ 400 relax = true; 401 goto ports_exhausted; 402 } 403 return NULL; 404 success: 405 *port_ret = port; 406 *tb_ret = tb; 407 *tb2_ret = tb2; 408 *head2_ret = head2; 409 return head; 410 } 411 412 static inline int sk_reuseport_match(struct inet_bind_bucket *tb, 413 const struct sock *sk) 414 { 415 if (tb->fastreuseport <= 0) 416 return 0; 417 if (!sk->sk_reuseport) 418 return 0; 419 if (rcu_access_pointer(sk->sk_reuseport_cb)) 420 return 0; 421 if (!uid_eq(tb->fastuid, sk_uid(sk))) 422 return 0; 423 /* We only need to check the fast_* fields if this tb was once marked 424 * without fastreuseport and then was reset, as we can only know that 425 * the socket they describe doesn't have any conflicts with the socks 426 * on the owners list. 427 */ 428 if (tb->fastreuseport == FASTREUSEPORT_ANY) 429 return 1; 430 if (tb->fast_sk_family != sk->sk_family || 431 tb->fast_ipv6_only != ipv6_only_sock(sk) || 432 tb->fast_bound_dev_if != sk->sk_bound_dev_if || 433 tb->fast_sk_reuse != sk->sk_reuse) 434 return 0; 435 #if IS_ENABLED(CONFIG_IPV6) 436 if (tb->fast_sk_family == AF_INET6) 437 return ipv6_rcv_saddr_equal(&tb->fast_v6_rcv_saddr, 438 inet6_rcv_saddr(sk), 439 tb->fast_rcv_saddr, 440 sk->sk_rcv_saddr, 441 tb->fast_ipv6_only, 442 ipv6_only_sock(sk), true, false); 443 #endif 444 return ipv4_rcv_saddr_equal(tb->fast_rcv_saddr, sk->sk_rcv_saddr, 445 ipv6_only_sock(sk), true, false); 446 } 447 448 void inet_csk_update_fastreuse(const struct sock *sk, 449 struct inet_bind_bucket *tb, 450 struct inet_bind2_bucket *tb2) 451 { 452 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; 453 454 if (hlist_is_singular_node(&tb2->bhash_node, &tb->bhash2) && 455 hlist_empty(&tb2->owners)) { 456 tb->fastreuse = reuse; 457 if (sk->sk_reuseport) { 458 tb->fastreuseport = FASTREUSEPORT_ANY; 459 tb->fastuid = sk_uid(sk); 460 tb->fast_rcv_saddr = sk->sk_rcv_saddr; 461 tb->fast_ipv6_only = ipv6_only_sock(sk); 462 tb->fast_sk_family = sk->sk_family; 463 tb->fast_sk_reuse = sk->sk_reuse; 464 tb->fast_bound_dev_if = sk->sk_bound_dev_if; 465 #if IS_ENABLED(CONFIG_IPV6) 466 tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr; 467 #endif 468 } else { 469 tb->fastreuseport = 0; 470 } 471 } else { 472 if (!reuse) 473 tb->fastreuse = 0; 474 if (sk->sk_reuseport) { 475 /* We didn't match or we don't have fastreuseport set on 476 * the tb, but we have sk_reuseport set on this socket 477 * and we know that there are no bind conflicts with 478 * this socket in this tb, so reset our tb's reuseport 479 * settings so that any subsequent sockets that match 480 * our current socket will be put on the fast path. 481 * 482 * If we reset we need to set FASTREUSEPORT_STRICT so we 483 * do extra checking for all subsequent sk_reuseport 484 * socks. 485 */ 486 if (!sk_reuseport_match(tb, sk)) { 487 tb->fastreuseport = FASTREUSEPORT_STRICT; 488 tb->fastuid = sk_uid(sk); 489 tb->fast_rcv_saddr = sk->sk_rcv_saddr; 490 tb->fast_ipv6_only = ipv6_only_sock(sk); 491 tb->fast_sk_family = sk->sk_family; 492 tb->fast_sk_reuse = sk->sk_reuse; 493 tb->fast_bound_dev_if = sk->sk_bound_dev_if; 494 #if IS_ENABLED(CONFIG_IPV6) 495 tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr; 496 #endif 497 } 498 } else { 499 tb->fastreuseport = 0; 500 } 501 } 502 503 tb2->fastreuse = tb->fastreuse; 504 tb2->fastreuseport = tb->fastreuseport; 505 } 506 507 /* Obtain a reference to a local port for the given sock, 508 * if snum is zero it means select any available local port. 509 * We try to allocate an odd port (and leave even ports for connect()) 510 */ 511 int inet_csk_get_port(struct sock *sk, unsigned short snum) 512 { 513 bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; 514 bool found_port = false, check_bind_conflict = true; 515 bool bhash_created = false, bhash2_created = false; 516 struct inet_hashinfo *hinfo = tcp_get_hashinfo(sk); 517 int ret = -EADDRINUSE, port = snum, l3mdev; 518 struct inet_bind_hashbucket *head, *head2; 519 struct inet_bind2_bucket *tb2 = NULL; 520 struct inet_bind_bucket *tb = NULL; 521 bool head2_lock_acquired = false; 522 struct net *net = sock_net(sk); 523 524 l3mdev = inet_sk_bound_l3mdev(sk); 525 526 if (!port) { 527 head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port); 528 if (!head) 529 return ret; 530 531 head2_lock_acquired = true; 532 533 if (tb && tb2) 534 goto success; 535 found_port = true; 536 } else { 537 head = &hinfo->bhash[inet_bhashfn(net, port, 538 hinfo->bhash_size)]; 539 spin_lock_bh(&head->lock); 540 inet_bind_bucket_for_each(tb, &head->chain) 541 if (inet_bind_bucket_match(tb, net, port, l3mdev)) 542 break; 543 } 544 545 if (!tb) { 546 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net, 547 head, port, l3mdev); 548 if (!tb) 549 goto fail_unlock; 550 bhash_created = true; 551 } 552 553 if (!found_port) { 554 if (!hlist_empty(&tb->bhash2)) { 555 if (sk->sk_reuse == SK_FORCE_REUSE || 556 (tb->fastreuse > 0 && reuse) || 557 sk_reuseport_match(tb, sk)) 558 check_bind_conflict = false; 559 } 560 561 if (check_bind_conflict && inet_use_hash2_on_bind(sk)) { 562 if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true)) 563 goto fail_unlock; 564 } 565 566 head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); 567 spin_lock(&head2->lock); 568 head2_lock_acquired = true; 569 tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); 570 } 571 572 if (!tb2) { 573 tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, 574 net, head2, tb, sk); 575 if (!tb2) 576 goto fail_unlock; 577 bhash2_created = true; 578 } 579 580 if (!found_port && check_bind_conflict) { 581 if (inet_csk_bind_conflict(sk, tb, tb2, true, true)) 582 goto fail_unlock; 583 } 584 585 success: 586 inet_csk_update_fastreuse(sk, tb, tb2); 587 588 if (!inet_csk(sk)->icsk_bind_hash) 589 inet_bind_hash(sk, tb, tb2, port); 590 WARN_ON(inet_csk(sk)->icsk_bind_hash != tb); 591 WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2); 592 ret = 0; 593 594 fail_unlock: 595 if (ret) { 596 if (bhash2_created) 597 inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, tb2); 598 if (bhash_created) 599 inet_bind_bucket_destroy(tb); 600 } 601 if (head2_lock_acquired) 602 spin_unlock(&head2->lock); 603 spin_unlock_bh(&head->lock); 604 return ret; 605 } 606 EXPORT_SYMBOL_GPL(inet_csk_get_port); 607 608 /* 609 * Wait for an incoming connection, avoid race conditions. This must be called 610 * with the socket locked. 611 */ 612 static int inet_csk_wait_for_connect(struct sock *sk, long timeo) 613 { 614 struct inet_connection_sock *icsk = inet_csk(sk); 615 DEFINE_WAIT(wait); 616 int err; 617 618 /* 619 * True wake-one mechanism for incoming connections: only 620 * one process gets woken up, not the 'whole herd'. 621 * Since we do not 'race & poll' for established sockets 622 * anymore, the common case will execute the loop only once. 623 * 624 * Subtle issue: "add_wait_queue_exclusive()" will be added 625 * after any current non-exclusive waiters, and we know that 626 * it will always _stay_ after any new non-exclusive waiters 627 * because all non-exclusive waiters are added at the 628 * beginning of the wait-queue. As such, it's ok to "drop" 629 * our exclusiveness temporarily when we get woken up without 630 * having to remove and re-insert us on the wait queue. 631 */ 632 for (;;) { 633 prepare_to_wait_exclusive(sk_sleep(sk), &wait, 634 TASK_INTERRUPTIBLE); 635 release_sock(sk); 636 if (reqsk_queue_empty(&icsk->icsk_accept_queue)) 637 timeo = schedule_timeout(timeo); 638 sched_annotate_sleep(); 639 lock_sock(sk); 640 err = 0; 641 if (!reqsk_queue_empty(&icsk->icsk_accept_queue)) 642 break; 643 err = -EINVAL; 644 if (sk->sk_state != TCP_LISTEN) 645 break; 646 err = sock_intr_errno(timeo); 647 if (signal_pending(current)) 648 break; 649 err = -EAGAIN; 650 if (!timeo) 651 break; 652 } 653 finish_wait(sk_sleep(sk), &wait); 654 return err; 655 } 656 657 /* 658 * This will accept the next outstanding connection. 659 */ 660 struct sock *inet_csk_accept(struct sock *sk, struct proto_accept_arg *arg) 661 { 662 struct inet_connection_sock *icsk = inet_csk(sk); 663 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 664 struct request_sock *req; 665 struct sock *newsk; 666 int error; 667 668 lock_sock(sk); 669 670 /* We need to make sure that this socket is listening, 671 * and that it has something pending. 672 */ 673 error = -EINVAL; 674 if (sk->sk_state != TCP_LISTEN) 675 goto out_err; 676 677 /* Find already established connection */ 678 if (reqsk_queue_empty(queue)) { 679 long timeo = sock_rcvtimeo(sk, arg->flags & O_NONBLOCK); 680 681 /* If this is a non blocking socket don't sleep */ 682 error = -EAGAIN; 683 if (!timeo) 684 goto out_err; 685 686 error = inet_csk_wait_for_connect(sk, timeo); 687 if (error) 688 goto out_err; 689 } 690 req = reqsk_queue_remove(queue, sk); 691 arg->is_empty = reqsk_queue_empty(queue); 692 newsk = req->sk; 693 694 if (sk->sk_protocol == IPPROTO_TCP && 695 tcp_sk(newsk)->syn_fastopen_child) { 696 spin_lock_bh(&queue->fastopenq.lock); 697 if (tcp_rsk(req)->tfo_listener) { 698 /* We are still waiting for the final ACK from 3WHS 699 * so can't free req now. Instead, we set req->sk to 700 * NULL to signify that the child socket is taken 701 * so reqsk_fastopen_remove() will free the req 702 * when 3WHS finishes (or is aborted). 703 */ 704 req->sk = NULL; 705 req = NULL; 706 } 707 spin_unlock_bh(&queue->fastopenq.lock); 708 } 709 710 release_sock(sk); 711 712 if (req) 713 reqsk_put(req); 714 715 inet_init_csk_locks(newsk); 716 return newsk; 717 718 out_err: 719 release_sock(sk); 720 arg->err = error; 721 return NULL; 722 } 723 724 /* 725 * Using different timers for retransmit, delayed acks and probes 726 * We may wish use just one timer maintaining a list of expire jiffies 727 * to optimize. 728 */ 729 void inet_csk_init_xmit_timers(struct sock *sk, 730 void (*retransmit_handler)(struct timer_list *t), 731 void (*delack_handler)(struct timer_list *t), 732 void (*keepalive_handler)(struct timer_list *t)) 733 { 734 struct inet_connection_sock *icsk = inet_csk(sk); 735 736 timer_setup(&sk->tcp_retransmit_timer, retransmit_handler, 0); 737 timer_setup(&icsk->icsk_delack_timer, delack_handler, 0); 738 timer_setup(&icsk->icsk_keepalive_timer, keepalive_handler, 0); 739 icsk->icsk_pending = icsk->icsk_ack.pending = 0; 740 } 741 742 void inet_csk_clear_xmit_timers(struct sock *sk) 743 { 744 struct inet_connection_sock *icsk = inet_csk(sk); 745 746 smp_store_release(&icsk->icsk_pending, 0); 747 smp_store_release(&icsk->icsk_ack.pending, 0); 748 749 sk_stop_timer(sk, &sk->tcp_retransmit_timer); 750 sk_stop_timer(sk, &icsk->icsk_delack_timer); 751 sk_stop_timer(sk, &icsk->icsk_keepalive_timer); 752 } 753 754 void inet_csk_clear_xmit_timers_sync(struct sock *sk) 755 { 756 struct inet_connection_sock *icsk = inet_csk(sk); 757 758 /* ongoing timer handlers need to acquire socket lock. */ 759 sock_not_owned_by_me(sk); 760 761 smp_store_release(&icsk->icsk_pending, 0); 762 smp_store_release(&icsk->icsk_ack.pending, 0); 763 764 sk_stop_timer_sync(sk, &sk->tcp_retransmit_timer); 765 sk_stop_timer_sync(sk, &icsk->icsk_delack_timer); 766 sk_stop_timer_sync(sk, &icsk->icsk_keepalive_timer); 767 } 768 769 struct dst_entry *inet_csk_route_req(const struct sock *sk, 770 struct flowi4 *fl4, 771 const struct request_sock *req) 772 { 773 const struct inet_request_sock *ireq = inet_rsk(req); 774 struct net *net = read_pnet(&ireq->ireq_net); 775 struct ip_options_rcu *opt; 776 struct rtable *rt; 777 778 rcu_read_lock(); 779 opt = rcu_dereference(ireq->ireq_opt); 780 781 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark, 782 ip_sock_rt_tos(sk), ip_sock_rt_scope(sk), 783 sk->sk_protocol, inet_sk_flowi_flags(sk), 784 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, 785 ireq->ir_loc_addr, ireq->ir_rmt_port, 786 htons(ireq->ir_num), sk_uid(sk)); 787 security_req_classify_flow(req, flowi4_to_flowi_common(fl4)); 788 rt = ip_route_output_flow(net, fl4, sk); 789 if (IS_ERR(rt)) 790 goto no_route; 791 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway) 792 goto route_err; 793 rcu_read_unlock(); 794 return &rt->dst; 795 796 route_err: 797 ip_rt_put(rt); 798 no_route: 799 rcu_read_unlock(); 800 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 801 return NULL; 802 } 803 804 struct dst_entry *inet_csk_route_child_sock(const struct sock *sk, 805 struct sock *newsk, 806 const struct request_sock *req) 807 { 808 const struct inet_request_sock *ireq = inet_rsk(req); 809 struct net *net = read_pnet(&ireq->ireq_net); 810 struct inet_sock *newinet = inet_sk(newsk); 811 struct ip_options_rcu *opt; 812 struct flowi4 *fl4; 813 struct rtable *rt; 814 815 opt = rcu_dereference(ireq->ireq_opt); 816 fl4 = &newinet->cork.fl.u.ip4; 817 818 flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark, 819 ip_sock_rt_tos(sk), ip_sock_rt_scope(sk), 820 sk->sk_protocol, inet_sk_flowi_flags(sk), 821 (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr, 822 ireq->ir_loc_addr, ireq->ir_rmt_port, 823 htons(ireq->ir_num), sk_uid(sk)); 824 security_req_classify_flow(req, flowi4_to_flowi_common(fl4)); 825 rt = ip_route_output_flow(net, fl4, sk); 826 if (IS_ERR(rt)) 827 goto no_route; 828 if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway) 829 goto route_err; 830 return &rt->dst; 831 832 route_err: 833 ip_rt_put(rt); 834 no_route: 835 __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 836 return NULL; 837 } 838 839 /* Decide when to expire the request and when to resend SYN-ACK */ 840 static void syn_ack_recalc(struct request_sock *req, 841 const int max_syn_ack_retries, 842 const u8 rskq_defer_accept, 843 int *expire, int *resend) 844 { 845 if (!rskq_defer_accept) { 846 *expire = req->num_timeout >= max_syn_ack_retries; 847 *resend = 1; 848 return; 849 } 850 *expire = req->num_timeout >= max_syn_ack_retries && 851 (!inet_rsk(req)->acked || req->num_timeout >= rskq_defer_accept); 852 /* Do not resend while waiting for data after ACK, 853 * start to resend on end of deferring period to give 854 * last chance for data or ACK to create established socket. 855 */ 856 *resend = !inet_rsk(req)->acked || 857 req->num_timeout >= rskq_defer_accept - 1; 858 } 859 860 static struct request_sock * 861 reqsk_alloc_noprof(const struct request_sock_ops *ops, struct sock *sk_listener, 862 bool attach_listener) 863 { 864 struct request_sock *req; 865 866 req = kmem_cache_alloc_noprof(ops->slab, GFP_ATOMIC | __GFP_NOWARN); 867 if (!req) 868 return NULL; 869 req->rsk_listener = NULL; 870 if (attach_listener) { 871 if (unlikely(!refcount_inc_not_zero(&sk_listener->sk_refcnt))) { 872 kmem_cache_free(ops->slab, req); 873 return NULL; 874 } 875 req->rsk_listener = sk_listener; 876 } 877 req->rsk_ops = ops; 878 req_to_sk(req)->sk_prot = sk_listener->sk_prot; 879 sk_node_init(&req_to_sk(req)->sk_node); 880 sk_tx_queue_clear(req_to_sk(req)); 881 RCU_INIT_POINTER(req->saved_syn, NULL); 882 req->syncookie = 0; 883 req->num_timeout = 0; 884 req->num_retrans = 0; 885 req->sk = NULL; 886 refcount_set(&req->rsk_refcnt, 0); 887 888 return req; 889 } 890 #define reqsk_alloc(...) alloc_hooks(reqsk_alloc_noprof(__VA_ARGS__)) 891 892 struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops, 893 struct sock *sk_listener, 894 bool attach_listener) 895 { 896 struct request_sock *req = reqsk_alloc(ops, sk_listener, 897 attach_listener); 898 899 if (req) { 900 struct inet_request_sock *ireq = inet_rsk(req); 901 902 ireq->ireq_opt = NULL; 903 #if IS_ENABLED(CONFIG_IPV6) 904 ireq->pktopts = NULL; 905 #endif 906 atomic64_set(&ireq->ir_cookie, 0); 907 ireq->ireq_state = TCP_NEW_SYN_RECV; 908 write_pnet(&ireq->ireq_net, sock_net(sk_listener)); 909 ireq->ireq_family = sk_listener->sk_family; 910 } 911 912 return req; 913 } 914 915 void __reqsk_free(struct request_sock *req) 916 { 917 req->rsk_ops->destructor(req); 918 if (req->rsk_listener) 919 sock_put(req->rsk_listener); 920 kfree_rcu(unrcu_pointer(req->saved_syn), rcu); 921 kmem_cache_free(req->rsk_ops->slab, req); 922 } 923 924 static struct request_sock *inet_reqsk_clone(struct request_sock *req, 925 struct sock *sk) 926 { 927 struct sock *req_sk, *nreq_sk; 928 struct request_sock *nreq; 929 930 nreq = kmem_cache_alloc(req->rsk_ops->slab, GFP_ATOMIC | __GFP_NOWARN); 931 if (!nreq) { 932 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); 933 934 /* paired with refcount_inc_not_zero() in reuseport_migrate_sock() */ 935 sock_put(sk); 936 return NULL; 937 } 938 939 req_sk = req_to_sk(req); 940 nreq_sk = req_to_sk(nreq); 941 942 memcpy(nreq_sk, req_sk, 943 offsetof(struct sock, sk_dontcopy_begin)); 944 unsafe_memcpy(&nreq_sk->sk_dontcopy_end, &req_sk->sk_dontcopy_end, 945 req->rsk_ops->obj_size - offsetof(struct sock, sk_dontcopy_end), 946 /* alloc is larger than struct, see above */); 947 948 sk_node_init(&nreq_sk->sk_node); 949 nreq_sk->sk_tx_queue_mapping = req_sk->sk_tx_queue_mapping; 950 #ifdef CONFIG_SOCK_RX_QUEUE_MAPPING 951 nreq_sk->sk_rx_queue_mapping = req_sk->sk_rx_queue_mapping; 952 #endif 953 nreq_sk->sk_incoming_cpu = req_sk->sk_incoming_cpu; 954 955 nreq->rsk_listener = sk; 956 957 if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) { 958 struct fastopen_queue *fastopenq; 959 960 /* reqsk_fastopen_remove() will uncharge nreq->rsk_listener, 961 * that is @sk, so charge it here. Unlike the listener 962 * being closed, @sk is live and needs its lock. 963 */ 964 fastopenq = &inet_csk(sk)->icsk_accept_queue.fastopenq; 965 spin_lock_bh(&fastopenq->lock); 966 fastopenq->qlen++; 967 spin_unlock_bh(&fastopenq->lock); 968 969 /* We need not acquire fastopenq->lock 970 * because the child socket is locked in inet_csk_listen_stop(). 971 */ 972 rcu_assign_pointer(tcp_sk(nreq->sk)->fastopen_rsk, nreq); 973 } 974 975 if (rsk_is_mptcp(req)) 976 mptcp_subflow_reqsk_clone(req, nreq); 977 978 return nreq; 979 } 980 981 static void reqsk_queue_migrated(struct request_sock_queue *queue, 982 const struct request_sock *req) 983 { 984 if (req->num_timeout == 0) 985 atomic_inc(&queue->young); 986 atomic_inc(&queue->qlen); 987 } 988 989 static void reqsk_migrate_reset(struct request_sock *req) 990 { 991 RCU_INIT_POINTER(req->saved_syn, NULL); 992 #if IS_ENABLED(CONFIG_IPV6) 993 inet_rsk(req)->ipv6_opt = NULL; 994 inet_rsk(req)->pktopts = NULL; 995 #else 996 inet_rsk(req)->ireq_opt = NULL; 997 #endif 998 } 999 1000 /* return true if req was found in the ehash table */ 1001 static bool reqsk_queue_unlink(struct request_sock *req) 1002 { 1003 struct sock *sk = req_to_sk(req); 1004 bool found = false; 1005 1006 if (sk_hashed(sk)) { 1007 struct inet_hashinfo *hashinfo = tcp_get_hashinfo(sk); 1008 spinlock_t *lock; 1009 1010 lock = inet_ehash_lockp(hashinfo, req->rsk_hash); 1011 spin_lock(lock); 1012 found = __sk_nulls_del_node_init_rcu(sk); 1013 spin_unlock(lock); 1014 } 1015 1016 return found; 1017 } 1018 1019 static bool __inet_csk_reqsk_queue_drop(struct sock *sk, 1020 struct request_sock *req, 1021 bool from_timer) 1022 { 1023 bool unlinked = reqsk_queue_unlink(req); 1024 1025 if (!from_timer && timer_delete_sync(&req->rsk_timer)) 1026 reqsk_put(req); 1027 1028 if (unlinked) { 1029 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req); 1030 reqsk_put(req); 1031 } 1032 1033 return unlinked; 1034 } 1035 1036 bool inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req) 1037 { 1038 return __inet_csk_reqsk_queue_drop(sk, req, false); 1039 } 1040 1041 void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req) 1042 { 1043 inet_csk_reqsk_queue_drop(sk, req); 1044 reqsk_put(req); 1045 } 1046 1047 static void reqsk_timer_handler(struct timer_list *t) 1048 { 1049 struct request_sock *req = timer_container_of(req, t, rsk_timer); 1050 struct request_sock *nreq = NULL, *oreq = req; 1051 struct sock *sk_listener = req->rsk_listener; 1052 struct inet_connection_sock *icsk; 1053 struct request_sock_queue *queue; 1054 struct net *net; 1055 int max_syn_ack_retries, qlen, expire = 0, resend = 0; 1056 1057 if (inet_sk_state_load(sk_listener) != TCP_LISTEN) { 1058 struct sock *nsk; 1059 1060 nsk = reuseport_migrate_sock(sk_listener, req_to_sk(req), NULL); 1061 if (!nsk) 1062 goto drop; 1063 1064 nreq = inet_reqsk_clone(req, nsk); 1065 if (!nreq) 1066 goto drop; 1067 1068 /* The new timer for the cloned req can decrease the 2 1069 * by calling inet_csk_reqsk_queue_drop_and_put(), so 1070 * hold another count to prevent use-after-free and 1071 * call reqsk_put() just before return. 1072 */ 1073 timer_setup(&nreq->rsk_timer, reqsk_timer_handler, TIMER_PINNED); 1074 refcount_set_release(&nreq->rsk_refcnt, 2 + 1); 1075 reqsk_queue_migrated(&inet_csk(nsk)->icsk_accept_queue, req); 1076 1077 req = nreq; 1078 sk_listener = nsk; 1079 } 1080 1081 icsk = inet_csk(sk_listener); 1082 net = sock_net(sk_listener); 1083 max_syn_ack_retries = READ_ONCE(icsk->icsk_syn_retries) ? : 1084 READ_ONCE(net->ipv4.sysctl_tcp_synack_retries); 1085 /* Normally all the openreqs are young and become mature 1086 * (i.e. converted to established socket) for first timeout. 1087 * If synack was not acknowledged for 1 second, it means 1088 * one of the following things: synack was lost, ack was lost, 1089 * rtt is high or nobody planned to ack (i.e. synflood). 1090 * When server is a bit loaded, queue is populated with old 1091 * open requests, reducing effective size of queue. 1092 * When server is well loaded, queue size reduces to zero 1093 * after several minutes of work. It is not synflood, 1094 * it is normal operation. The solution is pruning 1095 * too old entries overriding normal timeout, when 1096 * situation becomes dangerous. 1097 * 1098 * Essentially, we reserve half of room for young 1099 * embrions; and abort old ones without pity, if old 1100 * ones are about to clog our table. 1101 */ 1102 queue = &icsk->icsk_accept_queue; 1103 qlen = reqsk_queue_len(queue); 1104 if ((qlen << 1) > max(8U, READ_ONCE(sk_listener->sk_max_ack_backlog))) { 1105 int young = reqsk_queue_len_young(queue) << 1; 1106 1107 while (max_syn_ack_retries > 2) { 1108 if (qlen < young) 1109 break; 1110 max_syn_ack_retries--; 1111 young <<= 1; 1112 } 1113 } 1114 1115 syn_ack_recalc(req, max_syn_ack_retries, READ_ONCE(queue->rskq_defer_accept), 1116 &expire, &resend); 1117 tcp_syn_ack_timeout(req); 1118 1119 if (!expire && 1120 (!resend || 1121 !tcp_rtx_synack(sk_listener, req) || 1122 inet_rsk(req)->acked)) { 1123 if (req->num_retrans > 1 && tcp_rsk(req)->accecn_ok) 1124 tcp_rsk(req)->accecn_fail_mode |= TCP_ACCECN_ACE_FAIL_SEND; 1125 if (req->num_timeout++ == 0) 1126 atomic_dec(&queue->young); 1127 mod_timer(&req->rsk_timer, jiffies + tcp_reqsk_timeout(req)); 1128 1129 if (!nreq) 1130 return; 1131 1132 if (!inet_ehash_insert(req_to_sk(nreq), req_to_sk(oreq), NULL)) { 1133 /* delete timer */ 1134 __inet_csk_reqsk_queue_drop(sk_listener, nreq, false); 1135 goto no_ownership; 1136 } 1137 1138 __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQSUCCESS); 1139 reqsk_migrate_reset(oreq); 1140 reqsk_queue_removed(&inet_csk(oreq->rsk_listener)->icsk_accept_queue, oreq); 1141 reqsk_put(oreq); 1142 1143 reqsk_put(nreq); 1144 return; 1145 } 1146 1147 /* Even if we can clone the req, we may need not retransmit any more 1148 * SYN+ACKs (nreq->num_timeout > max_syn_ack_retries, etc), or another 1149 * CPU may win the "own_req" race so that inet_ehash_insert() fails. 1150 */ 1151 if (nreq) { 1152 __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQFAILURE); 1153 /* The timer was not armed, drop its reference here. */ 1154 reqsk_put(nreq); 1155 no_ownership: 1156 reqsk_migrate_reset(nreq); 1157 reqsk_queue_removed(queue, nreq); 1158 /* Drop the temporary reference and the one reserved for ehash. */ 1159 reqsk_put(nreq); 1160 reqsk_put(nreq); 1161 } 1162 1163 drop: 1164 __inet_csk_reqsk_queue_drop(oreq->rsk_listener, oreq, true); 1165 reqsk_put(oreq); 1166 } 1167 1168 static bool reqsk_queue_hash_req(struct request_sock *req) 1169 { 1170 bool found_dup_sk = false; 1171 1172 if (!inet_ehash_insert(req_to_sk(req), NULL, &found_dup_sk)) 1173 return false; 1174 1175 /* The timer needs to be setup after a successful insertion. */ 1176 req->timeout = tcp_timeout_init((struct sock *)req); 1177 timer_setup(&req->rsk_timer, reqsk_timer_handler, TIMER_PINNED); 1178 1179 preempt_disable_nested(); 1180 1181 mod_timer(&req->rsk_timer, jiffies + req->timeout); 1182 1183 /* before letting lookups find us, make sure all req fields 1184 * are committed to memory and refcnt initialized. 1185 */ 1186 smp_wmb(); 1187 refcount_set(&req->rsk_refcnt, 2 + 1); 1188 1189 preempt_enable_nested(); 1190 1191 return true; 1192 } 1193 1194 bool inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req) 1195 { 1196 if (!reqsk_queue_hash_req(req)) 1197 return false; 1198 1199 inet_csk_reqsk_queue_added(sk); 1200 return true; 1201 } 1202 1203 static void inet_clone_ulp(const struct request_sock *req, struct sock *newsk, 1204 const gfp_t priority) 1205 { 1206 struct inet_connection_sock *icsk = inet_csk(newsk); 1207 1208 if (!icsk->icsk_ulp_ops) 1209 return; 1210 1211 icsk->icsk_ulp_ops->clone(req, newsk, priority); 1212 } 1213 1214 /** 1215 * inet_csk_clone_lock - clone an inet socket, and lock its clone 1216 * @sk: the socket to clone 1217 * @req: request_sock 1218 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc) 1219 * 1220 * Caller must unlock socket even in error path (bh_unlock_sock(newsk)) 1221 */ 1222 struct sock *inet_csk_clone_lock(const struct sock *sk, 1223 const struct request_sock *req, 1224 const gfp_t priority) 1225 { 1226 struct sock *newsk = sk_clone_lock(sk, priority); 1227 struct inet_connection_sock *newicsk; 1228 const struct inet_request_sock *ireq; 1229 struct inet_sock *newinet; 1230 1231 if (!newsk) 1232 return NULL; 1233 1234 newicsk = inet_csk(newsk); 1235 newinet = inet_sk(newsk); 1236 ireq = inet_rsk(req); 1237 1238 newicsk->icsk_bind_hash = NULL; 1239 newicsk->icsk_bind2_hash = NULL; 1240 1241 newinet->inet_dport = ireq->ir_rmt_port; 1242 newinet->inet_num = ireq->ir_num; 1243 newinet->inet_sport = htons(ireq->ir_num); 1244 1245 newsk->sk_bound_dev_if = ireq->ir_iif; 1246 1247 newsk->sk_daddr = ireq->ir_rmt_addr; 1248 newsk->sk_rcv_saddr = ireq->ir_loc_addr; 1249 newinet->inet_saddr = ireq->ir_loc_addr; 1250 1251 #if IS_ENABLED(CONFIG_IPV6) 1252 newsk->sk_v6_daddr = ireq->ir_v6_rmt_addr; 1253 newsk->sk_v6_rcv_saddr = ireq->ir_v6_loc_addr; 1254 #endif 1255 1256 /* listeners have SOCK_RCU_FREE, not the children */ 1257 sock_reset_flag(newsk, SOCK_RCU_FREE); 1258 1259 inet_sk(newsk)->mc_list = NULL; 1260 1261 newsk->sk_mark = inet_rsk(req)->ir_mark; 1262 atomic64_set(&newsk->sk_cookie, 1263 atomic64_read(&inet_rsk(req)->ir_cookie)); 1264 1265 newicsk->icsk_retransmits = 0; 1266 newicsk->icsk_backoff = 0; 1267 newicsk->icsk_probes_out = 0; 1268 newicsk->icsk_probes_tstamp = 0; 1269 1270 /* Deinitialize accept_queue to trap illegal accesses. */ 1271 memset(&newicsk->icsk_accept_queue, 0, 1272 sizeof(newicsk->icsk_accept_queue)); 1273 1274 newicsk->unhashed_state = 0; 1275 inet_sk_set_state(newsk, TCP_SYN_RECV); 1276 1277 inet_clone_ulp(req, newsk, priority); 1278 1279 security_inet_csk_clone(newsk, req); 1280 1281 return newsk; 1282 } 1283 1284 /* 1285 * At this point, there should be no process reference to this 1286 * socket, and thus no user references at all. Therefore we 1287 * can assume the socket waitqueue is inactive and nobody will 1288 * try to jump onto it. 1289 */ 1290 void inet_csk_destroy_sock(struct sock *sk) 1291 { 1292 WARN_ON(sk->sk_state != TCP_CLOSE); 1293 WARN_ON(!sock_flag(sk, SOCK_DEAD)); 1294 1295 /* It cannot be in hash table! */ 1296 WARN_ON(!sk_unhashed(sk)); 1297 1298 /* If it has not 0 inet_sk(sk)->inet_num, it must be bound */ 1299 WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash); 1300 1301 sk->sk_prot->destroy(sk); 1302 1303 sk_stream_kill_queues(sk); 1304 1305 xfrm_sk_free_policy(sk); 1306 1307 tcp_orphan_count_dec(); 1308 1309 sock_put(sk); 1310 } 1311 1312 void inet_csk_prepare_for_destroy_sock(struct sock *sk) 1313 { 1314 /* The below has to be done to allow calling inet_csk_destroy_sock */ 1315 tcp_clear_sock_ops_cb_flags(sk); 1316 sock_set_flag(sk, SOCK_DEAD); 1317 tcp_orphan_count_inc(); 1318 } 1319 1320 /* This function allows to force a closure of a socket after the call to 1321 * tcp_create_openreq_child(). 1322 */ 1323 void inet_csk_prepare_forced_close(struct sock *sk) 1324 __releases(&sk->sk_lock.slock) 1325 { 1326 /* sk_clone_lock locked the socket and set refcnt to 2 */ 1327 bh_unlock_sock(sk); 1328 sock_put(sk); 1329 inet_csk_prepare_for_destroy_sock(sk); 1330 inet_sk(sk)->inet_num = 0; 1331 } 1332 1333 static int inet_ulp_can_listen(const struct sock *sk) 1334 { 1335 const struct inet_connection_sock *icsk = inet_csk(sk); 1336 1337 if (icsk->icsk_ulp_ops && !icsk->icsk_ulp_ops->clone) 1338 return -EINVAL; 1339 1340 return 0; 1341 } 1342 1343 /* Free all the reqs queued in rskq_rst_head. */ 1344 static void reqsk_fastopen_purge_rst(struct request_sock_queue *queue) 1345 { 1346 struct request_sock *next, *req; 1347 1348 spin_lock_bh(&queue->fastopenq.lock); 1349 req = queue->fastopenq.rskq_rst_head; 1350 queue->fastopenq.rskq_rst_head = NULL; 1351 spin_unlock_bh(&queue->fastopenq.lock); 1352 while (req != NULL) { 1353 next = req->dl_next; 1354 reqsk_put(req); 1355 req = next; 1356 } 1357 } 1358 1359 static void reqsk_queue_alloc(struct request_sock_queue *queue) 1360 { 1361 queue->fastopenq.rskq_rst_head = NULL; 1362 queue->fastopenq.rskq_rst_tail = NULL; 1363 queue->fastopenq.qlen = 0; 1364 1365 queue->rskq_accept_head = NULL; 1366 } 1367 1368 int inet_csk_listen_start(struct sock *sk) 1369 { 1370 struct inet_connection_sock *icsk = inet_csk(sk); 1371 struct inet_sock *inet = inet_sk(sk); 1372 int err; 1373 1374 err = inet_ulp_can_listen(sk); 1375 if (unlikely(err)) 1376 return err; 1377 1378 reqsk_queue_alloc(&icsk->icsk_accept_queue); 1379 1380 sk->sk_ack_backlog = 0; 1381 inet_csk_delack_init(sk); 1382 1383 /* There is race window here: we announce ourselves listening, 1384 * but this transition is still not validated by get_port(). 1385 * It is OK, because this socket enters to hash table only 1386 * after validation is complete. 1387 */ 1388 inet_sk_state_store(sk, TCP_LISTEN); 1389 err = sk->sk_prot->get_port(sk, inet->inet_num); 1390 if (!err) { 1391 inet->inet_sport = htons(inet->inet_num); 1392 1393 sk_dst_reset(sk); 1394 err = sk->sk_prot->hash(sk); 1395 1396 if (likely(!err)) 1397 return 0; 1398 } 1399 1400 inet_sk_set_state(sk, TCP_CLOSE); 1401 1402 /* A Fast Open child accepted before a previous shutdown() of this 1403 * listener might have queued its request in rskq_rst_head while 1404 * we were transiently in TCP_LISTEN state. 1405 */ 1406 reqsk_fastopen_purge_rst(&icsk->icsk_accept_queue); 1407 return err; 1408 } 1409 1410 static void inet_child_forget(struct sock *sk, struct request_sock *req, 1411 struct sock *child) 1412 { 1413 sk->sk_prot->disconnect(child, O_NONBLOCK); 1414 1415 sock_orphan(child); 1416 1417 tcp_orphan_count_inc(); 1418 1419 if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) { 1420 BUG_ON(rcu_access_pointer(tcp_sk(child)->fastopen_rsk) != req); 1421 BUG_ON(sk != req->rsk_listener); 1422 1423 /* Paranoid, to prevent race condition if 1424 * an inbound pkt destined for child is 1425 * blocked by sock lock in tcp_v4_rcv(). 1426 * Also to satisfy an assertion in 1427 * tcp_v4_destroy_sock(). 1428 */ 1429 RCU_INIT_POINTER(tcp_sk(child)->fastopen_rsk, NULL); 1430 } 1431 inet_csk_destroy_sock(child); 1432 } 1433 1434 struct sock *inet_csk_reqsk_queue_add(struct sock *sk, 1435 struct request_sock *req, 1436 struct sock *child) 1437 { 1438 struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue; 1439 1440 spin_lock(&queue->rskq_lock); 1441 if (unlikely(sk->sk_state != TCP_LISTEN)) { 1442 inet_child_forget(sk, req, child); 1443 child = NULL; 1444 } else { 1445 req->sk = child; 1446 req->dl_next = NULL; 1447 if (queue->rskq_accept_head == NULL) 1448 WRITE_ONCE(queue->rskq_accept_head, req); 1449 else 1450 queue->rskq_accept_tail->dl_next = req; 1451 queue->rskq_accept_tail = req; 1452 sk_acceptq_added(sk); 1453 } 1454 spin_unlock(&queue->rskq_lock); 1455 return child; 1456 } 1457 1458 struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child, 1459 struct request_sock *req, bool own_req) 1460 { 1461 if (own_req) { 1462 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 1463 reqsk_queue_removed(&inet_csk(req->rsk_listener)->icsk_accept_queue, req); 1464 1465 if (sk != req->rsk_listener) { 1466 /* another listening sk has been selected, 1467 * migrate the req to it. 1468 */ 1469 struct request_sock *nreq; 1470 1471 /* hold a refcnt for the nreq->rsk_listener 1472 * which is assigned in inet_reqsk_clone() 1473 */ 1474 sock_hold(sk); 1475 nreq = inet_reqsk_clone(req, sk); 1476 if (!nreq) { 1477 inet_child_forget(sk, req, child); 1478 goto child_put; 1479 } 1480 1481 refcount_set_release(&nreq->rsk_refcnt, 1); 1482 if (inet_csk_reqsk_queue_add(sk, nreq, child)) { 1483 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQSUCCESS); 1484 reqsk_migrate_reset(req); 1485 reqsk_put(req); 1486 return child; 1487 } 1488 1489 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); 1490 reqsk_migrate_reset(nreq); 1491 reqsk_put(nreq); 1492 } else if (inet_csk_reqsk_queue_add(sk, req, child)) { 1493 return child; 1494 } 1495 } 1496 /* Too bad, another child took ownership of the request, undo. */ 1497 child_put: 1498 bh_unlock_sock(child); 1499 sock_put(child); 1500 return NULL; 1501 } 1502 1503 /* 1504 * This routine closes sockets which have been at least partially 1505 * opened, but not yet accepted. 1506 */ 1507 void inet_csk_listen_stop(struct sock *sk) 1508 { 1509 struct inet_connection_sock *icsk = inet_csk(sk); 1510 struct request_sock_queue *queue = &icsk->icsk_accept_queue; 1511 struct request_sock *req; 1512 1513 /* Following specs, it would be better either to send FIN 1514 * (and enter FIN-WAIT-1, it is normal close) 1515 * or to send active reset (abort). 1516 * Certainly, it is pretty dangerous while synflood, but it is 1517 * bad justification for our negligence 8) 1518 * To be honest, we are not able to make either 1519 * of the variants now. --ANK 1520 */ 1521 while ((req = reqsk_queue_remove(queue, sk)) != NULL) { 1522 struct sock *child = req->sk, *nsk; 1523 struct request_sock *nreq; 1524 1525 local_bh_disable(); 1526 bh_lock_sock(child); 1527 WARN_ON(sock_owned_by_user(child)); 1528 sock_hold(child); 1529 1530 nsk = reuseport_migrate_sock(sk, child, NULL); 1531 if (nsk) { 1532 nreq = inet_reqsk_clone(req, nsk); 1533 if (nreq) { 1534 refcount_set_release(&nreq->rsk_refcnt, 1); 1535 1536 rcu_read_lock(); 1537 if (inet_csk_reqsk_queue_add(nsk, nreq, child)) { 1538 __NET_INC_STATS(sock_net(nsk), 1539 LINUX_MIB_TCPMIGRATEREQSUCCESS); 1540 reqsk_migrate_reset(req); 1541 READ_ONCE(nsk->sk_data_ready)(nsk); 1542 } else { 1543 __NET_INC_STATS(sock_net(nsk), 1544 LINUX_MIB_TCPMIGRATEREQFAILURE); 1545 reqsk_migrate_reset(nreq); 1546 reqsk_put(nreq); 1547 } 1548 rcu_read_unlock(); 1549 1550 /* inet_csk_reqsk_queue_add() has already 1551 * called inet_child_forget() on failure case. 1552 */ 1553 goto skip_child_forget; 1554 } 1555 } 1556 1557 inet_child_forget(sk, req, child); 1558 skip_child_forget: 1559 reqsk_put(req); 1560 bh_unlock_sock(child); 1561 local_bh_enable(); 1562 sock_put(child); 1563 1564 if (!has_current_bpf_ctx()) 1565 cond_resched(); 1566 } 1567 reqsk_fastopen_purge_rst(queue); 1568 WARN_ON_ONCE(sk->sk_ack_backlog); 1569 } 1570 1571 static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl) 1572 { 1573 const struct inet_sock *inet = inet_sk(sk); 1574 struct flowi4 *fl4; 1575 struct rtable *rt; 1576 1577 rcu_read_lock(); 1578 fl4 = &fl->u.ip4; 1579 inet_sk_init_flowi4(inet, fl4); 1580 rt = ip_route_output_flow(sock_net(sk), fl4, sk); 1581 if (IS_ERR(rt)) 1582 rt = NULL; 1583 if (rt) 1584 sk_setup_caps(sk, &rt->dst); 1585 rcu_read_unlock(); 1586 1587 return &rt->dst; 1588 } 1589 1590 struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu) 1591 { 1592 struct dst_entry *dst = __sk_dst_check(sk, 0); 1593 struct inet_sock *inet = inet_sk(sk); 1594 1595 if (!dst) { 1596 dst = inet_csk_rebuild_route(sk, &inet->cork.fl); 1597 if (!dst) 1598 goto out; 1599 } 1600 dst->ops->update_pmtu(dst, sk, NULL, mtu, true); 1601 1602 dst = __sk_dst_check(sk, 0); 1603 if (!dst) 1604 dst = inet_csk_rebuild_route(sk, &inet->cork.fl); 1605 out: 1606 return dst; 1607 } 1608