1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Implementation of the Transmission Control Protocol(TCP). 8 * 9 * IPv4 specific functions 10 * 11 * code split from: 12 * linux/ipv4/tcp.c 13 * linux/ipv4/tcp_input.c 14 * linux/ipv4/tcp_output.c 15 * 16 * See tcp.c for author information 17 */ 18 19 /* 20 * Changes: 21 * David S. Miller : New socket lookup architecture. 22 * This code is dedicated to John Dyson. 23 * David S. Miller : Change semantics of established hash, 24 * half is devoted to TIME_WAIT sockets 25 * and the rest go in the other half. 26 * Andi Kleen : Add support for syncookies and fixed 27 * some bugs: ip options weren't passed to 28 * the TCP layer, missed a check for an 29 * ACK bit. 30 * Andi Kleen : Implemented fast path mtu discovery. 31 * Fixed many serious bugs in the 32 * request_sock handling and moved 33 * most of it into the af independent code. 34 * Added tail drop and some other bugfixes. 35 * Added new listen semantics. 36 * Mike McLagan : Routing by source 37 * Juan Jose Ciarlante: ip_dynaddr bits 38 * Andi Kleen: various fixes. 39 * Vitaly E. Lavrov : Transparent proxy revived after year 40 * coma. 41 * Andi Kleen : Fix new listen. 42 * Andi Kleen : Fix accept error reporting. 43 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which 44 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind 45 * a single port at the same time. 46 */ 47 48 #define pr_fmt(fmt) "TCP: " fmt 49 50 #include <linux/bottom_half.h> 51 #include <linux/types.h> 52 #include <linux/fcntl.h> 53 #include <linux/module.h> 54 #include <linux/random.h> 55 #include <linux/cache.h> 56 #include <linux/fips.h> 57 #include <linux/jhash.h> 58 #include <linux/init.h> 59 #include <linux/times.h> 60 #include <linux/slab.h> 61 #include <linux/sched.h> 62 #include <linux/sock_diag.h> 63 64 #include <net/aligned_data.h> 65 #include <net/net_namespace.h> 66 #include <net/icmp.h> 67 #include <net/inet_hashtables.h> 68 #include <net/tcp.h> 69 #include <net/tcp_ecn.h> 70 #include <net/transp_v6.h> 71 #include <net/ipv6.h> 72 #include <net/inet_common.h> 73 #include <net/inet_ecn.h> 74 #include <net/timewait_sock.h> 75 #include <net/xfrm.h> 76 #include <net/secure_seq.h> 77 #include <net/busy_poll.h> 78 #include <net/rstreason.h> 79 #include <net/psp.h> 80 81 #include <linux/inet.h> 82 #include <linux/ipv6.h> 83 #include <linux/stddef.h> 84 #include <linux/proc_fs.h> 85 #include <linux/seq_file.h> 86 #include <linux/inetdevice.h> 87 #include <linux/btf_ids.h> 88 #include <linux/skbuff_ref.h> 89 90 #include <crypto/md5.h> 91 #include <crypto/utils.h> 92 93 #include <trace/events/tcp.h> 94 95 #ifdef CONFIG_TCP_MD5SIG 96 static void tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, 97 __be32 daddr, __be32 saddr, const struct tcphdr *th); 98 #endif 99 100 struct inet_hashinfo tcp_hashinfo; 101 102 static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = { 103 .bh_lock = INIT_LOCAL_LOCK(bh_lock), 104 }; 105 106 static DEFINE_MUTEX(tcp_exit_batch_mutex); 107 108 INDIRECT_CALLABLE_SCOPE union tcp_seq_and_ts_off 109 tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb) 110 { 111 return secure_tcp_seq_and_ts_off(net, 112 ip_hdr(skb)->daddr, 113 ip_hdr(skb)->saddr, 114 tcp_hdr(skb)->dest, 115 tcp_hdr(skb)->source); 116 } 117 118 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp) 119 { 120 int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse); 121 const struct inet_timewait_sock *tw = inet_twsk(sktw); 122 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw); 123 struct tcp_sock *tp = tcp_sk(sk); 124 int ts_recent_stamp; 125 u32 reuse_thresh; 126 127 if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2) 128 reuse = 0; 129 130 if (reuse == 2) { 131 /* Still does not detect *everything* that goes through 132 * lo, since we require a loopback src or dst address 133 * or direct binding to 'lo' interface. 134 */ 135 bool loopback = false; 136 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX) 137 loopback = true; 138 #if IS_ENABLED(CONFIG_IPV6) 139 if (tw->tw_family == AF_INET6) { 140 if (ipv6_addr_loopback(&tw->tw_v6_daddr) || 141 ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) || 142 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) || 143 ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr)) 144 loopback = true; 145 } else 146 #endif 147 { 148 if (ipv4_is_loopback(tw->tw_daddr) || 149 ipv4_is_loopback(tw->tw_rcv_saddr)) 150 loopback = true; 151 } 152 if (!loopback) 153 reuse = 0; 154 } 155 156 /* With PAWS, it is safe from the viewpoint 157 of data integrity. Even without PAWS it is safe provided sequence 158 spaces do not overlap i.e. at data rates <= 80Mbit/sec. 159 160 Actually, the idea is close to VJ's one, only timestamp cache is 161 held not per host, but per port pair and TW bucket is used as state 162 holder. 163 164 If TW bucket has been already destroyed we fall back to VJ's scheme 165 and use initial timestamp retrieved from peer table. 166 */ 167 ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp); 168 reuse_thresh = READ_ONCE(tw->tw_entry_stamp) + 169 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay); 170 if (ts_recent_stamp && 171 (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) { 172 /* inet_twsk_hashdance_schedule() sets sk_refcnt after putting twsk 173 * and releasing the bucket lock. 174 */ 175 if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt))) 176 return 0; 177 178 /* In case of repair and re-using TIME-WAIT sockets we still 179 * want to be sure that it is safe as above but honor the 180 * sequence numbers and time stamps set as part of the repair 181 * process. 182 * 183 * Without this check re-using a TIME-WAIT socket with TCP 184 * repair would accumulate a -1 on the repair assigned 185 * sequence number. The first time it is reused the sequence 186 * is -1, the second time -2, etc. This fixes that issue 187 * without appearing to create any others. 188 */ 189 if (likely(!tp->repair)) { 190 u32 seq = tcptw->tw_snd_nxt + 65535 + 2; 191 192 if (!seq) 193 seq = 1; 194 WRITE_ONCE(tp->write_seq, seq); 195 tp->rx_opt.ts_recent = READ_ONCE(tcptw->tw_ts_recent); 196 tp->rx_opt.ts_recent_stamp = ts_recent_stamp; 197 } 198 199 return 1; 200 } 201 202 return 0; 203 } 204 205 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr, 206 int addr_len) 207 { 208 /* This check is replicated from tcp_v4_connect() and intended to 209 * prevent BPF program called below from accessing bytes that are out 210 * of the bound specified by user in addr_len. 211 */ 212 if (addr_len < sizeof(struct sockaddr_in)) 213 return -EINVAL; 214 215 sock_owned_by_me(sk); 216 217 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr, &addr_len); 218 } 219 220 /* This will initiate an outgoing connection. */ 221 int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len) 222 { 223 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr; 224 struct inet_timewait_death_row *tcp_death_row; 225 struct inet_sock *inet = inet_sk(sk); 226 struct tcp_sock *tp = tcp_sk(sk); 227 struct ip_options_rcu *inet_opt; 228 struct net *net = sock_net(sk); 229 __be16 orig_sport, orig_dport; 230 __be32 daddr, nexthop; 231 struct flowi4 *fl4; 232 struct rtable *rt; 233 int err; 234 235 if (addr_len < sizeof(struct sockaddr_in)) 236 return -EINVAL; 237 238 if (usin->sin_family != AF_INET) 239 return -EAFNOSUPPORT; 240 241 if (psp_sk_assoc(sk)) 242 return -EINVAL; 243 244 nexthop = daddr = usin->sin_addr.s_addr; 245 inet_opt = rcu_dereference_protected(inet->inet_opt, 246 lockdep_sock_is_held(sk)); 247 if (inet_opt && inet_opt->opt.srr) { 248 if (!daddr) 249 return -EINVAL; 250 nexthop = inet_opt->opt.faddr; 251 } 252 253 orig_sport = inet->inet_sport; 254 orig_dport = usin->sin_port; 255 fl4 = &inet->cork.fl.u.ip4; 256 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr, 257 sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport, 258 orig_dport, sk); 259 if (IS_ERR(rt)) { 260 err = PTR_ERR(rt); 261 if (err == -ENETUNREACH) 262 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); 263 return err; 264 } 265 266 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { 267 ip_rt_put(rt); 268 return -ENETUNREACH; 269 } 270 271 if (!inet_opt || !inet_opt->opt.srr) 272 daddr = fl4->daddr; 273 274 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; 275 276 if (!inet->inet_saddr) { 277 err = inet_bhash2_update_saddr(sk, &fl4->saddr, AF_INET); 278 if (err) { 279 ip_rt_put(rt); 280 return err; 281 } 282 } else { 283 sk_rcv_saddr_set(sk, inet->inet_saddr); 284 } 285 286 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) { 287 /* Reset inherited state */ 288 tp->rx_opt.ts_recent = 0; 289 tp->rx_opt.ts_recent_stamp = 0; 290 if (likely(!tp->repair)) 291 WRITE_ONCE(tp->write_seq, 0); 292 } 293 294 inet->inet_dport = usin->sin_port; 295 sk_daddr_set(sk, daddr); 296 297 inet_csk(sk)->icsk_ext_hdr_len = 0; 298 if (inet_opt) 299 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen; 300 301 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT; 302 303 /* Socket identity is still unknown (sport may be zero). 304 * However we set state to SYN-SENT and not releasing socket 305 * lock select source port, enter ourselves into the hash tables and 306 * complete initialization after this. 307 */ 308 tcp_set_state(sk, TCP_SYN_SENT); 309 err = inet_hash_connect(tcp_death_row, sk); 310 if (err) 311 goto failure; 312 313 sk_set_txhash(sk); 314 315 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport, 316 inet->inet_sport, inet->inet_dport, sk); 317 if (IS_ERR(rt)) { 318 err = PTR_ERR(rt); 319 rt = NULL; 320 goto failure; 321 } 322 tp->tcp_usec_ts = dst_tcp_usec_ts(&rt->dst); 323 /* OK, now commit destination to socket. */ 324 sk->sk_gso_type = SKB_GSO_TCPV4; 325 sk_setup_caps(sk, &rt->dst); 326 rt = NULL; 327 328 if (likely(!tp->repair)) { 329 union tcp_seq_and_ts_off st; 330 331 st = secure_tcp_seq_and_ts_off(net, 332 inet->inet_saddr, 333 inet->inet_daddr, 334 inet->inet_sport, 335 usin->sin_port); 336 if (!tp->write_seq) 337 WRITE_ONCE(tp->write_seq, st.seq); 338 WRITE_ONCE(tp->tsoffset, st.ts_off); 339 } 340 341 atomic_set(&inet->inet_id, get_random_u16()); 342 343 if (tcp_fastopen_defer_connect(sk, &err)) 344 return err; 345 if (err) 346 goto failure; 347 348 err = tcp_connect(sk); 349 350 if (err) 351 goto failure; 352 353 return 0; 354 355 failure: 356 /* 357 * This unhashes the socket and releases the local port, 358 * if necessary. 359 */ 360 tcp_set_state(sk, TCP_CLOSE); 361 inet_bhash2_reset_saddr(sk); 362 ip_rt_put(rt); 363 sk->sk_route_caps = 0; 364 inet->inet_dport = 0; 365 return err; 366 } 367 368 /* 369 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191. 370 * It can be called through tcp_release_cb() if socket was owned by user 371 * at the time tcp_v4_err() was called to handle ICMP message. 372 */ 373 void tcp_v4_mtu_reduced(struct sock *sk) 374 { 375 struct inet_sock *inet = inet_sk(sk); 376 struct dst_entry *dst; 377 u32 mtu, dmtu; 378 379 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) 380 return; 381 mtu = READ_ONCE(tcp_sk(sk)->mtu_info); 382 dst = inet_csk_update_pmtu(sk, mtu); 383 if (!dst) 384 return; 385 386 /* Something is about to be wrong... Remember soft error 387 * for the case, if this connection will not able to recover. 388 */ 389 dmtu = dst4_mtu(dst); 390 if (mtu < dmtu && ip_dont_fragment(sk, dst)) 391 WRITE_ONCE(sk->sk_err_soft, EMSGSIZE); 392 393 if (inet->pmtudisc != IP_PMTUDISC_DONT && 394 ip_sk_accept_pmtu(sk) && 395 inet_csk(sk)->icsk_pmtu_cookie > dmtu) { 396 tcp_sync_mss(sk, dmtu); 397 398 /* Resend the TCP packet because it's 399 * clear that the old packet has been 400 * dropped. This is the new "fast" path mtu 401 * discovery. 402 */ 403 tcp_simple_retransmit(sk); 404 } /* else let the usual retransmit timer handle it */ 405 } 406 407 static void do_redirect(struct sk_buff *skb, struct sock *sk) 408 { 409 struct dst_entry *dst = __sk_dst_check(sk, 0); 410 411 if (dst) 412 dst->ops->redirect(dst, sk, skb); 413 } 414 415 416 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */ 417 void tcp_req_err(struct sock *sk, u32 seq, bool abort) 418 { 419 struct request_sock *req = inet_reqsk(sk); 420 struct net *net = sock_net(sk); 421 422 /* ICMPs are not backlogged, hence we cannot get 423 * an established socket here. 424 */ 425 if (seq != tcp_rsk(req)->snt_isn) { 426 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS); 427 } else if (abort) { 428 /* 429 * Still in SYN_RECV, just remove it silently. 430 * There is no good way to pass the error to the newly 431 * created socket, and POSIX does not want network 432 * errors returned from accept(). 433 */ 434 inet_csk_reqsk_queue_drop(req->rsk_listener, req); 435 tcp_listendrop(req->rsk_listener); 436 } 437 reqsk_put(req); 438 } 439 440 /* TCP-LD (RFC 6069) logic */ 441 void tcp_ld_RTO_revert(struct sock *sk, u32 seq) 442 { 443 struct inet_connection_sock *icsk = inet_csk(sk); 444 struct tcp_sock *tp = tcp_sk(sk); 445 struct sk_buff *skb; 446 s32 remaining; 447 u32 delta_us; 448 449 if (sock_owned_by_user(sk)) 450 return; 451 452 if (seq != tp->snd_una || !icsk->icsk_retransmits || 453 !icsk->icsk_backoff) 454 return; 455 456 skb = tcp_rtx_queue_head(sk); 457 if (WARN_ON_ONCE(!skb)) 458 return; 459 460 icsk->icsk_backoff--; 461 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT; 462 icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk)); 463 464 tcp_mstamp_refresh(tp); 465 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb)); 466 remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us); 467 468 if (remaining > 0) { 469 tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false); 470 } else { 471 /* RTO revert clocked out retransmission. 472 * Will retransmit now. 473 */ 474 tcp_retransmit_timer(sk); 475 } 476 } 477 478 /* 479 * This routine is called by the ICMP module when it gets some 480 * sort of error condition. If err < 0 then the socket should 481 * be closed and the error returned to the user. If err > 0 482 * it's just the icmp type << 8 | icmp code. After adjustment 483 * header points to the first 8 bytes of the tcp header. We need 484 * to find the appropriate port. 485 * 486 * The locking strategy used here is very "optimistic". When 487 * someone else accesses the socket the ICMP is just dropped 488 * and for some paths there is no check at all. 489 * A more general error queue to queue errors for later handling 490 * is probably better. 491 * 492 */ 493 494 int tcp_v4_err(struct sk_buff *skb, u32 info) 495 { 496 const struct iphdr *iph = (const struct iphdr *)skb->data; 497 struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2)); 498 struct net *net = dev_net_rcu(skb->dev); 499 const int type = icmp_hdr(skb)->type; 500 const int code = icmp_hdr(skb)->code; 501 struct request_sock *fastopen; 502 struct tcp_sock *tp; 503 u32 seq, snd_una; 504 struct sock *sk; 505 int err; 506 507 sk = __inet_lookup_established(net, iph->daddr, th->dest, iph->saddr, 508 ntohs(th->source), inet_iif(skb), 0); 509 if (!sk) { 510 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS); 511 return -ENOENT; 512 } 513 if (sk->sk_state == TCP_TIME_WAIT) { 514 /* To increase the counter of ignored icmps for TCP-AO */ 515 tcp_ao_ignore_icmp(sk, AF_INET, type, code); 516 inet_twsk_put(inet_twsk(sk)); 517 return 0; 518 } 519 seq = ntohl(th->seq); 520 if (sk->sk_state == TCP_NEW_SYN_RECV) { 521 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB || 522 type == ICMP_TIME_EXCEEDED || 523 (type == ICMP_DEST_UNREACH && 524 (code == ICMP_NET_UNREACH || 525 code == ICMP_HOST_UNREACH))); 526 return 0; 527 } 528 529 if (tcp_ao_ignore_icmp(sk, AF_INET, type, code)) { 530 sock_put(sk); 531 return 0; 532 } 533 534 bh_lock_sock(sk); 535 /* If too many ICMPs get dropped on busy 536 * servers this needs to be solved differently. 537 * We do take care of PMTU discovery (RFC1191) special case : 538 * we can receive locally generated ICMP messages while socket is held. 539 */ 540 if (sock_owned_by_user(sk)) { 541 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)) 542 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS); 543 } 544 if (sk->sk_state == TCP_CLOSE || inet_test_bit(DEFER_CONNECT, sk)) 545 goto out; 546 547 if (static_branch_unlikely(&ip4_min_ttl)) { 548 /* min_ttl can be changed concurrently from do_ip_setsockopt() */ 549 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) { 550 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 551 goto out; 552 } 553 } 554 555 tp = tcp_sk(sk); 556 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */ 557 fastopen = rcu_dereference(tp->fastopen_rsk); 558 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una; 559 if (sk->sk_state != TCP_LISTEN && 560 !between(seq, snd_una, tp->snd_nxt)) { 561 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS); 562 goto out; 563 } 564 565 switch (type) { 566 case ICMP_REDIRECT: 567 if (!sock_owned_by_user(sk)) 568 do_redirect(skb, sk); 569 goto out; 570 case ICMP_SOURCE_QUENCH: 571 /* Just silently ignore these. */ 572 goto out; 573 case ICMP_PARAMETERPROB: 574 err = EPROTO; 575 break; 576 case ICMP_DEST_UNREACH: 577 if (code > NR_ICMP_UNREACH) 578 goto out; 579 580 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */ 581 /* We are not interested in TCP_LISTEN and open_requests 582 * (SYN-ACKs send out by Linux are always <576bytes so 583 * they should go through unfragmented). 584 */ 585 if (sk->sk_state == TCP_LISTEN) 586 goto out; 587 588 WRITE_ONCE(tp->mtu_info, info); 589 if (!sock_owned_by_user(sk)) { 590 tcp_v4_mtu_reduced(sk); 591 } else { 592 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags)) 593 sock_hold(sk); 594 } 595 goto out; 596 } 597 598 err = icmp_err_convert[code].errno; 599 /* check if this ICMP message allows revert of backoff. 600 * (see RFC 6069) 601 */ 602 if (!fastopen && 603 (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH)) 604 tcp_ld_RTO_revert(sk, seq); 605 break; 606 case ICMP_TIME_EXCEEDED: 607 err = EHOSTUNREACH; 608 break; 609 default: 610 goto out; 611 } 612 613 switch (sk->sk_state) { 614 case TCP_SYN_SENT: 615 case TCP_SYN_RECV: 616 /* Only in fast or simultaneous open. If a fast open socket is 617 * already accepted it is treated as a connected one below. 618 */ 619 if (fastopen && !fastopen->sk) 620 break; 621 622 ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th); 623 624 if (!sock_owned_by_user(sk)) 625 tcp_done_with_error(sk, err); 626 else 627 WRITE_ONCE(sk->sk_err_soft, err); 628 goto out; 629 } 630 631 /* If we've already connected we will keep trying 632 * until we time out, or the user gives up. 633 * 634 * rfc1122 4.2.3.9 allows to consider as hard errors 635 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too, 636 * but it is obsoleted by pmtu discovery). 637 * 638 * Note, that in modern internet, where routing is unreliable 639 * and in each dark corner broken firewalls sit, sending random 640 * errors ordered by their masters even this two messages finally lose 641 * their original sense (even Linux sends invalid PORT_UNREACHs) 642 * 643 * Now we are in compliance with RFCs. 644 * --ANK (980905) 645 */ 646 647 if (!sock_owned_by_user(sk) && 648 inet_test_bit(RECVERR, sk)) { 649 WRITE_ONCE(sk->sk_err, err); 650 sk_error_report(sk); 651 } else { /* Only an error on timeout */ 652 WRITE_ONCE(sk->sk_err_soft, err); 653 } 654 655 out: 656 bh_unlock_sock(sk); 657 sock_put(sk); 658 return 0; 659 } 660 661 #define REPLY_OPTIONS_LEN (MAX_TCP_OPTION_SPACE / sizeof(__be32)) 662 663 static bool tcp_v4_ao_sign_reset(const struct sock *sk, struct sk_buff *skb, 664 const struct tcp_ao_hdr *aoh, 665 struct ip_reply_arg *arg, struct tcphdr *reply, 666 __be32 reply_options[REPLY_OPTIONS_LEN]) 667 { 668 #ifdef CONFIG_TCP_AO 669 int sdif = tcp_v4_sdif(skb); 670 int dif = inet_iif(skb); 671 int l3index = sdif ? dif : 0; 672 bool allocated_traffic_key; 673 struct tcp_ao_key *key; 674 char *traffic_key; 675 bool drop = true; 676 u32 ao_sne = 0; 677 u8 keyid; 678 679 rcu_read_lock(); 680 if (tcp_ao_prepare_reset(sk, skb, aoh, l3index, ntohl(reply->seq), 681 &key, &traffic_key, &allocated_traffic_key, 682 &keyid, &ao_sne)) 683 goto out; 684 685 reply_options[0] = htonl((TCPOPT_AO << 24) | (tcp_ao_len(key) << 16) | 686 (aoh->rnext_keyid << 8) | keyid); 687 arg->iov[0].iov_len += tcp_ao_len_aligned(key); 688 reply->doff = arg->iov[0].iov_len / 4; 689 690 if (tcp_ao_hash_hdr(AF_INET, (char *)&reply_options[1], 691 key, traffic_key, 692 (union tcp_ao_addr *)&ip_hdr(skb)->saddr, 693 (union tcp_ao_addr *)&ip_hdr(skb)->daddr, 694 reply, ao_sne)) 695 goto out; 696 drop = false; 697 out: 698 rcu_read_unlock(); 699 if (allocated_traffic_key) 700 kfree(traffic_key); 701 return drop; 702 #else 703 return true; 704 #endif 705 } 706 707 /* 708 * This routine will send an RST to the other tcp. 709 * 710 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.) 711 * for reset. 712 * Answer: if a packet caused RST, it is not for a socket 713 * existing in our system, if it is matched to a socket, 714 * it is just duplicate segment or bug in other side's TCP. 715 * So that we build reply only basing on parameters 716 * arrived with segment. 717 * Exception: precedence violation. We do not implement it in any case. 718 */ 719 720 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb, 721 enum sk_rst_reason reason) 722 { 723 const struct tcphdr *th = tcp_hdr(skb); 724 struct { 725 struct tcphdr th; 726 __be32 opt[REPLY_OPTIONS_LEN]; 727 } rep; 728 const __u8 *md5_hash_location = NULL; 729 const struct tcp_ao_hdr *aoh; 730 struct ip_reply_arg arg; 731 #ifdef CONFIG_TCP_MD5SIG 732 struct tcp_md5sig_key *key = NULL; 733 unsigned char newhash[16]; 734 struct sock *sk1 = NULL; 735 #endif 736 u64 transmit_time = 0; 737 struct sock *ctl_sk; 738 struct net *net; 739 u32 txhash = 0; 740 741 /* Never send a reset in response to a reset. */ 742 if (th->rst) 743 return; 744 745 /* If sk not NULL, it means we did a successful lookup and incoming 746 * route had to be correct. prequeue might have dropped our dst. 747 */ 748 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL) 749 return; 750 751 /* Swap the send and the receive. */ 752 memset(&rep, 0, sizeof(rep)); 753 rep.th.dest = th->source; 754 rep.th.source = th->dest; 755 rep.th.doff = sizeof(struct tcphdr) / 4; 756 rep.th.rst = 1; 757 758 if (th->ack) { 759 rep.th.seq = th->ack_seq; 760 } else { 761 rep.th.ack = 1; 762 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin + 763 skb->len - (th->doff << 2)); 764 } 765 766 memset(&arg, 0, sizeof(arg)); 767 arg.iov[0].iov_base = (unsigned char *)&rep; 768 arg.iov[0].iov_len = sizeof(rep.th); 769 770 net = sk ? sock_net(sk) : skb_dst_dev_net_rcu(skb); 771 772 /* Invalid TCP option size or twice included auth */ 773 if (tcp_parse_auth_options(tcp_hdr(skb), &md5_hash_location, &aoh)) 774 return; 775 776 if (aoh && tcp_v4_ao_sign_reset(sk, skb, aoh, &arg, &rep.th, rep.opt)) 777 return; 778 779 #ifdef CONFIG_TCP_MD5SIG 780 rcu_read_lock(); 781 if (sk && sk_fullsock(sk)) { 782 const union tcp_md5_addr *addr; 783 int l3index; 784 785 /* sdif set, means packet ingressed via a device 786 * in an L3 domain and inet_iif is set to it. 787 */ 788 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0; 789 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr; 790 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET); 791 } else if (md5_hash_location) { 792 const union tcp_md5_addr *addr; 793 int sdif = tcp_v4_sdif(skb); 794 int dif = inet_iif(skb); 795 int l3index; 796 797 /* 798 * active side is lost. Try to find listening socket through 799 * source port, and then find md5 key through listening socket. 800 * we are not loose security here: 801 * Incoming packet is checked with md5 hash with finding key, 802 * no RST generated if md5 hash doesn't match. 803 */ 804 sk1 = __inet_lookup_listener(net, NULL, 0, ip_hdr(skb)->saddr, 805 th->source, ip_hdr(skb)->daddr, 806 ntohs(th->source), dif, sdif); 807 /* don't send rst if it can't find key */ 808 if (!sk1) 809 goto out; 810 811 /* sdif set, means packet ingressed via a device 812 * in an L3 domain and dif is set to it. 813 */ 814 l3index = sdif ? dif : 0; 815 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr; 816 key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET); 817 if (!key) 818 goto out; 819 820 tcp_v4_md5_hash_skb(newhash, key, NULL, skb); 821 if (crypto_memneq(md5_hash_location, newhash, 16)) 822 goto out; 823 } 824 825 if (key) { 826 rep.opt[0] = htonl((TCPOPT_NOP << 24) | 827 (TCPOPT_NOP << 16) | 828 (TCPOPT_MD5SIG << 8) | 829 TCPOLEN_MD5SIG); 830 /* Update length and the length the header thinks exists */ 831 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED; 832 rep.th.doff = arg.iov[0].iov_len / 4; 833 834 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1], 835 key, ip_hdr(skb)->saddr, 836 ip_hdr(skb)->daddr, &rep.th); 837 } 838 #endif 839 /* Can't co-exist with TCPMD5, hence check rep.opt[0] */ 840 if (rep.opt[0] == 0) { 841 __be32 mrst = mptcp_reset_option(skb); 842 843 if (mrst) { 844 rep.opt[0] = mrst; 845 arg.iov[0].iov_len += sizeof(mrst); 846 rep.th.doff = arg.iov[0].iov_len / 4; 847 } 848 } 849 850 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr, 851 ip_hdr(skb)->saddr, /* XXX */ 852 arg.iov[0].iov_len, IPPROTO_TCP, 0); 853 arg.csumoffset = offsetof(struct tcphdr, check) / 2; 854 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0; 855 856 /* When socket is gone, all binding information is lost. 857 * routing might fail in this case. No choice here, if we choose to force 858 * input interface, we will misroute in case of asymmetric route. 859 */ 860 if (sk) 861 arg.bound_dev_if = sk->sk_bound_dev_if; 862 863 trace_tcp_send_reset(sk, skb, reason); 864 865 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) != 866 offsetof(struct inet_timewait_sock, tw_bound_dev_if)); 867 868 /* ECN bits of TW reset are cleared */ 869 arg.tos = ip_hdr(skb)->tos & ~INET_ECN_MASK; 870 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL); 871 local_bh_disable(); 872 local_lock_nested_bh(&ipv4_tcp_sk.bh_lock); 873 ctl_sk = this_cpu_read(ipv4_tcp_sk.sock); 874 875 sock_net_set(ctl_sk, net); 876 if (sk) { 877 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ? 878 inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark); 879 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ? 880 inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority); 881 transmit_time = tcp_transmit_time(sk); 882 xfrm_sk_clone_policy(ctl_sk, sk); 883 txhash = (sk->sk_state == TCP_TIME_WAIT) ? 884 inet_twsk(sk)->tw_txhash : sk->sk_txhash; 885 } else { 886 ctl_sk->sk_mark = 0; 887 ctl_sk->sk_priority = 0; 888 } 889 ip_send_unicast_reply(ctl_sk, sk, 890 skb, &TCP_SKB_CB(skb)->header.h4.opt, 891 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr, 892 &arg, arg.iov[0].iov_len, 893 transmit_time, txhash); 894 895 xfrm_sk_free_policy(ctl_sk); 896 sock_net_set(ctl_sk, &init_net); 897 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS); 898 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS); 899 local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock); 900 local_bh_enable(); 901 902 #ifdef CONFIG_TCP_MD5SIG 903 out: 904 rcu_read_unlock(); 905 #endif 906 } 907 908 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states 909 outside socket context is ugly, certainly. What can I do? 910 */ 911 912 static void tcp_v4_send_ack(const struct sock *sk, 913 struct sk_buff *skb, u32 seq, u32 ack, 914 u32 win, u32 tsval, u32 tsecr, int oif, 915 struct tcp_key *key, 916 int reply_flags, u8 tos, u32 txhash) 917 { 918 const struct tcphdr *th = tcp_hdr(skb); 919 struct { 920 struct tcphdr th; 921 __be32 opt[(MAX_TCP_OPTION_SPACE >> 2)]; 922 } rep; 923 struct net *net = sock_net(sk); 924 struct ip_reply_arg arg; 925 struct sock *ctl_sk; 926 u64 transmit_time; 927 928 memset(&rep.th, 0, sizeof(struct tcphdr)); 929 memset(&arg, 0, sizeof(arg)); 930 931 arg.iov[0].iov_base = (unsigned char *)&rep; 932 arg.iov[0].iov_len = sizeof(rep.th); 933 if (tsecr) { 934 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | 935 (TCPOPT_TIMESTAMP << 8) | 936 TCPOLEN_TIMESTAMP); 937 rep.opt[1] = htonl(tsval); 938 rep.opt[2] = htonl(tsecr); 939 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED; 940 } 941 942 /* Swap the send and the receive. */ 943 rep.th.dest = th->source; 944 rep.th.source = th->dest; 945 rep.th.doff = arg.iov[0].iov_len / 4; 946 rep.th.seq = htonl(seq); 947 rep.th.ack_seq = htonl(ack); 948 rep.th.ack = 1; 949 rep.th.window = htons(win); 950 951 #ifdef CONFIG_TCP_MD5SIG 952 if (tcp_key_is_md5(key)) { 953 int offset = (tsecr) ? 3 : 0; 954 955 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) | 956 (TCPOPT_NOP << 16) | 957 (TCPOPT_MD5SIG << 8) | 958 TCPOLEN_MD5SIG); 959 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED; 960 rep.th.doff = arg.iov[0].iov_len/4; 961 962 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset], 963 key->md5_key, ip_hdr(skb)->saddr, 964 ip_hdr(skb)->daddr, &rep.th); 965 } 966 #endif 967 #ifdef CONFIG_TCP_AO 968 if (tcp_key_is_ao(key)) { 969 int offset = (tsecr) ? 3 : 0; 970 971 rep.opt[offset++] = htonl((TCPOPT_AO << 24) | 972 (tcp_ao_len(key->ao_key) << 16) | 973 (key->ao_key->sndid << 8) | 974 key->rcv_next); 975 arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key); 976 rep.th.doff = arg.iov[0].iov_len / 4; 977 memset((u8 *)&rep.opt[offset] + tcp_ao_maclen(key->ao_key), 978 TCPOPT_NOP, tcp_ao_len_aligned(key->ao_key) - 979 tcp_ao_len(key->ao_key)); 980 981 tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset], 982 key->ao_key, key->traffic_key, 983 (union tcp_ao_addr *)&ip_hdr(skb)->saddr, 984 (union tcp_ao_addr *)&ip_hdr(skb)->daddr, 985 &rep.th, key->sne); 986 } 987 #endif 988 arg.flags = reply_flags; 989 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr, 990 ip_hdr(skb)->saddr, /* XXX */ 991 arg.iov[0].iov_len, IPPROTO_TCP, 0); 992 arg.csumoffset = offsetof(struct tcphdr, check) / 2; 993 if (oif) 994 arg.bound_dev_if = oif; 995 arg.tos = tos; 996 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL); 997 local_bh_disable(); 998 local_lock_nested_bh(&ipv4_tcp_sk.bh_lock); 999 ctl_sk = this_cpu_read(ipv4_tcp_sk.sock); 1000 sock_net_set(ctl_sk, net); 1001 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ? 1002 inet_twsk(sk)->tw_mark : READ_ONCE(sk->sk_mark); 1003 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ? 1004 inet_twsk(sk)->tw_priority : READ_ONCE(sk->sk_priority); 1005 transmit_time = tcp_transmit_time(sk); 1006 ip_send_unicast_reply(ctl_sk, sk, 1007 skb, &TCP_SKB_CB(skb)->header.h4.opt, 1008 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr, 1009 &arg, arg.iov[0].iov_len, 1010 transmit_time, txhash); 1011 1012 sock_net_set(ctl_sk, &init_net); 1013 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS); 1014 local_unlock_nested_bh(&ipv4_tcp_sk.bh_lock); 1015 local_bh_enable(); 1016 } 1017 1018 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb, 1019 enum tcp_tw_status tw_status) 1020 { 1021 struct inet_timewait_sock *tw = inet_twsk(sk); 1022 struct tcp_timewait_sock *tcptw = tcp_twsk(sk); 1023 struct tcp_key key = {}; 1024 u8 tos = tw->tw_tos; 1025 1026 /* Cleaning only ECN bits of TW ACKs of oow data or is paws_reject, 1027 * while not cleaning ECN bits of other TW ACKs to avoid these ACKs 1028 * being placed in a different service queues (Classic rather than L4S) 1029 */ 1030 if (tw_status == TCP_TW_ACK_OOW) 1031 tos &= ~INET_ECN_MASK; 1032 1033 #ifdef CONFIG_TCP_AO 1034 struct tcp_ao_info *ao_info; 1035 1036 if (static_branch_unlikely(&tcp_ao_needed.key)) { 1037 /* FIXME: the segment to-be-acked is not verified yet */ 1038 ao_info = rcu_dereference(tcptw->ao_info); 1039 if (ao_info) { 1040 const struct tcp_ao_hdr *aoh; 1041 1042 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) { 1043 inet_twsk_put(tw); 1044 return; 1045 } 1046 1047 if (aoh) 1048 key.ao_key = tcp_ao_established_key(sk, ao_info, 1049 aoh->rnext_keyid, -1); 1050 } 1051 } 1052 if (key.ao_key) { 1053 struct tcp_ao_key *rnext_key; 1054 1055 key.traffic_key = snd_other_key(key.ao_key); 1056 key.sne = READ_ONCE(ao_info->snd_sne); 1057 rnext_key = READ_ONCE(ao_info->rnext_key); 1058 key.rcv_next = rnext_key->rcvid; 1059 key.type = TCP_KEY_AO; 1060 #else 1061 if (0) { 1062 #endif 1063 } else if (static_branch_tcp_md5()) { 1064 key.md5_key = tcp_twsk_md5_key(tcptw); 1065 if (key.md5_key) 1066 key.type = TCP_KEY_MD5; 1067 } 1068 1069 tcp_v4_send_ack(sk, skb, 1070 tcptw->tw_snd_nxt, READ_ONCE(tcptw->tw_rcv_nxt), 1071 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, 1072 tcp_tw_tsval(tcptw), 1073 READ_ONCE(tcptw->tw_ts_recent), 1074 tw->tw_bound_dev_if, &key, 1075 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0, 1076 tos, 1077 tw->tw_txhash); 1078 1079 inet_twsk_put(tw); 1080 } 1081 1082 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, 1083 struct request_sock *req) 1084 { 1085 struct tcp_key key = {}; 1086 1087 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV 1088 * sk->sk_state == TCP_SYN_RECV -> for Fast Open. 1089 */ 1090 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 : 1091 tcp_sk(sk)->snd_nxt; 1092 1093 #ifdef CONFIG_TCP_AO 1094 if (static_branch_unlikely(&tcp_ao_needed.key) && 1095 tcp_rsk_used_ao(req)) { 1096 const union tcp_md5_addr *addr; 1097 const struct tcp_ao_hdr *aoh; 1098 int l3index; 1099 1100 /* Invalid TCP option size or twice included auth */ 1101 if (tcp_parse_auth_options(tcp_hdr(skb), NULL, &aoh)) 1102 return; 1103 if (!aoh) 1104 return; 1105 1106 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr; 1107 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0; 1108 key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET, 1109 aoh->rnext_keyid, -1); 1110 if (unlikely(!key.ao_key)) { 1111 /* Send ACK with any matching MKT for the peer */ 1112 key.ao_key = tcp_ao_do_lookup(sk, l3index, addr, AF_INET, -1, -1); 1113 /* Matching key disappeared (user removed the key?) 1114 * let the handshake timeout. 1115 */ 1116 if (!key.ao_key) { 1117 net_info_ratelimited("TCP-AO key for (%pI4, %d)->(%pI4, %d) suddenly disappeared, won't ACK new connection\n", 1118 addr, 1119 ntohs(tcp_hdr(skb)->source), 1120 &ip_hdr(skb)->daddr, 1121 ntohs(tcp_hdr(skb)->dest)); 1122 return; 1123 } 1124 } 1125 key.traffic_key = kmalloc(tcp_ao_digest_size(key.ao_key), GFP_ATOMIC); 1126 if (!key.traffic_key) 1127 return; 1128 1129 key.type = TCP_KEY_AO; 1130 key.rcv_next = aoh->keyid; 1131 tcp_v4_ao_calc_key_rsk(key.ao_key, key.traffic_key, req); 1132 #else 1133 if (0) { 1134 #endif 1135 } else if (static_branch_tcp_md5()) { 1136 const union tcp_md5_addr *addr; 1137 int l3index; 1138 1139 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr; 1140 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0; 1141 key.md5_key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET); 1142 if (key.md5_key) 1143 key.type = TCP_KEY_MD5; 1144 } 1145 1146 /* Cleaning ECN bits of TW ACKs of oow data or is paws_reject */ 1147 tcp_v4_send_ack(sk, skb, seq, 1148 tcp_rsk(req)->rcv_nxt, 1149 tcp_synack_window(req) >> inet_rsk(req)->rcv_wscale, 1150 tcp_rsk_tsval(tcp_rsk(req)), 1151 req->ts_recent, 1152 0, &key, 1153 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0, 1154 ip_hdr(skb)->tos & ~INET_ECN_MASK, 1155 READ_ONCE(tcp_rsk(req)->txhash)); 1156 if (tcp_key_is_ao(&key)) 1157 kfree(key.traffic_key); 1158 } 1159 1160 /* 1161 * Send a SYN-ACK after having received a SYN. 1162 * This still operates on a request_sock only, not on a big 1163 * socket. 1164 */ 1165 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst, 1166 struct flowi *fl, 1167 struct request_sock *req, 1168 struct tcp_fastopen_cookie *foc, 1169 enum tcp_synack_type synack_type, 1170 struct sk_buff *syn_skb) 1171 { 1172 struct inet_request_sock *ireq = inet_rsk(req); 1173 struct flowi4 fl4; 1174 int err = -1; 1175 struct sk_buff *skb; 1176 u8 tos; 1177 1178 /* First, grab a route. */ 1179 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL) 1180 return -1; 1181 1182 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb); 1183 1184 if (skb) { 1185 tcp_rsk(req)->syn_ect_snt = inet_sk(sk)->tos & INET_ECN_MASK; 1186 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr); 1187 1188 tos = READ_ONCE(inet_sk(sk)->tos); 1189 1190 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) 1191 tos = (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) | 1192 (tos & INET_ECN_MASK); 1193 1194 if (!INET_ECN_is_capable(tos) && 1195 tcp_bpf_ca_needs_ecn((struct sock *)req)) 1196 tos |= INET_ECN_ECT_0; 1197 1198 rcu_read_lock(); 1199 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr, 1200 ireq->ir_rmt_addr, 1201 rcu_dereference(ireq->ireq_opt), 1202 tos); 1203 rcu_read_unlock(); 1204 err = net_xmit_eval(err); 1205 } 1206 1207 return err; 1208 } 1209 1210 /* 1211 * IPv4 request_sock destructor. 1212 */ 1213 static void tcp_v4_reqsk_destructor(struct request_sock *req) 1214 { 1215 kfree_rcu(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1), rcu); 1216 } 1217 1218 #ifdef CONFIG_TCP_MD5SIG 1219 /* 1220 * RFC2385 MD5 checksumming requires a mapping of 1221 * IP address->MD5 Key. 1222 * We need to maintain these in the sk structure. 1223 */ 1224 1225 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_md5_needed, HZ); 1226 1227 static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new) 1228 { 1229 if (!old) 1230 return true; 1231 1232 /* l3index always overrides non-l3index */ 1233 if (old->l3index && new->l3index == 0) 1234 return false; 1235 if (old->l3index == 0 && new->l3index) 1236 return true; 1237 1238 return old->prefixlen < new->prefixlen; 1239 } 1240 1241 /* Find the Key structure for an address. */ 1242 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 1243 const union tcp_md5_addr *addr, 1244 int family, bool any_l3index) 1245 { 1246 const struct tcp_sock *tp = tcp_sk(sk); 1247 struct tcp_md5sig_key *key; 1248 const struct tcp_md5sig_info *md5sig; 1249 __be32 mask; 1250 struct tcp_md5sig_key *best_match = NULL; 1251 bool match; 1252 1253 /* caller either holds rcu_read_lock() or socket lock */ 1254 md5sig = rcu_dereference_check(tp->md5sig_info, 1255 lockdep_sock_is_held(sk)); 1256 if (!md5sig) 1257 return NULL; 1258 1259 hlist_for_each_entry_rcu(key, &md5sig->head, node, 1260 lockdep_sock_is_held(sk)) { 1261 if (key->family != family) 1262 continue; 1263 if (!any_l3index && key->flags & TCP_MD5SIG_FLAG_IFINDEX && 1264 key->l3index != l3index) 1265 continue; 1266 if (family == AF_INET) { 1267 mask = inet_make_mask(key->prefixlen); 1268 match = (key->addr.a4.s_addr & mask) == 1269 (addr->a4.s_addr & mask); 1270 #if IS_ENABLED(CONFIG_IPV6) 1271 } else if (family == AF_INET6) { 1272 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6, 1273 key->prefixlen); 1274 #endif 1275 } else { 1276 match = false; 1277 } 1278 1279 if (match && better_md5_match(best_match, key)) 1280 best_match = key; 1281 } 1282 return best_match; 1283 } 1284 1285 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk, 1286 const union tcp_md5_addr *addr, 1287 int family, u8 prefixlen, 1288 int l3index, u8 flags) 1289 { 1290 const struct tcp_sock *tp = tcp_sk(sk); 1291 struct tcp_md5sig_key *key; 1292 unsigned int size = sizeof(struct in_addr); 1293 const struct tcp_md5sig_info *md5sig; 1294 1295 /* caller either holds rcu_read_lock() or socket lock */ 1296 md5sig = rcu_dereference_check(tp->md5sig_info, 1297 lockdep_sock_is_held(sk)); 1298 if (!md5sig) 1299 return NULL; 1300 #if IS_ENABLED(CONFIG_IPV6) 1301 if (family == AF_INET6) 1302 size = sizeof(struct in6_addr); 1303 #endif 1304 hlist_for_each_entry_rcu(key, &md5sig->head, node, 1305 lockdep_sock_is_held(sk)) { 1306 if (key->family != family) 1307 continue; 1308 if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX)) 1309 continue; 1310 if (key->l3index != l3index) 1311 continue; 1312 if (!memcmp(&key->addr, addr, size) && 1313 key->prefixlen == prefixlen) 1314 return key; 1315 } 1316 return NULL; 1317 } 1318 1319 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 1320 const struct sock *addr_sk) 1321 { 1322 const union tcp_md5_addr *addr; 1323 int l3index; 1324 1325 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), 1326 addr_sk->sk_bound_dev_if); 1327 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr; 1328 return tcp_md5_do_lookup(sk, l3index, addr, AF_INET); 1329 } 1330 1331 static int tcp_md5sig_info_add(struct sock *sk, gfp_t gfp) 1332 { 1333 struct tcp_sock *tp = tcp_sk(sk); 1334 struct tcp_md5sig_info *md5sig; 1335 1336 md5sig = kmalloc_obj(*md5sig, gfp); 1337 if (!md5sig) 1338 return -ENOMEM; 1339 1340 sk_gso_disable(sk); 1341 INIT_HLIST_HEAD(&md5sig->head); 1342 rcu_assign_pointer(tp->md5sig_info, md5sig); 1343 return 0; 1344 } 1345 1346 /* This can be called on a newly created socket, from other files */ 1347 static int __tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1348 int family, u8 prefixlen, int l3index, u8 flags, 1349 const u8 *newkey, u8 newkeylen, gfp_t gfp) 1350 { 1351 /* Add Key to the list */ 1352 struct tcp_md5sig_key *key; 1353 struct tcp_sock *tp = tcp_sk(sk); 1354 struct tcp_md5sig_info *md5sig; 1355 1356 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags); 1357 if (key) { 1358 /* Pre-existing entry - just update that one. 1359 * Note that the key might be used concurrently. 1360 * data_race() is telling kcsan that we do not care of 1361 * key mismatches, since changing MD5 key on live flows 1362 * can lead to packet drops. 1363 */ 1364 data_race(memcpy(key->key, newkey, newkeylen)); 1365 1366 /* Pairs with READ_ONCE() in tcp_md5_hash_key(). 1367 * Also note that a reader could catch new key->keylen value 1368 * but old key->key[], this is the reason we use __GFP_ZERO 1369 * at sock_kmalloc() time below these lines. 1370 */ 1371 WRITE_ONCE(key->keylen, newkeylen); 1372 1373 return 0; 1374 } 1375 1376 md5sig = rcu_dereference_protected(tp->md5sig_info, 1377 lockdep_sock_is_held(sk)); 1378 1379 key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO); 1380 if (!key) 1381 return -ENOMEM; 1382 1383 memcpy(key->key, newkey, newkeylen); 1384 key->keylen = newkeylen; 1385 key->family = family; 1386 key->prefixlen = prefixlen; 1387 key->l3index = l3index; 1388 key->flags = flags; 1389 memcpy(&key->addr, addr, 1390 (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) : 1391 sizeof(struct in_addr)); 1392 hlist_add_head_rcu(&key->node, &md5sig->head); 1393 return 0; 1394 } 1395 1396 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 1397 int family, u8 prefixlen, int l3index, u8 flags, 1398 const u8 *newkey, u8 newkeylen) 1399 { 1400 struct tcp_sock *tp = tcp_sk(sk); 1401 1402 if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) { 1403 if (fips_enabled) { 1404 pr_warn_once("TCP-MD5 support is disabled due to FIPS\n"); 1405 return -EOPNOTSUPP; 1406 } 1407 1408 if (tcp_md5sig_info_add(sk, GFP_KERNEL)) 1409 return -ENOMEM; 1410 1411 if (!static_branch_inc(&tcp_md5_needed.key)) { 1412 struct tcp_md5sig_info *md5sig; 1413 1414 md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); 1415 rcu_assign_pointer(tp->md5sig_info, NULL); 1416 kfree_rcu(md5sig, rcu); 1417 return -EUSERS; 1418 } 1419 } 1420 1421 return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, flags, 1422 newkey, newkeylen, GFP_KERNEL); 1423 } 1424 1425 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 1426 int family, u8 prefixlen, int l3index, 1427 struct tcp_md5sig_key *key) 1428 { 1429 struct tcp_sock *tp = tcp_sk(sk); 1430 1431 if (!rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk))) { 1432 1433 if (tcp_md5sig_info_add(sk, sk_gfp_mask(sk, GFP_ATOMIC))) 1434 return -ENOMEM; 1435 1436 if (!static_key_fast_inc_not_disabled(&tcp_md5_needed.key.key)) { 1437 struct tcp_md5sig_info *md5sig; 1438 1439 md5sig = rcu_dereference_protected(tp->md5sig_info, lockdep_sock_is_held(sk)); 1440 net_warn_ratelimited("Too many TCP-MD5 keys in the system\n"); 1441 rcu_assign_pointer(tp->md5sig_info, NULL); 1442 kfree_rcu(md5sig, rcu); 1443 return -EUSERS; 1444 } 1445 } 1446 1447 return __tcp_md5_do_add(sk, addr, family, prefixlen, l3index, 1448 key->flags, key->key, key->keylen, 1449 sk_gfp_mask(sk, GFP_ATOMIC)); 1450 } 1451 1452 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family, 1453 u8 prefixlen, int l3index, u8 flags) 1454 { 1455 struct tcp_md5sig_key *key; 1456 1457 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags); 1458 if (!key) 1459 return -ENOENT; 1460 hlist_del_rcu(&key->node); 1461 atomic_sub(sizeof(*key), &sk->sk_omem_alloc); 1462 kfree_rcu(key, rcu); 1463 return 0; 1464 } 1465 1466 void tcp_clear_md5_list(struct sock *sk) 1467 { 1468 struct tcp_sock *tp = tcp_sk(sk); 1469 struct tcp_md5sig_key *key; 1470 struct hlist_node *n; 1471 struct tcp_md5sig_info *md5sig; 1472 1473 md5sig = rcu_dereference_protected(tp->md5sig_info, 1); 1474 1475 hlist_for_each_entry_safe(key, n, &md5sig->head, node) { 1476 hlist_del_rcu(&key->node); 1477 atomic_sub(sizeof(*key), &sk->sk_omem_alloc); 1478 kfree_rcu(key, rcu); 1479 } 1480 } 1481 1482 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname, 1483 sockptr_t optval, int optlen) 1484 { 1485 struct tcp_md5sig cmd; 1486 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr; 1487 const union tcp_md5_addr *addr; 1488 u8 prefixlen = 32; 1489 int l3index = 0; 1490 bool l3flag; 1491 u8 flags; 1492 1493 if (optlen < sizeof(cmd)) 1494 return -EINVAL; 1495 1496 if (copy_from_sockptr(&cmd, optval, sizeof(cmd))) 1497 return -EFAULT; 1498 1499 if (sin->sin_family != AF_INET) 1500 return -EINVAL; 1501 1502 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 1503 l3flag = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX; 1504 1505 if (optname == TCP_MD5SIG_EXT && 1506 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) { 1507 prefixlen = cmd.tcpm_prefixlen; 1508 if (prefixlen > 32) 1509 return -EINVAL; 1510 } 1511 1512 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex && 1513 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) { 1514 struct net_device *dev; 1515 1516 rcu_read_lock(); 1517 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex); 1518 if (dev && netif_is_l3_master(dev)) 1519 l3index = dev->ifindex; 1520 1521 rcu_read_unlock(); 1522 1523 /* ok to reference set/not set outside of rcu; 1524 * right now device MUST be an L3 master 1525 */ 1526 if (!dev || !l3index) 1527 return -EINVAL; 1528 } 1529 1530 addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr; 1531 1532 if (!cmd.tcpm_keylen) 1533 return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags); 1534 1535 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN) 1536 return -EINVAL; 1537 1538 /* Don't allow keys for peers that have a matching TCP-AO key. 1539 * See the comment in tcp_ao_add_cmd() 1540 */ 1541 if (tcp_ao_required(sk, addr, AF_INET, l3flag ? l3index : -1, false)) 1542 return -EKEYREJECTED; 1543 1544 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags, 1545 cmd.tcpm_key, cmd.tcpm_keylen); 1546 } 1547 1548 static void tcp_v4_md5_hash_headers(struct md5_ctx *ctx, 1549 __be32 daddr, __be32 saddr, 1550 const struct tcphdr *th, int nbytes) 1551 { 1552 struct { 1553 struct tcp4_pseudohdr ip; 1554 struct tcphdr tcp; 1555 } h; 1556 1557 h.ip.saddr = saddr; 1558 h.ip.daddr = daddr; 1559 h.ip.pad = 0; 1560 h.ip.protocol = IPPROTO_TCP; 1561 h.ip.len = cpu_to_be16(nbytes); 1562 h.tcp = *th; 1563 h.tcp.check = 0; 1564 md5_update(ctx, (const u8 *)&h, sizeof(h.ip) + sizeof(h.tcp)); 1565 } 1566 1567 static noinline_for_stack void 1568 tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, 1569 __be32 daddr, __be32 saddr, const struct tcphdr *th) 1570 { 1571 struct md5_ctx ctx; 1572 1573 md5_init(&ctx); 1574 tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, th->doff << 2); 1575 tcp_md5_hash_key(&ctx, key); 1576 md5_final(&ctx, md5_hash); 1577 } 1578 1579 noinline_for_stack void 1580 tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 1581 const struct sock *sk, const struct sk_buff *skb) 1582 { 1583 const struct tcphdr *th = tcp_hdr(skb); 1584 __be32 saddr, daddr; 1585 struct md5_ctx ctx; 1586 1587 if (sk) { /* valid for establish/request sockets */ 1588 saddr = sk->sk_rcv_saddr; 1589 daddr = sk->sk_daddr; 1590 } else { 1591 const struct iphdr *iph = ip_hdr(skb); 1592 saddr = iph->saddr; 1593 daddr = iph->daddr; 1594 } 1595 1596 md5_init(&ctx); 1597 tcp_v4_md5_hash_headers(&ctx, daddr, saddr, th, skb->len); 1598 tcp_md5_hash_skb_data(&ctx, skb, th->doff << 2); 1599 tcp_md5_hash_key(&ctx, key); 1600 md5_final(&ctx, md5_hash); 1601 } 1602 1603 #endif 1604 1605 static void tcp_v4_init_req(struct request_sock *req, 1606 const struct sock *sk_listener, 1607 struct sk_buff *skb) 1608 { 1609 struct inet_request_sock *ireq = inet_rsk(req); 1610 struct net *net = sock_net(sk_listener); 1611 1612 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr); 1613 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr); 1614 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb)); 1615 } 1616 1617 static struct dst_entry *tcp_v4_route_req(const struct sock *sk, 1618 struct sk_buff *skb, 1619 struct flowi *fl, 1620 struct request_sock *req, 1621 u32 tw_isn) 1622 { 1623 tcp_v4_init_req(req, sk, skb); 1624 1625 if (security_inet_conn_request(sk, skb, req)) 1626 return NULL; 1627 1628 return inet_csk_route_req(sk, &fl->u.ip4, req); 1629 } 1630 1631 struct request_sock_ops tcp_request_sock_ops __read_mostly = { 1632 .family = PF_INET, 1633 .obj_size = sizeof(struct tcp_request_sock), 1634 .send_ack = tcp_v4_reqsk_send_ack, 1635 .destructor = tcp_v4_reqsk_destructor, 1636 .send_reset = tcp_v4_send_reset, 1637 }; 1638 1639 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = { 1640 .mss_clamp = TCP_MSS_DEFAULT, 1641 #ifdef CONFIG_TCP_MD5SIG 1642 .req_md5_lookup = tcp_v4_md5_lookup, 1643 .calc_md5_hash = tcp_v4_md5_hash_skb, 1644 #endif 1645 #ifdef CONFIG_TCP_AO 1646 .ao_lookup = tcp_v4_ao_lookup_rsk, 1647 .ao_calc_key = tcp_v4_ao_calc_key_rsk, 1648 .ao_synack_hash = tcp_v4_ao_synack_hash, 1649 #endif 1650 #ifdef CONFIG_SYN_COOKIES 1651 .cookie_init_seq = cookie_v4_init_sequence, 1652 #endif 1653 .route_req = tcp_v4_route_req, 1654 .init_seq_and_ts_off = tcp_v4_init_seq_and_ts_off, 1655 .send_synack = tcp_v4_send_synack, 1656 }; 1657 1658 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb) 1659 { 1660 /* Never answer to SYNs send to broadcast or multicast */ 1661 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) 1662 goto drop; 1663 1664 return tcp_conn_request(&tcp_request_sock_ops, 1665 &tcp_request_sock_ipv4_ops, sk, skb); 1666 1667 drop: 1668 tcp_listendrop(sk); 1669 return 0; 1670 } 1671 1672 1673 /* 1674 * The three way handshake has completed - we got a valid synack - 1675 * now create the new socket. 1676 */ 1677 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 1678 struct request_sock *req, 1679 struct dst_entry *dst, 1680 struct request_sock *req_unhash, 1681 bool *own_req, 1682 void (*opt_child_init)(struct sock *newsk, 1683 const struct sock *sk)) 1684 { 1685 struct inet_request_sock *ireq; 1686 bool found_dup_sk = false; 1687 struct inet_sock *newinet; 1688 struct tcp_sock *newtp; 1689 struct sock *newsk; 1690 #ifdef CONFIG_TCP_MD5SIG 1691 const union tcp_md5_addr *addr; 1692 struct tcp_md5sig_key *key; 1693 int l3index; 1694 #endif 1695 struct ip_options_rcu *inet_opt; 1696 1697 if (sk_acceptq_is_full(sk)) 1698 goto exit_overflow; 1699 1700 newsk = tcp_create_openreq_child(sk, req, skb); 1701 if (!newsk) 1702 goto exit_nonewsk; 1703 1704 newsk->sk_gso_type = SKB_GSO_TCPV4; 1705 inet_sk_rx_dst_set(newsk, skb); 1706 1707 newtp = tcp_sk(newsk); 1708 newinet = inet_sk(newsk); 1709 ireq = inet_rsk(req); 1710 inet_opt = rcu_dereference(ireq->ireq_opt); 1711 RCU_INIT_POINTER(newinet->inet_opt, inet_opt); 1712 newinet->mc_index = inet_iif(skb); 1713 newinet->mc_ttl = ip_hdr(skb)->ttl; 1714 newinet->rcv_tos = ip_hdr(skb)->tos; 1715 inet_csk(newsk)->icsk_ext_hdr_len = 0; 1716 if (inet_opt) 1717 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen; 1718 atomic_set(&newinet->inet_id, get_random_u16()); 1719 1720 /* Set ToS of the new socket based upon the value of incoming SYN. 1721 * ECT bits are set later in tcp_init_transfer(). 1722 */ 1723 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos)) 1724 newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK; 1725 1726 #if IS_ENABLED(CONFIG_IPV6) 1727 /* Detach IPv6 state before a route lookup can fail. */ 1728 if (opt_child_init) 1729 opt_child_init(newsk, sk); 1730 #endif 1731 1732 if (!dst) { 1733 dst = inet_csk_route_child_sock(sk, newsk, req); 1734 if (!dst) 1735 goto put_and_exit; 1736 } else { 1737 /* syncookie case : see end of cookie_v4_check() */ 1738 } 1739 sk_setup_caps(newsk, dst); 1740 1741 tcp_ca_openreq_child(newsk, dst); 1742 1743 tcp_sync_mss(newsk, dst4_mtu(dst)); 1744 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); 1745 1746 tcp_initialize_rcv_mss(newsk); 1747 1748 #ifdef CONFIG_TCP_MD5SIG 1749 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif); 1750 /* Copy over the MD5 key from the original socket */ 1751 addr = (union tcp_md5_addr *)&newinet->inet_daddr; 1752 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET); 1753 if (key && !tcp_rsk_used_ao(req)) { 1754 if (tcp_md5_key_copy(newsk, addr, AF_INET, 32, l3index, key)) 1755 goto put_and_exit; 1756 sk_gso_disable(newsk); 1757 } 1758 #endif 1759 #ifdef CONFIG_TCP_AO 1760 if (tcp_ao_copy_all_matching(sk, newsk, req, skb, AF_INET)) 1761 goto put_and_exit; /* OOM, release back memory */ 1762 #endif 1763 1764 if (__inet_inherit_port(sk, newsk) < 0) 1765 goto put_and_exit; 1766 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash), 1767 &found_dup_sk); 1768 if (likely(*own_req)) { 1769 tcp_move_syn(newtp, req); 1770 ireq->ireq_opt = NULL; 1771 } else { 1772 newinet->inet_opt = NULL; 1773 1774 if (!req_unhash && found_dup_sk) { 1775 /* This code path should only be executed in the 1776 * syncookie case only 1777 */ 1778 bh_unlock_sock(newsk); 1779 sock_put(newsk); 1780 newsk = NULL; 1781 } 1782 } 1783 return newsk; 1784 1785 exit_overflow: 1786 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS); 1787 exit_nonewsk: 1788 dst_release(dst); 1789 exit: 1790 tcp_listendrop(sk); 1791 return NULL; 1792 put_and_exit: 1793 newinet->inet_opt = NULL; 1794 inet_csk_prepare_forced_close(newsk); 1795 tcp_done(newsk); 1796 goto exit; 1797 } 1798 1799 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb) 1800 { 1801 #ifdef CONFIG_SYN_COOKIES 1802 const struct tcphdr *th = tcp_hdr(skb); 1803 1804 if (!th->syn) 1805 sk = cookie_v4_check(sk, skb); 1806 #endif 1807 return sk; 1808 } 1809 1810 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 1811 struct tcphdr *th, u32 *cookie) 1812 { 1813 u16 mss = 0; 1814 #ifdef CONFIG_SYN_COOKIES 1815 mss = tcp_get_syncookie_mss(&tcp_request_sock_ops, 1816 &tcp_request_sock_ipv4_ops, sk, th); 1817 if (mss) { 1818 *cookie = __cookie_v4_init_sequence(iph, th, &mss); 1819 tcp_synq_overflow(sk); 1820 } 1821 #endif 1822 return mss; 1823 } 1824 1825 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, 1826 u32)); 1827 /* The socket must have it's spinlock held when we get 1828 * here, unless it is a TCP_LISTEN socket. 1829 * 1830 * We have a potential double-lock case here, so even when 1831 * doing backlog processing we use the BH locking scheme. 1832 * This is because we cannot sleep with the original spinlock 1833 * held. 1834 */ 1835 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb) 1836 { 1837 enum skb_drop_reason reason; 1838 1839 reason = psp_sk_rx_policy_check(sk, skb); 1840 if (reason) 1841 goto err_discard; 1842 1843 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ 1844 struct dst_entry *dst; 1845 1846 dst = rcu_dereference_protected(sk->sk_rx_dst, 1847 lockdep_sock_is_held(sk)); 1848 1849 sock_rps_save_rxhash(sk, skb); 1850 sk_mark_napi_id(sk, skb); 1851 if (dst && unlikely(dst != skb_dst(skb))) { 1852 if (sk->sk_rx_dst_ifindex != skb->skb_iif || 1853 !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check, 1854 dst, 0)) { 1855 RCU_INIT_POINTER(sk->sk_rx_dst, NULL); 1856 dst_release(dst); 1857 } 1858 } 1859 tcp_rcv_established(sk, skb); 1860 return 0; 1861 } 1862 1863 if (tcp_checksum_complete(skb)) 1864 goto csum_err; 1865 1866 if (sk->sk_state == TCP_LISTEN) { 1867 struct sock *nsk = tcp_v4_cookie_check(sk, skb); 1868 1869 if (!nsk) 1870 return 0; 1871 if (nsk != sk) { 1872 reason = tcp_child_process(sk, nsk, skb); 1873 sock_put(nsk); 1874 if (reason) 1875 goto reset; 1876 return 0; 1877 } 1878 } else 1879 sock_rps_save_rxhash(sk, skb); 1880 1881 reason = tcp_rcv_state_process(sk, skb); 1882 if (reason) 1883 goto reset; 1884 return 0; 1885 1886 reset: 1887 tcp_v4_send_reset(sk, skb, sk_rst_convert_drop_reason(reason)); 1888 discard: 1889 sk_skb_reason_drop(sk, skb, reason); 1890 /* Be careful here. If this function gets more complicated and 1891 * gcc suffers from register pressure on the x86, sk (in %ebx) 1892 * might be destroyed here. This current version compiles correctly, 1893 * but you have been warned. 1894 */ 1895 return 0; 1896 1897 csum_err: 1898 reason = SKB_DROP_REASON_TCP_CSUM; 1899 trace_tcp_bad_csum(skb); 1900 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS); 1901 err_discard: 1902 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS); 1903 goto discard; 1904 } 1905 1906 enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb) 1907 { 1908 u32 tail_gso_size, tail_gso_segs; 1909 struct skb_shared_info *shinfo; 1910 const struct tcphdr *th; 1911 struct tcphdr *thtail; 1912 struct sk_buff *tail; 1913 unsigned int hdrlen; 1914 bool fragstolen; 1915 u32 gso_segs; 1916 u32 gso_size; 1917 u64 limit; 1918 int delta; 1919 int err; 1920 1921 /* In case all data was pulled from skb frags (in __pskb_pull_tail()), 1922 * we can fix skb->truesize to its real value to avoid future drops. 1923 * This is valid because skb is not yet charged to the socket. 1924 * It has been noticed pure SACK packets were sometimes dropped 1925 * (if cooked by drivers without copybreak feature). 1926 */ 1927 skb_condense(skb); 1928 1929 tcp_cleanup_skb(skb); 1930 1931 if (unlikely(tcp_checksum_complete(skb))) { 1932 bh_unlock_sock(sk); 1933 trace_tcp_bad_csum(skb); 1934 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS); 1935 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS); 1936 return SKB_DROP_REASON_TCP_CSUM; 1937 } 1938 1939 /* Attempt coalescing to last skb in backlog, even if we are 1940 * above the limits. 1941 * This is okay because skb capacity is limited to MAX_SKB_FRAGS. 1942 */ 1943 th = (const struct tcphdr *)skb->data; 1944 hdrlen = th->doff * 4; 1945 1946 tail = sk->sk_backlog.tail; 1947 if (!tail) 1948 goto no_coalesce; 1949 thtail = (struct tcphdr *)tail->data; 1950 1951 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq || 1952 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield || 1953 ((TCP_SKB_CB(tail)->tcp_flags | 1954 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) || 1955 !((TCP_SKB_CB(tail)->tcp_flags & 1956 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) || 1957 ((TCP_SKB_CB(tail)->tcp_flags ^ 1958 TCP_SKB_CB(skb)->tcp_flags) & 1959 (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) || 1960 /* Nothing follows a FIN, see tcp_try_coalesce() */ 1961 (TCP_SKB_CB(tail)->tcp_flags & TCPHDR_FIN) || 1962 !tcp_skb_can_collapse_rx(tail, skb) || 1963 thtail->doff != th->doff || 1964 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) || 1965 /* prior to PSP Rx policy check, retain exact PSP metadata */ 1966 psp_skb_coalesce_diff(tail, skb)) 1967 goto no_coalesce; 1968 1969 __skb_pull(skb, hdrlen); 1970 1971 shinfo = skb_shinfo(skb); 1972 gso_size = shinfo->gso_size ?: skb->len; 1973 gso_segs = shinfo->gso_segs ?: 1; 1974 1975 shinfo = skb_shinfo(tail); 1976 tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen); 1977 tail_gso_segs = shinfo->gso_segs ?: 1; 1978 1979 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) { 1980 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq; 1981 1982 if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) { 1983 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq; 1984 thtail->window = th->window; 1985 } 1986 1987 /* We have to update both TCP_SKB_CB(tail)->tcp_flags and 1988 * thtail->fin, so that the fast path in tcp_rcv_established() 1989 * is not entered if we append a packet with a FIN. 1990 * SYN, RST, URG are not present. 1991 * ACK is set on both packets. 1992 * PSH : we do not really care in TCP stack, 1993 * at least for 'GRO' packets. 1994 */ 1995 thtail->fin |= th->fin; 1996 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags; 1997 1998 if (TCP_SKB_CB(skb)->has_rxtstamp) { 1999 TCP_SKB_CB(tail)->has_rxtstamp = true; 2000 tail->tstamp = skb->tstamp; 2001 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp; 2002 } 2003 2004 /* Not as strict as GRO. We only need to carry mss max value */ 2005 shinfo->gso_size = max(gso_size, tail_gso_size); 2006 shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF); 2007 2008 sk->sk_backlog.len += delta; 2009 __NET_INC_STATS(sock_net(sk), 2010 LINUX_MIB_TCPBACKLOGCOALESCE); 2011 kfree_skb_partial(skb, fragstolen); 2012 return SKB_NOT_DROPPED_YET; 2013 } 2014 __skb_push(skb, hdrlen); 2015 2016 no_coalesce: 2017 /* sk->sk_backlog.len is reset only at the end of __release_sock(). 2018 * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach 2019 * sk_rcvbuf in normal conditions. 2020 */ 2021 limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1; 2022 2023 limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1; 2024 2025 /* Only socket owner can try to collapse/prune rx queues 2026 * to reduce memory overhead, so add a little headroom here. 2027 * Few sockets backlog are possibly concurrently non empty. 2028 */ 2029 limit += 64 * 1024; 2030 2031 limit = min_t(u64, limit, UINT_MAX); 2032 2033 err = sk_add_backlog(sk, skb, limit); 2034 if (unlikely(err)) { 2035 bh_unlock_sock(sk); 2036 if (err == -ENOMEM) { 2037 __NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP); 2038 return SKB_DROP_REASON_PFMEMALLOC; 2039 } 2040 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP); 2041 return SKB_DROP_REASON_SOCKET_BACKLOG; 2042 } 2043 return SKB_NOT_DROPPED_YET; 2044 } 2045 2046 static void tcp_v4_restore_cb(struct sk_buff *skb) 2047 { 2048 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4, 2049 sizeof(struct inet_skb_parm)); 2050 } 2051 2052 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph, 2053 const struct tcphdr *th) 2054 { 2055 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB() 2056 * barrier() makes sure compiler wont play fool^Waliasing games. 2057 */ 2058 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb), 2059 sizeof(struct inet_skb_parm)); 2060 barrier(); 2061 2062 TCP_SKB_CB(skb)->seq = ntohl(th->seq); 2063 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + 2064 skb->len - th->doff * 4); 2065 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); 2066 TCP_SKB_CB(skb)->tcp_flags = tcp_flags_ntohs(th); 2067 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph); 2068 TCP_SKB_CB(skb)->sacked = 0; 2069 TCP_SKB_CB(skb)->has_rxtstamp = 2070 skb->tstamp || skb_hwtstamps(skb)->hwtstamp; 2071 } 2072 2073 /* 2074 * From tcp_input.c 2075 */ 2076 2077 int tcp_v4_rcv(struct sk_buff *skb) 2078 { 2079 struct net *net = dev_net_rcu(skb->dev); 2080 enum skb_drop_reason drop_reason; 2081 enum tcp_tw_status tw_status; 2082 int sdif = inet_sdif(skb); 2083 int dif = inet_iif(skb); 2084 const struct iphdr *iph; 2085 const struct tcphdr *th; 2086 struct sock *sk = NULL; 2087 bool refcounted; 2088 int ret; 2089 u32 isn; 2090 2091 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 2092 if (skb->pkt_type != PACKET_HOST) 2093 goto discard_it; 2094 2095 /* Count it even if it's bad */ 2096 __TCP_INC_STATS(net, TCP_MIB_INSEGS); 2097 2098 if (!pskb_may_pull(skb, sizeof(struct tcphdr))) 2099 goto discard_it; 2100 2101 th = (const struct tcphdr *)skb->data; 2102 2103 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) { 2104 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL; 2105 goto bad_packet; 2106 } 2107 if (!pskb_may_pull(skb, th->doff * 4)) 2108 goto discard_it; 2109 2110 /* An explanation is required here, I think. 2111 * Packet length and doff are validated by header prediction, 2112 * provided case of th->doff==0 is eliminated. 2113 * So, we defer the checks. */ 2114 2115 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo)) 2116 goto csum_error; 2117 2118 th = (const struct tcphdr *)skb->data; 2119 iph = ip_hdr(skb); 2120 lookup: 2121 sk = __inet_lookup_skb(skb, __tcp_hdrlen(th), th->source, 2122 th->dest, sdif, &refcounted); 2123 if (!sk) 2124 goto no_tcp_socket; 2125 2126 if (sk->sk_state == TCP_TIME_WAIT) 2127 goto do_time_wait; 2128 2129 if (sk->sk_state == TCP_NEW_SYN_RECV) { 2130 struct request_sock *req = inet_reqsk(sk); 2131 bool req_stolen = false; 2132 struct sock *nsk; 2133 2134 sk = req->rsk_listener; 2135 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) 2136 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 2137 else 2138 drop_reason = tcp_inbound_hash(sk, req, skb, 2139 &iph->saddr, &iph->daddr, 2140 AF_INET, dif, sdif); 2141 if (unlikely(drop_reason)) { 2142 sk_drops_skbadd(sk, skb); 2143 reqsk_put(req); 2144 goto discard_it; 2145 } 2146 if (tcp_checksum_complete(skb)) { 2147 reqsk_put(req); 2148 goto csum_error; 2149 } 2150 if (unlikely(sk->sk_state != TCP_LISTEN)) { 2151 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb); 2152 if (!nsk) { 2153 inet_csk_reqsk_queue_drop_and_put(sk, req); 2154 goto lookup; 2155 } 2156 sk = nsk; 2157 /* reuseport_migrate_sock() has already held one sk_refcnt 2158 * before returning. 2159 */ 2160 } else { 2161 /* We own a reference on the listener, increase it again 2162 * as we might lose it too soon. 2163 */ 2164 sock_hold(sk); 2165 } 2166 refcounted = true; 2167 nsk = NULL; 2168 drop_reason = tcp_filter(sk, skb); 2169 if (!drop_reason) { 2170 th = (const struct tcphdr *)skb->data; 2171 iph = ip_hdr(skb); 2172 tcp_v4_fill_cb(skb, iph, th); 2173 nsk = tcp_check_req(sk, skb, req, false, &req_stolen, 2174 &drop_reason); 2175 } 2176 if (!nsk) { 2177 reqsk_put(req); 2178 if (req_stolen) { 2179 /* Another cpu got exclusive access to req 2180 * and created a full blown socket. 2181 * Try to feed this packet to this socket 2182 * instead of discarding it. 2183 */ 2184 tcp_v4_restore_cb(skb); 2185 sock_put(sk); 2186 goto lookup; 2187 } 2188 goto discard_and_relse; 2189 } 2190 nf_reset_ct(skb); 2191 if (nsk == sk) { 2192 reqsk_put(req); 2193 tcp_v4_restore_cb(skb); 2194 } else { 2195 drop_reason = tcp_child_process(sk, nsk, skb); 2196 if (drop_reason) { 2197 enum sk_rst_reason rst_reason; 2198 2199 rst_reason = sk_rst_convert_drop_reason(drop_reason); 2200 tcp_v4_send_reset(nsk, skb, rst_reason); 2201 sock_put(nsk); 2202 goto discard_and_relse; 2203 } 2204 sock_put(nsk); 2205 sock_put(sk); 2206 return 0; 2207 } 2208 } 2209 2210 isn = 0; 2211 process: 2212 if (static_branch_unlikely(&ip4_min_ttl)) { 2213 /* min_ttl can be changed concurrently from do_ip_setsockopt() */ 2214 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) { 2215 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); 2216 drop_reason = SKB_DROP_REASON_TCP_MINTTL; 2217 goto discard_and_relse; 2218 } 2219 } 2220 2221 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) { 2222 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 2223 goto discard_and_relse; 2224 } 2225 2226 drop_reason = tcp_inbound_hash(sk, NULL, skb, &iph->saddr, &iph->daddr, 2227 AF_INET, dif, sdif); 2228 if (drop_reason) 2229 goto discard_and_relse; 2230 2231 nf_reset_ct(skb); 2232 2233 drop_reason = tcp_filter(sk, skb); 2234 if (drop_reason) 2235 goto discard_and_relse; 2236 2237 th = (const struct tcphdr *)skb->data; 2238 iph = ip_hdr(skb); 2239 tcp_v4_fill_cb(skb, iph, th); 2240 TCP_SKB_CB(skb)->tcp_tw_isn = isn; 2241 2242 skb->dev = NULL; 2243 2244 if (sk->sk_state == TCP_LISTEN) { 2245 ret = tcp_v4_do_rcv(sk, skb); 2246 goto put_and_return; 2247 } 2248 2249 sk_incoming_cpu_update(sk); 2250 2251 bh_lock_sock_nested(sk); 2252 tcp_segs_in(tcp_sk(sk), skb); 2253 ret = 0; 2254 if (!sock_owned_by_user(sk)) { 2255 ret = tcp_v4_do_rcv(sk, skb); 2256 } else { 2257 drop_reason = tcp_add_backlog(sk, skb); 2258 if (drop_reason) 2259 goto discard_and_relse; 2260 } 2261 bh_unlock_sock(sk); 2262 2263 put_and_return: 2264 if (refcounted) 2265 sock_put(sk); 2266 2267 return ret; 2268 2269 no_tcp_socket: 2270 drop_reason = SKB_DROP_REASON_NO_SOCKET; 2271 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) 2272 goto discard_it; 2273 2274 tcp_v4_fill_cb(skb, iph, th); 2275 2276 if (tcp_checksum_complete(skb)) { 2277 csum_error: 2278 drop_reason = SKB_DROP_REASON_TCP_CSUM; 2279 trace_tcp_bad_csum(skb); 2280 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS); 2281 bad_packet: 2282 __TCP_INC_STATS(net, TCP_MIB_INERRS); 2283 } else { 2284 tcp_v4_send_reset(NULL, skb, sk_rst_convert_drop_reason(drop_reason)); 2285 } 2286 2287 discard_it: 2288 SKB_DR_OR(drop_reason, NOT_SPECIFIED); 2289 /* Discard frame. */ 2290 sk_skb_reason_drop(sk, skb, drop_reason); 2291 return 0; 2292 2293 discard_and_relse: 2294 sk_drops_skbadd(sk, skb); 2295 if (refcounted) 2296 sock_put(sk); 2297 goto discard_it; 2298 2299 do_time_wait: 2300 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) { 2301 drop_reason = SKB_DROP_REASON_XFRM_POLICY; 2302 inet_twsk_put(inet_twsk(sk)); 2303 goto discard_it; 2304 } 2305 2306 tcp_v4_fill_cb(skb, iph, th); 2307 2308 if (tcp_checksum_complete(skb)) { 2309 inet_twsk_put(inet_twsk(sk)); 2310 goto csum_error; 2311 } 2312 2313 tw_status = tcp_timewait_state_process(inet_twsk(sk), skb, th, &isn, 2314 &drop_reason); 2315 switch (tw_status) { 2316 case TCP_TW_SYN: { 2317 struct sock *sk2 = inet_lookup_listener(net, skb, __tcp_hdrlen(th), 2318 iph->saddr, th->source, 2319 iph->daddr, th->dest, 2320 inet_iif(skb), 2321 sdif); 2322 if (sk2) { 2323 inet_twsk_deschedule_put(inet_twsk(sk)); 2324 sk = sk2; 2325 tcp_v4_restore_cb(skb); 2326 refcounted = false; 2327 goto process; 2328 } 2329 2330 drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb); 2331 if (drop_reason) { 2332 inet_twsk_put(inet_twsk(sk)); 2333 goto discard_it; 2334 } 2335 } 2336 /* to ACK */ 2337 fallthrough; 2338 case TCP_TW_ACK: 2339 case TCP_TW_ACK_OOW: 2340 tcp_v4_timewait_ack(sk, skb, tw_status); 2341 break; 2342 case TCP_TW_RST: 2343 tcp_v4_send_reset(sk, skb, SK_RST_REASON_TCP_TIMEWAIT_SOCKET); 2344 inet_twsk_deschedule_put(inet_twsk(sk)); 2345 goto discard_it; 2346 case TCP_TW_SUCCESS:; 2347 } 2348 goto discard_it; 2349 } 2350 2351 static struct timewait_sock_ops tcp_timewait_sock_ops = { 2352 .twsk_obj_size = sizeof(struct tcp_timewait_sock), 2353 }; 2354 2355 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) 2356 { 2357 struct dst_entry *dst = skb_dst(skb); 2358 2359 if (dst && dst_hold_safe(dst)) { 2360 rcu_assign_pointer(sk->sk_rx_dst, dst); 2361 sk->sk_rx_dst_ifindex = skb->skb_iif; 2362 } 2363 } 2364 2365 const struct inet_connection_sock_af_ops ipv4_specific = { 2366 .queue_xmit = ip_queue_xmit, 2367 .rebuild_header = inet_sk_rebuild_header, 2368 .sk_rx_dst_set = inet_sk_rx_dst_set, 2369 .conn_request = tcp_v4_conn_request, 2370 .syn_recv_sock = tcp_v4_syn_recv_sock, 2371 .net_header_len = sizeof(struct iphdr), 2372 .setsockopt = ip_setsockopt, 2373 .getsockopt = ip_getsockopt, 2374 .mtu_reduced = tcp_v4_mtu_reduced, 2375 }; 2376 2377 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2378 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = { 2379 #ifdef CONFIG_TCP_MD5SIG 2380 .md5_lookup = tcp_v4_md5_lookup, 2381 .calc_md5_hash = tcp_v4_md5_hash_skb, 2382 .md5_parse = tcp_v4_parse_md5_keys, 2383 #endif 2384 #ifdef CONFIG_TCP_AO 2385 .ao_lookup = tcp_v4_ao_lookup, 2386 .calc_ao_hash = tcp_v4_ao_hash_skb, 2387 .ao_parse = tcp_v4_parse_ao, 2388 .ao_calc_key_sk = tcp_v4_ao_calc_key_sk, 2389 #endif 2390 }; 2391 2392 static void tcp4_destruct_sock(struct sock *sk) 2393 { 2394 tcp_md5_destruct_sock(sk); 2395 tcp_ao_destroy_sock(sk, false); 2396 inet_sock_destruct(sk); 2397 } 2398 #endif 2399 2400 /* NOTE: A lot of things set to zero explicitly by call to 2401 * sk_alloc() so need not be done here. 2402 */ 2403 static int tcp_v4_init_sock(struct sock *sk) 2404 { 2405 struct inet_connection_sock *icsk = inet_csk(sk); 2406 2407 tcp_init_sock(sk); 2408 2409 icsk->icsk_af_ops = &ipv4_specific; 2410 2411 #if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO) 2412 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific; 2413 sk->sk_destruct = tcp4_destruct_sock; 2414 #endif 2415 2416 return 0; 2417 } 2418 2419 static void tcp_release_user_frags(struct sock *sk) 2420 { 2421 #ifdef CONFIG_PAGE_POOL 2422 unsigned long index; 2423 void *netmem; 2424 2425 xa_for_each(&sk->sk_user_frags, index, netmem) 2426 WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem)); 2427 #endif 2428 } 2429 2430 void tcp_v4_destroy_sock(struct sock *sk) 2431 { 2432 struct tcp_sock *tp = tcp_sk(sk); 2433 2434 tcp_release_user_frags(sk); 2435 2436 xa_destroy(&sk->sk_user_frags); 2437 2438 trace_tcp_destroy_sock(sk); 2439 2440 tcp_clear_xmit_timers(sk); 2441 2442 tcp_cleanup_congestion_control(sk); 2443 2444 tcp_cleanup_ulp(sk); 2445 2446 /* Cleanup up the write buffer. */ 2447 tcp_write_queue_purge(sk); 2448 2449 /* Check if we want to disable active TFO */ 2450 tcp_fastopen_active_disable_ofo_check(sk); 2451 2452 /* Cleans up our, hopefully empty, out_of_order_queue. */ 2453 skb_rbtree_purge(&tp->out_of_order_queue); 2454 2455 /* Clean up a referenced TCP bind bucket. */ 2456 if (inet_csk(sk)->icsk_bind_hash) 2457 inet_put_port(sk); 2458 2459 BUG_ON(rcu_access_pointer(tp->fastopen_rsk)); 2460 2461 /* If socket is aborted during connect operation */ 2462 tcp_free_fastopen_req(tp); 2463 tcp_fastopen_destroy_cipher(sk); 2464 tcp_saved_syn_free(tp); 2465 2466 sk_sockets_allocated_dec(sk); 2467 } 2468 2469 #ifdef CONFIG_PROC_FS 2470 /* Proc filesystem TCP sock list dumping. */ 2471 2472 static unsigned short seq_file_family(const struct seq_file *seq); 2473 2474 static bool seq_sk_match(struct seq_file *seq, const struct sock *sk) 2475 { 2476 unsigned short family = seq_file_family(seq); 2477 2478 /* AF_UNSPEC is used as a match all */ 2479 return ((family == AF_UNSPEC || family == sk->sk_family) && 2480 net_eq(sock_net(sk), seq_file_net(seq))); 2481 } 2482 2483 /* Find a non empty bucket (starting from st->bucket) 2484 * and return the first sk from it. 2485 */ 2486 static void *listening_get_first(struct seq_file *seq) 2487 { 2488 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2489 struct tcp_iter_state *st = seq->private; 2490 2491 st->offset = 0; 2492 for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) { 2493 struct inet_listen_hashbucket *ilb2; 2494 struct hlist_nulls_node *node; 2495 struct sock *sk; 2496 2497 ilb2 = &hinfo->lhash2[st->bucket]; 2498 if (hlist_nulls_empty(&ilb2->nulls_head)) 2499 continue; 2500 2501 spin_lock(&ilb2->lock); 2502 sk_nulls_for_each(sk, node, &ilb2->nulls_head) { 2503 if (seq_sk_match(seq, sk)) 2504 return sk; 2505 } 2506 spin_unlock(&ilb2->lock); 2507 } 2508 2509 return NULL; 2510 } 2511 2512 /* Find the next sk of "cur" within the same bucket (i.e. st->bucket). 2513 * If "cur" is the last one in the st->bucket, 2514 * call listening_get_first() to return the first sk of the next 2515 * non empty bucket. 2516 */ 2517 static void *listening_get_next(struct seq_file *seq, void *cur) 2518 { 2519 struct tcp_iter_state *st = seq->private; 2520 struct inet_listen_hashbucket *ilb2; 2521 struct hlist_nulls_node *node; 2522 struct inet_hashinfo *hinfo; 2523 struct sock *sk = cur; 2524 2525 ++st->num; 2526 ++st->offset; 2527 2528 sk = sk_nulls_next(sk); 2529 sk_nulls_for_each_from(sk, node) { 2530 if (seq_sk_match(seq, sk)) 2531 return sk; 2532 } 2533 2534 hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2535 ilb2 = &hinfo->lhash2[st->bucket]; 2536 spin_unlock(&ilb2->lock); 2537 ++st->bucket; 2538 return listening_get_first(seq); 2539 } 2540 2541 static void *listening_get_idx(struct seq_file *seq, loff_t *pos) 2542 { 2543 struct tcp_iter_state *st = seq->private; 2544 void *rc; 2545 2546 st->bucket = 0; 2547 st->offset = 0; 2548 rc = listening_get_first(seq); 2549 2550 while (rc && *pos) { 2551 rc = listening_get_next(seq, rc); 2552 --*pos; 2553 } 2554 return rc; 2555 } 2556 2557 static inline bool empty_bucket(struct inet_hashinfo *hinfo, 2558 const struct tcp_iter_state *st) 2559 { 2560 return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain); 2561 } 2562 2563 /* 2564 * Get first established socket starting from bucket given in st->bucket. 2565 * If st->bucket is zero, the very first socket in the hash is returned. 2566 */ 2567 static void *established_get_first(struct seq_file *seq) 2568 { 2569 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2570 struct tcp_iter_state *st = seq->private; 2571 2572 st->offset = 0; 2573 for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) { 2574 struct sock *sk; 2575 struct hlist_nulls_node *node; 2576 spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket); 2577 2578 cond_resched(); 2579 2580 /* Lockless fast path for the common case of empty buckets */ 2581 if (empty_bucket(hinfo, st)) 2582 continue; 2583 2584 spin_lock_bh(lock); 2585 sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) { 2586 if (seq_sk_match(seq, sk)) 2587 return sk; 2588 } 2589 spin_unlock_bh(lock); 2590 } 2591 2592 return NULL; 2593 } 2594 2595 static void *established_get_next(struct seq_file *seq, void *cur) 2596 { 2597 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2598 struct tcp_iter_state *st = seq->private; 2599 struct hlist_nulls_node *node; 2600 struct sock *sk = cur; 2601 2602 ++st->num; 2603 ++st->offset; 2604 2605 sk = sk_nulls_next(sk); 2606 2607 sk_nulls_for_each_from(sk, node) { 2608 if (seq_sk_match(seq, sk)) 2609 return sk; 2610 } 2611 2612 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); 2613 ++st->bucket; 2614 return established_get_first(seq); 2615 } 2616 2617 static void *established_get_idx(struct seq_file *seq, loff_t pos) 2618 { 2619 struct tcp_iter_state *st = seq->private; 2620 void *rc; 2621 2622 st->bucket = 0; 2623 rc = established_get_first(seq); 2624 2625 while (rc && pos) { 2626 rc = established_get_next(seq, rc); 2627 --pos; 2628 } 2629 return rc; 2630 } 2631 2632 static void *tcp_get_idx(struct seq_file *seq, loff_t pos) 2633 { 2634 void *rc; 2635 struct tcp_iter_state *st = seq->private; 2636 2637 st->state = TCP_SEQ_STATE_LISTENING; 2638 rc = listening_get_idx(seq, &pos); 2639 2640 if (!rc) { 2641 st->state = TCP_SEQ_STATE_ESTABLISHED; 2642 rc = established_get_idx(seq, pos); 2643 } 2644 2645 return rc; 2646 } 2647 2648 static void *tcp_seek_last_pos(struct seq_file *seq) 2649 { 2650 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2651 struct tcp_iter_state *st = seq->private; 2652 int bucket = st->bucket; 2653 int offset = st->offset; 2654 int orig_num = st->num; 2655 void *rc = NULL; 2656 2657 switch (st->state) { 2658 case TCP_SEQ_STATE_LISTENING: 2659 if (st->bucket > hinfo->lhash2_mask) 2660 break; 2661 rc = listening_get_first(seq); 2662 while (offset-- && rc && bucket == st->bucket) 2663 rc = listening_get_next(seq, rc); 2664 if (rc) 2665 break; 2666 st->bucket = 0; 2667 st->state = TCP_SEQ_STATE_ESTABLISHED; 2668 fallthrough; 2669 case TCP_SEQ_STATE_ESTABLISHED: 2670 if (st->bucket > hinfo->ehash_mask) 2671 break; 2672 rc = established_get_first(seq); 2673 while (offset-- && rc && bucket == st->bucket) 2674 rc = established_get_next(seq, rc); 2675 } 2676 2677 st->num = orig_num; 2678 2679 return rc; 2680 } 2681 2682 void *tcp_seq_start(struct seq_file *seq, loff_t *pos) 2683 { 2684 struct tcp_iter_state *st = seq->private; 2685 void *rc; 2686 2687 if (*pos && *pos == st->last_pos) { 2688 rc = tcp_seek_last_pos(seq); 2689 if (rc) 2690 goto out; 2691 } 2692 2693 st->state = TCP_SEQ_STATE_LISTENING; 2694 st->num = 0; 2695 st->bucket = 0; 2696 st->offset = 0; 2697 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN; 2698 2699 out: 2700 st->last_pos = *pos; 2701 return rc; 2702 } 2703 2704 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos) 2705 { 2706 struct tcp_iter_state *st = seq->private; 2707 void *rc = NULL; 2708 2709 if (v == SEQ_START_TOKEN) { 2710 rc = tcp_get_idx(seq, 0); 2711 goto out; 2712 } 2713 2714 switch (st->state) { 2715 case TCP_SEQ_STATE_LISTENING: 2716 rc = listening_get_next(seq, v); 2717 if (!rc) { 2718 st->state = TCP_SEQ_STATE_ESTABLISHED; 2719 st->bucket = 0; 2720 st->offset = 0; 2721 rc = established_get_first(seq); 2722 } 2723 break; 2724 case TCP_SEQ_STATE_ESTABLISHED: 2725 rc = established_get_next(seq, v); 2726 break; 2727 } 2728 out: 2729 ++*pos; 2730 st->last_pos = *pos; 2731 return rc; 2732 } 2733 2734 void tcp_seq_stop(struct seq_file *seq, void *v) 2735 { 2736 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2737 struct tcp_iter_state *st = seq->private; 2738 2739 switch (st->state) { 2740 case TCP_SEQ_STATE_LISTENING: 2741 if (v != SEQ_START_TOKEN) 2742 spin_unlock(&hinfo->lhash2[st->bucket].lock); 2743 break; 2744 case TCP_SEQ_STATE_ESTABLISHED: 2745 if (v) 2746 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); 2747 break; 2748 } 2749 } 2750 2751 static void get_openreq4(const struct request_sock *req, 2752 struct seq_file *f, int i) 2753 { 2754 const struct inet_request_sock *ireq = inet_rsk(req); 2755 long delta = req->rsk_timer.expires - jiffies; 2756 2757 seq_printf(f, "%4d: %08X:%04X %08X:%04X" 2758 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d 0", 2759 i, 2760 ireq->ir_loc_addr, 2761 ireq->ir_num, 2762 ireq->ir_rmt_addr, 2763 ntohs(ireq->ir_rmt_port), 2764 TCP_SYN_RECV, 2765 0, 0, /* could print option size, but that is af dependent. */ 2766 1, /* timers active (only the expire timer) */ 2767 jiffies_delta_to_clock_t(delta), 2768 req->num_timeout, 2769 from_kuid_munged(seq_user_ns(f), 2770 sk_uid(req->rsk_listener)), 2771 0, /* non standard timer */ 2772 0, /* open_requests have no inode */ 2773 0); 2774 } 2775 2776 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i) 2777 { 2778 int timer_active; 2779 unsigned long timer_expires; 2780 const struct tcp_sock *tp = tcp_sk(sk); 2781 const struct inet_connection_sock *icsk = inet_csk(sk); 2782 const struct inet_sock *inet = inet_sk(sk); 2783 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq; 2784 __be32 dest = inet->inet_daddr; 2785 __be32 src = inet->inet_rcv_saddr; 2786 __u16 destp = ntohs(inet->inet_dport); 2787 __u16 srcp = ntohs(inet->inet_sport); 2788 u8 icsk_pending; 2789 int rx_queue; 2790 int state; 2791 2792 icsk_pending = smp_load_acquire(&icsk->icsk_pending); 2793 if (icsk_pending == ICSK_TIME_RETRANS || 2794 icsk_pending == ICSK_TIME_REO_TIMEOUT || 2795 icsk_pending == ICSK_TIME_LOSS_PROBE) { 2796 timer_active = 1; 2797 timer_expires = tcp_timeout_expires(sk); 2798 } else if (icsk_pending == ICSK_TIME_PROBE0) { 2799 timer_active = 4; 2800 timer_expires = tcp_timeout_expires(sk); 2801 } else if (timer_pending(&icsk->icsk_keepalive_timer)) { 2802 timer_active = 2; 2803 timer_expires = icsk->icsk_keepalive_timer.expires; 2804 } else { 2805 timer_active = 0; 2806 timer_expires = jiffies; 2807 } 2808 2809 state = inet_sk_state_load(sk); 2810 if (state == TCP_LISTEN) 2811 rx_queue = READ_ONCE(sk->sk_ack_backlog); 2812 else 2813 /* Because we don't lock the socket, 2814 * we might find a transient negative value. 2815 */ 2816 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) - 2817 READ_ONCE(tp->copied_seq), 0); 2818 2819 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX " 2820 "%08X %5u %8d %llu %d 0 %lu %lu %u %u %d", 2821 i, src, srcp, dest, destp, state, 2822 READ_ONCE(tp->write_seq) - tp->snd_una, 2823 rx_queue, 2824 timer_active, 2825 jiffies_delta_to_clock_t(timer_expires - jiffies), 2826 READ_ONCE(icsk->icsk_retransmits), 2827 from_kuid_munged(seq_user_ns(f), sk_uid(sk)), 2828 READ_ONCE(icsk->icsk_probes_out), 2829 sock_i_ino(sk), 2830 refcount_read(&sk->sk_refcnt), 2831 jiffies_to_clock_t(icsk->icsk_rto), 2832 jiffies_to_clock_t(icsk->icsk_ack.ato), 2833 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk), 2834 tcp_snd_cwnd(tp), 2835 state == TCP_LISTEN ? 2836 fastopenq->max_qlen : 2837 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh)); 2838 } 2839 2840 static void get_timewait4_sock(const struct inet_timewait_sock *tw, 2841 struct seq_file *f, int i) 2842 { 2843 long delta = tw->tw_timer.expires - jiffies; 2844 __be32 dest, src; 2845 __u16 destp, srcp; 2846 2847 dest = tw->tw_daddr; 2848 src = tw->tw_rcv_saddr; 2849 destp = ntohs(tw->tw_dport); 2850 srcp = ntohs(tw->tw_sport); 2851 2852 seq_printf(f, "%4d: %08X:%04X %08X:%04X" 2853 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d 0", 2854 i, src, srcp, dest, destp, READ_ONCE(tw->tw_substate), 0, 0, 2855 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0, 2856 refcount_read(&tw->tw_refcnt)); 2857 } 2858 2859 #define TMPSZ 150 2860 2861 static int tcp4_seq_show(struct seq_file *seq, void *v) 2862 { 2863 struct tcp_iter_state *st; 2864 struct sock *sk = v; 2865 2866 seq_setwidth(seq, TMPSZ - 1); 2867 if (v == SEQ_START_TOKEN) { 2868 seq_puts(seq, " sl local_address rem_address st tx_queue " 2869 "rx_queue tr tm->when retrnsmt uid timeout " 2870 "inode"); 2871 goto out; 2872 } 2873 st = seq->private; 2874 2875 if (sk->sk_state == TCP_TIME_WAIT) 2876 get_timewait4_sock(v, seq, st->num); 2877 else if (sk->sk_state == TCP_NEW_SYN_RECV) 2878 get_openreq4(v, seq, st->num); 2879 else 2880 get_tcp4_sock(v, seq, st->num); 2881 out: 2882 seq_pad(seq, '\n'); 2883 return 0; 2884 } 2885 2886 #ifdef CONFIG_BPF_SYSCALL 2887 union bpf_tcp_iter_batch_item { 2888 struct sock *sk; 2889 __u64 cookie; 2890 }; 2891 2892 struct bpf_tcp_iter_state { 2893 struct tcp_iter_state state; 2894 unsigned int cur_sk; 2895 unsigned int end_sk; 2896 unsigned int max_sk; 2897 union bpf_tcp_iter_batch_item *batch; 2898 }; 2899 2900 struct bpf_iter__tcp { 2901 __bpf_md_ptr(struct bpf_iter_meta *, meta); 2902 __bpf_md_ptr(struct sock_common *, sk_common); 2903 uid_t uid __aligned(8); 2904 }; 2905 2906 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta, 2907 struct sock_common *sk_common, uid_t uid) 2908 { 2909 struct bpf_iter__tcp ctx; 2910 2911 meta->seq_num--; /* skip SEQ_START_TOKEN */ 2912 ctx.meta = meta; 2913 ctx.sk_common = sk_common; 2914 ctx.uid = uid; 2915 return bpf_iter_run_prog(prog, &ctx); 2916 } 2917 2918 static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter) 2919 { 2920 union bpf_tcp_iter_batch_item *item; 2921 unsigned int cur_sk = iter->cur_sk; 2922 __u64 cookie; 2923 2924 /* Remember the cookies of the sockets we haven't seen yet, so we can 2925 * pick up where we left off next time around. 2926 */ 2927 while (cur_sk < iter->end_sk) { 2928 item = &iter->batch[cur_sk++]; 2929 cookie = sock_gen_cookie(item->sk); 2930 sock_gen_put(item->sk); 2931 item->cookie = cookie; 2932 } 2933 } 2934 2935 static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter, 2936 unsigned int new_batch_sz, gfp_t flags) 2937 { 2938 union bpf_tcp_iter_batch_item *new_batch; 2939 2940 new_batch = kvmalloc_objs(*new_batch, new_batch_sz, 2941 flags | __GFP_NOWARN); 2942 if (!new_batch) 2943 return -ENOMEM; 2944 2945 memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk); 2946 kvfree(iter->batch); 2947 iter->batch = new_batch; 2948 iter->max_sk = new_batch_sz; 2949 2950 return 0; 2951 } 2952 2953 static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk, 2954 union bpf_tcp_iter_batch_item *cookies, 2955 int n_cookies) 2956 { 2957 struct hlist_nulls_node *node; 2958 struct sock *sk; 2959 int i; 2960 2961 for (i = 0; i < n_cookies; i++) { 2962 sk = first_sk; 2963 sk_nulls_for_each_from(sk, node) 2964 if (cookies[i].cookie == atomic64_read(&sk->sk_cookie)) 2965 return sk; 2966 } 2967 2968 return NULL; 2969 } 2970 2971 static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq) 2972 { 2973 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 2974 struct bpf_tcp_iter_state *iter = seq->private; 2975 struct tcp_iter_state *st = &iter->state; 2976 unsigned int find_cookie = iter->cur_sk; 2977 unsigned int end_cookie = iter->end_sk; 2978 int resume_bucket = st->bucket; 2979 struct sock *sk; 2980 2981 if (end_cookie && find_cookie == end_cookie) 2982 ++st->bucket; 2983 2984 sk = listening_get_first(seq); 2985 iter->cur_sk = 0; 2986 iter->end_sk = 0; 2987 2988 if (sk && st->bucket == resume_bucket && end_cookie) { 2989 sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie], 2990 end_cookie - find_cookie); 2991 if (!sk) { 2992 spin_unlock(&hinfo->lhash2[st->bucket].lock); 2993 ++st->bucket; 2994 sk = listening_get_first(seq); 2995 } 2996 } 2997 2998 return sk; 2999 } 3000 3001 static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq) 3002 { 3003 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 3004 struct bpf_tcp_iter_state *iter = seq->private; 3005 struct tcp_iter_state *st = &iter->state; 3006 unsigned int find_cookie = iter->cur_sk; 3007 unsigned int end_cookie = iter->end_sk; 3008 int resume_bucket = st->bucket; 3009 struct sock *sk; 3010 3011 if (end_cookie && find_cookie == end_cookie) 3012 ++st->bucket; 3013 3014 sk = established_get_first(seq); 3015 iter->cur_sk = 0; 3016 iter->end_sk = 0; 3017 3018 if (sk && st->bucket == resume_bucket && end_cookie) { 3019 sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie], 3020 end_cookie - find_cookie); 3021 if (!sk) { 3022 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); 3023 ++st->bucket; 3024 sk = established_get_first(seq); 3025 } 3026 } 3027 3028 return sk; 3029 } 3030 3031 static struct sock *bpf_iter_tcp_resume(struct seq_file *seq) 3032 { 3033 struct bpf_tcp_iter_state *iter = seq->private; 3034 struct tcp_iter_state *st = &iter->state; 3035 struct sock *sk = NULL; 3036 3037 switch (st->state) { 3038 case TCP_SEQ_STATE_LISTENING: 3039 sk = bpf_iter_tcp_resume_listening(seq); 3040 if (sk) 3041 break; 3042 st->bucket = 0; 3043 st->state = TCP_SEQ_STATE_ESTABLISHED; 3044 fallthrough; 3045 case TCP_SEQ_STATE_ESTABLISHED: 3046 sk = bpf_iter_tcp_resume_established(seq); 3047 break; 3048 } 3049 3050 return sk; 3051 } 3052 3053 static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq, 3054 struct sock **start_sk) 3055 { 3056 struct bpf_tcp_iter_state *iter = seq->private; 3057 struct hlist_nulls_node *node; 3058 unsigned int expected = 1; 3059 struct sock *sk; 3060 3061 sock_hold(*start_sk); 3062 iter->batch[iter->end_sk++].sk = *start_sk; 3063 3064 sk = sk_nulls_next(*start_sk); 3065 *start_sk = NULL; 3066 sk_nulls_for_each_from(sk, node) { 3067 if (seq_sk_match(seq, sk)) { 3068 if (iter->end_sk < iter->max_sk) { 3069 sock_hold(sk); 3070 iter->batch[iter->end_sk++].sk = sk; 3071 } else if (!*start_sk) { 3072 /* Remember where we left off. */ 3073 *start_sk = sk; 3074 } 3075 expected++; 3076 } 3077 } 3078 3079 return expected; 3080 } 3081 3082 static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, 3083 struct sock **start_sk) 3084 { 3085 struct bpf_tcp_iter_state *iter = seq->private; 3086 struct hlist_nulls_node *node; 3087 struct sock *sk = *start_sk; 3088 unsigned int expected = 0; 3089 3090 *start_sk = NULL; 3091 sk_nulls_for_each_from(sk, node) { 3092 if (!seq_sk_match(seq, sk)) 3093 continue; 3094 expected++; 3095 if (iter->end_sk < iter->max_sk) { 3096 /* reqsk_queue_hash_req() inserts with sk_refcnt == 0 3097 * and refcount_set()s it after the bucket lock drops. 3098 */ 3099 if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) 3100 continue; 3101 iter->batch[iter->end_sk++].sk = sk; 3102 } else if (!*start_sk) { 3103 /* Remember where we left off. */ 3104 *start_sk = sk; 3105 } 3106 } 3107 3108 return expected; 3109 } 3110 3111 static unsigned int bpf_iter_fill_batch(struct seq_file *seq, 3112 struct sock **start_sk) 3113 { 3114 struct bpf_tcp_iter_state *iter = seq->private; 3115 struct tcp_iter_state *st = &iter->state; 3116 3117 if (st->state == TCP_SEQ_STATE_LISTENING) 3118 return bpf_iter_tcp_listening_batch(seq, start_sk); 3119 else 3120 return bpf_iter_tcp_established_batch(seq, start_sk); 3121 } 3122 3123 static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq) 3124 { 3125 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; 3126 struct bpf_tcp_iter_state *iter = seq->private; 3127 struct tcp_iter_state *st = &iter->state; 3128 3129 if (st->state == TCP_SEQ_STATE_LISTENING) 3130 spin_unlock(&hinfo->lhash2[st->bucket].lock); 3131 else 3132 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); 3133 } 3134 3135 static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) 3136 { 3137 struct bpf_tcp_iter_state *iter = seq->private; 3138 unsigned int expected; 3139 struct sock *sk; 3140 int err; 3141 3142 again: 3143 sk = bpf_iter_tcp_resume(seq); 3144 if (!sk) 3145 return NULL; /* Done */ 3146 3147 expected = bpf_iter_fill_batch(seq, &sk); 3148 if (likely(!sk)) 3149 goto done; 3150 3151 /* Batch size was too small. */ 3152 bpf_iter_tcp_unlock_bucket(seq); 3153 bpf_iter_tcp_put_batch(iter); 3154 err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2, 3155 GFP_USER); 3156 if (err) { 3157 iter->cur_sk = 0; 3158 iter->end_sk = 0; 3159 return ERR_PTR(err); 3160 } 3161 3162 sk = bpf_iter_tcp_resume(seq); 3163 if (!sk) 3164 return NULL; /* Done */ 3165 3166 expected = bpf_iter_fill_batch(seq, &sk); 3167 if (likely(!sk)) 3168 goto done; 3169 3170 /* Batch size was still too small. Hold onto the lock while we try 3171 * again with a larger batch to make sure the current bucket's size 3172 * does not change in the meantime. 3173 */ 3174 err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT); 3175 if (err) { 3176 bpf_iter_tcp_unlock_bucket(seq); 3177 return ERR_PTR(err); 3178 } 3179 3180 bpf_iter_fill_batch(seq, &sk); 3181 WARN_ON_ONCE(sk); 3182 done: 3183 bpf_iter_tcp_unlock_bucket(seq); 3184 if (unlikely(!iter->end_sk)) { 3185 ++iter->state.bucket; 3186 goto again; 3187 } 3188 return iter->batch[0].sk; 3189 } 3190 3191 static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos) 3192 { 3193 /* bpf iter does not support lseek, so it always 3194 * continue from where it was stop()-ped. 3195 */ 3196 if (*pos) 3197 return bpf_iter_tcp_batch(seq); 3198 3199 return SEQ_START_TOKEN; 3200 } 3201 3202 static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3203 { 3204 struct bpf_tcp_iter_state *iter = seq->private; 3205 struct tcp_iter_state *st = &iter->state; 3206 struct sock *sk; 3207 3208 /* Whenever seq_next() is called, the iter->cur_sk is 3209 * done with seq_show(), so advance to the next sk in 3210 * the batch. 3211 */ 3212 if (iter->cur_sk < iter->end_sk) { 3213 /* Keeping st->num consistent in tcp_iter_state. 3214 * bpf_iter_tcp does not use st->num. 3215 * meta.seq_num is used instead. 3216 */ 3217 st->num++; 3218 sock_gen_put(iter->batch[iter->cur_sk++].sk); 3219 } 3220 3221 if (iter->cur_sk < iter->end_sk) 3222 sk = iter->batch[iter->cur_sk].sk; 3223 else 3224 sk = bpf_iter_tcp_batch(seq); 3225 3226 ++*pos; 3227 /* Keeping st->last_pos consistent in tcp_iter_state. 3228 * bpf iter does not do lseek, so st->last_pos always equals to *pos. 3229 */ 3230 st->last_pos = *pos; 3231 return sk; 3232 } 3233 3234 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v) 3235 { 3236 struct bpf_iter_meta meta; 3237 struct bpf_prog *prog; 3238 struct sock *sk = v; 3239 uid_t uid; 3240 int ret; 3241 3242 if (v == SEQ_START_TOKEN) 3243 return 0; 3244 3245 if (sk_fullsock(sk)) 3246 lock_sock(sk); 3247 3248 if (unlikely(sk_unhashed(sk))) { 3249 ret = SEQ_SKIP; 3250 goto unlock; 3251 } 3252 3253 if (sk->sk_state == TCP_TIME_WAIT) { 3254 uid = 0; 3255 } else if (sk->sk_state == TCP_NEW_SYN_RECV) { 3256 const struct request_sock *req = v; 3257 3258 uid = from_kuid_munged(seq_user_ns(seq), 3259 sk_uid(req->rsk_listener)); 3260 } else { 3261 uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk)); 3262 } 3263 3264 meta.seq = seq; 3265 prog = bpf_iter_get_info(&meta, false); 3266 ret = tcp_prog_seq_show(prog, &meta, v, uid); 3267 3268 unlock: 3269 if (sk_fullsock(sk)) 3270 release_sock(sk); 3271 return ret; 3272 3273 } 3274 3275 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v) 3276 { 3277 struct bpf_tcp_iter_state *iter = seq->private; 3278 struct bpf_iter_meta meta; 3279 struct bpf_prog *prog; 3280 3281 if (!v) { 3282 meta.seq = seq; 3283 prog = bpf_iter_get_info(&meta, true); 3284 if (prog) 3285 (void)tcp_prog_seq_show(prog, &meta, v, 0); 3286 } 3287 3288 if (iter->cur_sk < iter->end_sk) 3289 bpf_iter_tcp_put_batch(iter); 3290 } 3291 3292 static const struct seq_operations bpf_iter_tcp_seq_ops = { 3293 .show = bpf_iter_tcp_seq_show, 3294 .start = bpf_iter_tcp_seq_start, 3295 .next = bpf_iter_tcp_seq_next, 3296 .stop = bpf_iter_tcp_seq_stop, 3297 }; 3298 #endif 3299 static unsigned short seq_file_family(const struct seq_file *seq) 3300 { 3301 const struct tcp_seq_afinfo *afinfo; 3302 3303 #ifdef CONFIG_BPF_SYSCALL 3304 /* Iterated from bpf_iter. Let the bpf prog to filter instead. */ 3305 if (seq->op == &bpf_iter_tcp_seq_ops) 3306 return AF_UNSPEC; 3307 #endif 3308 3309 /* Iterated from proc fs */ 3310 afinfo = pde_data(file_inode(seq->file)); 3311 return afinfo->family; 3312 } 3313 3314 static const struct seq_operations tcp4_seq_ops = { 3315 .show = tcp4_seq_show, 3316 .start = tcp_seq_start, 3317 .next = tcp_seq_next, 3318 .stop = tcp_seq_stop, 3319 }; 3320 3321 static struct tcp_seq_afinfo tcp4_seq_afinfo = { 3322 .family = AF_INET, 3323 }; 3324 3325 static int __net_init tcp4_proc_init_net(struct net *net) 3326 { 3327 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops, 3328 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo)) 3329 return -ENOMEM; 3330 return 0; 3331 } 3332 3333 static void __net_exit tcp4_proc_exit_net(struct net *net) 3334 { 3335 remove_proc_entry("tcp", net->proc_net); 3336 } 3337 3338 static struct pernet_operations tcp4_net_ops = { 3339 .init = tcp4_proc_init_net, 3340 .exit = tcp4_proc_exit_net, 3341 }; 3342 3343 int __init tcp4_proc_init(void) 3344 { 3345 return register_pernet_subsys(&tcp4_net_ops); 3346 } 3347 3348 void tcp4_proc_exit(void) 3349 { 3350 unregister_pernet_subsys(&tcp4_net_ops); 3351 } 3352 #endif /* CONFIG_PROC_FS */ 3353 3354 struct proto tcp_prot = { 3355 .name = "TCP", 3356 .owner = THIS_MODULE, 3357 .close = tcp_close, 3358 .pre_connect = tcp_v4_pre_connect, 3359 .connect = tcp_v4_connect, 3360 .disconnect = tcp_disconnect, 3361 .accept = inet_csk_accept, 3362 .ioctl = tcp_ioctl, 3363 .init = tcp_v4_init_sock, 3364 .destroy = tcp_v4_destroy_sock, 3365 .shutdown = tcp_shutdown, 3366 .setsockopt = tcp_setsockopt, 3367 .getsockopt = tcp_getsockopt, 3368 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt, 3369 .keepalive = tcp_set_keepalive, 3370 .recvmsg = tcp_recvmsg, 3371 .sendmsg = tcp_sendmsg, 3372 .splice_eof = tcp_splice_eof, 3373 .backlog_rcv = tcp_v4_do_rcv, 3374 .release_cb = tcp_release_cb, 3375 .hash = inet_hash, 3376 .unhash = inet_unhash, 3377 .get_port = inet_csk_get_port, 3378 .put_port = inet_put_port, 3379 #ifdef CONFIG_BPF_SYSCALL 3380 .psock_update_sk_prot = tcp_bpf_update_proto, 3381 #endif 3382 .enter_memory_pressure = tcp_enter_memory_pressure, 3383 .leave_memory_pressure = tcp_leave_memory_pressure, 3384 .stream_memory_free = tcp_stream_memory_free, 3385 .sockets_allocated = &tcp_sockets_allocated, 3386 3387 .memory_allocated = &net_aligned_data.tcp_memory_allocated, 3388 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 3389 3390 .memory_pressure = &tcp_memory_pressure, 3391 .sysctl_mem = sysctl_tcp_mem, 3392 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3393 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3394 .max_header = MAX_TCP_HEADER, 3395 .obj_size = sizeof(struct tcp_sock), 3396 .freeptr_offset = offsetof(struct tcp_sock, 3397 inet_conn.icsk_inet.sk.sk_freeptr), 3398 .slab_flags = SLAB_TYPESAFE_BY_RCU, 3399 .twsk_prot = &tcp_timewait_sock_ops, 3400 .rsk_prot = &tcp_request_sock_ops, 3401 .h.hashinfo = NULL, 3402 .no_autobind = true, 3403 .diag_destroy = tcp_abort, 3404 }; 3405 EXPORT_SYMBOL(tcp_prot); 3406 3407 static void __net_exit tcp_sk_exit(struct net *net) 3408 { 3409 if (net->ipv4.tcp_congestion_control) 3410 bpf_module_put(net->ipv4.tcp_congestion_control, 3411 net->ipv4.tcp_congestion_control->owner); 3412 } 3413 3414 static void __net_init tcp_set_hashinfo(struct net *net) 3415 { 3416 struct inet_hashinfo *hinfo; 3417 unsigned int ehash_entries; 3418 struct net *old_net; 3419 3420 if (net_eq(net, &init_net)) 3421 goto fallback; 3422 3423 old_net = current->nsproxy->net_ns; 3424 ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries); 3425 if (!ehash_entries) 3426 goto fallback; 3427 3428 ehash_entries = roundup_pow_of_two(ehash_entries); 3429 hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries); 3430 if (!hinfo) { 3431 pr_warn("Failed to allocate TCP ehash (entries: %u) " 3432 "for a netns, fallback to the global one\n", 3433 ehash_entries); 3434 fallback: 3435 hinfo = &tcp_hashinfo; 3436 ehash_entries = tcp_hashinfo.ehash_mask + 1; 3437 } 3438 3439 net->ipv4.tcp_death_row.hashinfo = hinfo; 3440 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2; 3441 net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128); 3442 } 3443 3444 static int __net_init tcp_sk_init(struct net *net) 3445 { 3446 net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN; 3447 net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL; 3448 net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON; 3449 net->ipv4.sysctl_tcp_ecn_fallback = 1; 3450 3451 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS; 3452 net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS; 3453 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD; 3454 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL; 3455 net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS; 3456 3457 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME; 3458 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES; 3459 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL; 3460 3461 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES; 3462 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES; 3463 net->ipv4.sysctl_tcp_syncookies = 1; 3464 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH; 3465 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1; 3466 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2; 3467 net->ipv4.sysctl_tcp_orphan_retries = 0; 3468 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT; 3469 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX; 3470 net->ipv4.sysctl_tcp_tw_reuse = 2; 3471 net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC; 3472 net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1; 3473 3474 refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1); 3475 tcp_set_hashinfo(net); 3476 3477 net->ipv4.sysctl_tcp_sack = 1; 3478 net->ipv4.sysctl_tcp_window_scaling = 1; 3479 net->ipv4.sysctl_tcp_timestamps = 1; 3480 net->ipv4.sysctl_tcp_early_retrans = 3; 3481 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION; 3482 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */ 3483 net->ipv4.sysctl_tcp_retrans_collapse = 1; 3484 net->ipv4.sysctl_tcp_max_reordering = 300; 3485 net->ipv4.sysctl_tcp_dsack = 1; 3486 net->ipv4.sysctl_tcp_app_win = 31; 3487 net->ipv4.sysctl_tcp_adv_win_scale = 1; 3488 net->ipv4.sysctl_tcp_frto = 2; 3489 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1; 3490 net->ipv4.sysctl_tcp_rcvbuf_low_rtt = USEC_PER_MSEC; 3491 /* This limits the percentage of the congestion window which we 3492 * will allow a single TSO frame to consume. Building TSO frames 3493 * which are too large can cause TCP streams to be bursty. 3494 */ 3495 net->ipv4.sysctl_tcp_tso_win_divisor = 3; 3496 /* Default TSQ limit of 4 MB */ 3497 net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20; 3498 3499 /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */ 3500 net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX; 3501 3502 net->ipv4.sysctl_tcp_min_tso_segs = 2; 3503 net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */ 3504 net->ipv4.sysctl_tcp_min_rtt_wlen = 300; 3505 net->ipv4.sysctl_tcp_autocorking = 1; 3506 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2; 3507 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200; 3508 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120; 3509 if (net != &init_net) { 3510 memcpy(net->ipv4.sysctl_tcp_rmem, 3511 init_net.ipv4.sysctl_tcp_rmem, 3512 sizeof(init_net.ipv4.sysctl_tcp_rmem)); 3513 memcpy(net->ipv4.sysctl_tcp_wmem, 3514 init_net.ipv4.sysctl_tcp_wmem, 3515 sizeof(init_net.ipv4.sysctl_tcp_wmem)); 3516 } 3517 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC; 3518 net->ipv4.sysctl_tcp_comp_sack_slack_ns = 10 * NSEC_PER_USEC; 3519 net->ipv4.sysctl_tcp_comp_sack_nr = 44; 3520 net->ipv4.sysctl_tcp_comp_sack_rtt_percent = 33; 3521 net->ipv4.sysctl_tcp_backlog_ack_defer = 1; 3522 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE; 3523 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0; 3524 atomic_set(&net->ipv4.tfo_active_disable_times, 0); 3525 3526 /* Set default values for PLB */ 3527 net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */ 3528 net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3; 3529 net->ipv4.sysctl_tcp_plb_rehash_rounds = 12; 3530 net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60; 3531 /* Default congestion threshold for PLB to mark a round is 50% */ 3532 net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2; 3533 3534 /* Reno is always built in */ 3535 if (!net_eq(net, &init_net) && 3536 bpf_try_module_get(init_net.ipv4.tcp_congestion_control, 3537 init_net.ipv4.tcp_congestion_control->owner)) 3538 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control; 3539 else 3540 net->ipv4.tcp_congestion_control = &tcp_reno; 3541 3542 net->ipv4.sysctl_tcp_syn_linear_timeouts = 4; 3543 net->ipv4.sysctl_tcp_shrink_window = 0; 3544 3545 net->ipv4.sysctl_tcp_pingpong_thresh = 1; 3546 net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN); 3547 net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC; 3548 3549 return 0; 3550 } 3551 3552 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list) 3553 { 3554 bool synced = false; 3555 struct net *net; 3556 3557 /* make sure concurrent calls to tcp_sk_exit_batch from net_cleanup_work 3558 * and failed setup_net error unwinding path are serialized. 3559 * 3560 * tcp_twsk_purge() handles twsk in any dead netns, not just those in 3561 * net_exit_list, the thread that dismantles a particular twsk must 3562 * do so without other thread progressing to refcount_dec_and_test() of 3563 * tcp_death_row.tw_refcount. 3564 */ 3565 mutex_lock(&tcp_exit_batch_mutex); 3566 3567 tcp_twsk_purge(net_exit_list); 3568 3569 /* inet_twsk_kill() unhashes a twsk before it is done with ehash 3570 * and bhash, and before it drops its tw_refcount reference, 3571 * so tcp_twsk_purge() can miss it. Such callers run with BH 3572 * disabled: wait for them before freeing a private ehash. 3573 */ 3574 list_for_each_entry(net, net_exit_list, exit_list) { 3575 if (net->ipv4.tcp_death_row.hashinfo->pernet) { 3576 synchronize_net(); 3577 synced = true; 3578 break; 3579 } 3580 } 3581 3582 list_for_each_entry(net, net_exit_list, exit_list) { 3583 inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo); 3584 /* Without a grace period, a concurrent inet_twsk_kill() might 3585 * still hold a tw_refcount reference. Let it drop it: struct net 3586 * is freed by a later cleanup_net(), after an RCU grace period. 3587 */ 3588 if (!refcount_dec_if_one(&net->ipv4.tcp_death_row.tw_refcount)) 3589 WARN_ON_ONCE(synced); 3590 tcp_fastopen_ctx_destroy(net); 3591 } 3592 3593 mutex_unlock(&tcp_exit_batch_mutex); 3594 } 3595 3596 static struct pernet_operations __net_initdata tcp_sk_ops = { 3597 .init = tcp_sk_init, 3598 .exit = tcp_sk_exit, 3599 .exit_batch = tcp_sk_exit_batch, 3600 }; 3601 3602 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 3603 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta, 3604 struct sock_common *sk_common, uid_t uid) 3605 3606 #define INIT_BATCH_SZ 16 3607 3608 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux) 3609 { 3610 struct bpf_tcp_iter_state *iter = priv_data; 3611 int err; 3612 3613 err = bpf_iter_init_seq_net(priv_data, aux); 3614 if (err) 3615 return err; 3616 3617 err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER); 3618 if (err) { 3619 bpf_iter_fini_seq_net(priv_data); 3620 return err; 3621 } 3622 3623 return 0; 3624 } 3625 3626 static void bpf_iter_fini_tcp(void *priv_data) 3627 { 3628 struct bpf_tcp_iter_state *iter = priv_data; 3629 3630 bpf_iter_fini_seq_net(priv_data); 3631 kvfree(iter->batch); 3632 } 3633 3634 static const struct bpf_iter_seq_info tcp_seq_info = { 3635 .seq_ops = &bpf_iter_tcp_seq_ops, 3636 .init_seq_private = bpf_iter_init_tcp, 3637 .fini_seq_private = bpf_iter_fini_tcp, 3638 .seq_priv_size = sizeof(struct bpf_tcp_iter_state), 3639 }; 3640 3641 static const struct bpf_func_proto * 3642 bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id, 3643 const struct bpf_prog *prog) 3644 { 3645 switch (func_id) { 3646 case BPF_FUNC_setsockopt: 3647 return &bpf_sk_setsockopt_proto; 3648 case BPF_FUNC_getsockopt: 3649 return &bpf_sk_getsockopt_proto; 3650 default: 3651 return NULL; 3652 } 3653 } 3654 3655 static struct bpf_iter_reg tcp_reg_info = { 3656 .target = "tcp", 3657 .ctx_arg_info_size = 1, 3658 .ctx_arg_info = { 3659 { offsetof(struct bpf_iter__tcp, sk_common), 3660 PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, 3661 }, 3662 .get_func_proto = bpf_iter_tcp_get_func_proto, 3663 .seq_info = &tcp_seq_info, 3664 }; 3665 3666 static void __init bpf_iter_register(void) 3667 { 3668 tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON]; 3669 if (bpf_iter_reg_target(&tcp_reg_info)) 3670 pr_warn("Warning: could not register bpf iterator tcp\n"); 3671 } 3672 3673 #endif 3674 3675 void __init tcp_v4_init(void) 3676 { 3677 int cpu, res; 3678 3679 for_each_possible_cpu(cpu) { 3680 struct sock *sk; 3681 3682 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW, 3683 IPPROTO_TCP, &init_net); 3684 if (res) 3685 panic("Failed to create the TCP control socket.\n"); 3686 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE); 3687 3688 /* Please enforce IP_DF and IPID==0 for RST and 3689 * ACK sent in SYN-RECV and TIME-WAIT state. 3690 */ 3691 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO; 3692 3693 sk->sk_clockid = CLOCK_MONOTONIC; 3694 3695 per_cpu(ipv4_tcp_sk.sock, cpu) = sk; 3696 } 3697 if (register_pernet_subsys(&tcp_sk_ops)) 3698 panic("Failed to create the TCP control socket.\n"); 3699 3700 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) 3701 bpf_iter_register(); 3702 #endif 3703 } 3704