1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * Definitions for the TCP module. 8 * 9 * Version: @(#)tcp.h 1.0.5 05/23/93 10 * 11 * Authors: Ross Biro 12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 13 */ 14 #ifndef _TCP_H 15 #define _TCP_H 16 17 #define FASTRETRANS_DEBUG 1 18 19 #include <linux/list.h> 20 #include <linux/tcp.h> 21 #include <linux/bug.h> 22 #include <linux/slab.h> 23 #include <linux/cache.h> 24 #include <linux/percpu.h> 25 #include <linux/skbuff.h> 26 #include <linux/kref.h> 27 #include <linux/ktime.h> 28 #include <linux/indirect_call_wrapper.h> 29 #include <linux/bits.h> 30 31 #include <net/inet_connection_sock.h> 32 #include <net/inet_timewait_sock.h> 33 #include <net/inet_hashtables.h> 34 #include <net/checksum.h> 35 #include <net/request_sock.h> 36 #include <net/sock_reuseport.h> 37 #include <net/sock.h> 38 #include <net/snmp.h> 39 #include <net/ip.h> 40 #include <net/tcp_states.h> 41 #include <net/tcp_ao.h> 42 #include <net/inet_ecn.h> 43 #include <net/dst.h> 44 #include <net/mptcp.h> 45 #include <net/xfrm.h> 46 #include <net/secure_seq.h> 47 48 #include <linux/seq_file.h> 49 #include <linux/memcontrol.h> 50 #include <linux/bpf-cgroup.h> 51 #include <linux/siphash.h> 52 53 extern struct inet_hashinfo tcp_hashinfo; 54 55 DECLARE_PER_CPU(unsigned int, tcp_orphan_count); 56 int tcp_orphan_count_sum(void); 57 58 static inline void tcp_orphan_count_inc(void) 59 { 60 this_cpu_inc(tcp_orphan_count); 61 } 62 63 static inline void tcp_orphan_count_dec(void) 64 { 65 this_cpu_dec(tcp_orphan_count); 66 } 67 68 void tcp_time_wait(struct sock *sk, int state, int timeo); 69 70 #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) 71 #define MAX_TCP_OPTION_SPACE 40 72 #define TCP_MIN_SND_MSS 48 73 #define TCP_MIN_GSO_SIZE (TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE) 74 75 /* 76 * Never offer a window over 32767 without using window scaling. Some 77 * poor stacks do signed 16bit maths! 78 */ 79 #define MAX_TCP_WINDOW 32767U 80 81 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */ 82 #define TCP_MIN_MSS 88U 83 84 /* The initial MTU to use for probing */ 85 #define TCP_BASE_MSS 1024 86 87 /* probing interval, default to 10 minutes as per RFC4821 */ 88 #define TCP_PROBE_INTERVAL 600 89 90 /* Specify interval when tcp mtu probing will stop */ 91 #define TCP_PROBE_THRESHOLD 8 92 93 /* After receiving this amount of duplicate ACKs fast retransmit starts. */ 94 #define TCP_FASTRETRANS_THRESH 3 95 96 /* Maximal number of ACKs sent quickly to accelerate slow-start. */ 97 #define TCP_MAX_QUICKACKS 16U 98 99 /* Maximal number of window scale according to RFC1323 */ 100 #define TCP_MAX_WSCALE 14U 101 102 /* Default sending frequency of accurate ECN option per RTT */ 103 #define TCP_ACCECN_OPTION_BEACON 3 104 105 /* urg_data states */ 106 #define TCP_URG_VALID 0x0100 107 #define TCP_URG_NOTYET 0x0200 108 #define TCP_URG_READ 0x0400 109 110 #define TCP_RETR1 3 /* 111 * This is how many retries it does before it 112 * tries to figure out if the gateway is 113 * down. Minimal RFC value is 3; it corresponds 114 * to ~3sec-8min depending on RTO. 115 */ 116 117 #define TCP_RETR2 15 /* 118 * This should take at least 119 * 90 minutes to time out. 120 * RFC1122 says that the limit is 100 sec. 121 * 15 is ~13-30min depending on RTO. 122 */ 123 124 #define TCP_SYN_RETRIES 6 /* This is how many retries are done 125 * when active opening a connection. 126 * RFC1122 says the minimum retry MUST 127 * be at least 180secs. Nevertheless 128 * this value is corresponding to 129 * 63secs of retransmission with the 130 * current initial RTO. 131 */ 132 133 #define TCP_SYNACK_RETRIES 5 /* This is how may retries are done 134 * when passive opening a connection. 135 * This is corresponding to 31secs of 136 * retransmission with the current 137 * initial RTO. 138 */ 139 140 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 141 * state, about 60 seconds */ 142 #define TCP_FIN_TIMEOUT TCP_TIMEWAIT_LEN 143 /* BSD style FIN_WAIT2 deadlock breaker. 144 * It used to be 3min, new value is 60sec, 145 * to combine FIN-WAIT-2 timeout with 146 * TIME-WAIT timer. 147 */ 148 #define TCP_FIN_TIMEOUT_MAX (120 * HZ) /* max TCP_LINGER2 value (two minutes) */ 149 150 #define TCP_DELACK_MAX ((unsigned)(HZ/5)) /* maximal time to delay before sending an ACK */ 151 static_assert((1 << ATO_BITS) > TCP_DELACK_MAX); 152 153 #if HZ >= 100 154 #define TCP_DELACK_MIN ((unsigned)(HZ/25)) /* minimal time to delay before sending an ACK */ 155 #define TCP_ATO_MIN ((unsigned)(HZ/25)) 156 #else 157 #define TCP_DELACK_MIN 4U 158 #define TCP_ATO_MIN 4U 159 #endif 160 #define TCP_RTO_MAX_SEC 120 161 #define TCP_RTO_MAX ((unsigned)(TCP_RTO_MAX_SEC * HZ)) 162 #define TCP_RTO_MIN ((unsigned)(HZ / 5)) 163 #define TCP_TIMEOUT_MIN (2U) /* Min timeout for TCP timers in jiffies */ 164 165 #define TCP_TIMEOUT_MIN_US (2*USEC_PER_MSEC) /* Min TCP timeout in microsecs */ 166 167 /* Max RTT sample for tcp_rtt_estimator(). srtt_us holds 8 times the RTT, and 168 * mdev_us and rttvar_us up to 4 times its deviation, all in u32, like the sum 169 * in __tcp_set_rto(). A larger sample could not raise the RTO, which 170 * TCP_RTO_MAX bounds. 171 */ 172 #define TCP_RTT_MAX_US (TCP_RTO_MAX_SEC * USEC_PER_SEC) 173 174 #define TCP_TIMEOUT_INIT ((unsigned)(1*HZ)) /* RFC6298 2.1 initial RTO value */ 175 #define TCP_TIMEOUT_FALLBACK ((unsigned)(3*HZ)) /* RFC 1122 initial RTO value, now 176 * used as a fallback RTO for the 177 * initial data transmission if no 178 * valid RTT sample has been acquired, 179 * most likely due to retrans in 3WHS. 180 */ 181 182 #define TCP_RESOURCE_PROBE_INTERVAL ((unsigned)(HZ/2U)) /* Maximal interval between probes 183 * for local resources. 184 */ 185 #define TCP_KEEPALIVE_TIME (120*60*HZ) /* two hours */ 186 #define TCP_KEEPALIVE_PROBES 9 /* Max of 9 keepalive probes */ 187 #define TCP_KEEPALIVE_INTVL (75*HZ) 188 189 #define MAX_TCP_KEEPIDLE 32767 190 #define MAX_TCP_KEEPINTVL 32767 191 #define MAX_TCP_KEEPCNT 127 192 #define MAX_TCP_SYNCNT 127 193 194 /* Ensure that TCP PAWS checks are relaxed after ~2147 seconds 195 * to avoid overflows. This assumes a clock smaller than 1 Mhz. 196 * Default clock is 1 Khz, tcp_usec_ts uses 1 Mhz. 197 */ 198 #define TCP_PAWS_WRAP (INT_MAX / USEC_PER_SEC) 199 200 #define TCP_PAWS_MSL 60 /* Per-host timestamps are invalidated 201 * after this time. It should be equal 202 * (or greater than) TCP_TIMEWAIT_LEN 203 * to provide reliability equal to one 204 * provided by timewait state. 205 */ 206 #define TCP_PAWS_WINDOW 1 /* Replay window for per-host 207 * timestamps. It must be less than 208 * minimal timewait lifetime. 209 */ 210 /* 211 * TCP option 212 */ 213 214 #define TCPOPT_NOP 1 /* Padding */ 215 #define TCPOPT_EOL 0 /* End of options */ 216 #define TCPOPT_MSS 2 /* Segment size negotiating */ 217 #define TCPOPT_WINDOW 3 /* Window scaling */ 218 #define TCPOPT_SACK_PERM 4 /* SACK Permitted */ 219 #define TCPOPT_SACK 5 /* SACK Block */ 220 #define TCPOPT_TIMESTAMP 8 /* Better RTT estimations/PAWS */ 221 #define TCPOPT_MD5SIG 19 /* MD5 Signature (RFC2385) */ 222 #define TCPOPT_AO 29 /* Authentication Option (RFC5925) */ 223 #define TCPOPT_MPTCP 30 /* Multipath TCP (RFC6824) */ 224 #define TCPOPT_FASTOPEN 34 /* Fast open (RFC7413) */ 225 #define TCPOPT_ACCECN0 172 /* 0xAC: Accurate ECN Order 0 */ 226 #define TCPOPT_ACCECN1 174 /* 0xAE: Accurate ECN Order 1 */ 227 #define TCPOPT_EXP 254 /* Experimental */ 228 /* Magic number to be after the option value for sharing TCP 229 * experimental options. See draft-ietf-tcpm-experimental-options-00.txt 230 */ 231 #define TCPOPT_FASTOPEN_MAGIC 0xF989 232 #define TCPOPT_SMC_MAGIC 0xE2D4C3D9 233 234 /* 235 * TCP option lengths 236 */ 237 238 #define TCPOLEN_MSS 4 239 #define TCPOLEN_WINDOW 3 240 #define TCPOLEN_SACK_PERM 2 241 #define TCPOLEN_TIMESTAMP 10 242 #define TCPOLEN_MD5SIG 18 243 #define TCPOLEN_FASTOPEN_BASE 2 244 #define TCPOLEN_ACCECN_BASE 2 245 #define TCPOLEN_EXP_FASTOPEN_BASE 4 246 #define TCPOLEN_EXP_SMC_BASE 6 247 248 /* But this is what stacks really send out. */ 249 #define TCPOLEN_TSTAMP_ALIGNED 12 250 #define TCPOLEN_WSCALE_ALIGNED 4 251 #define TCPOLEN_SACKPERM_ALIGNED 4 252 #define TCPOLEN_SACK_BASE 2 253 #define TCPOLEN_SACK_BASE_ALIGNED 4 254 #define TCPOLEN_SACK_PERBLOCK 8 255 #define TCPOLEN_MD5SIG_ALIGNED 20 256 #define TCPOLEN_MSS_ALIGNED 4 257 #define TCPOLEN_EXP_SMC_BASE_ALIGNED 8 258 #define TCPOLEN_ACCECN_PERFIELD 3 259 260 /* Maximum number of byte counters in AccECN option + size */ 261 #define TCP_ACCECN_NUMFIELDS 3 262 #define TCP_ACCECN_MAXSIZE (TCPOLEN_ACCECN_BASE + \ 263 TCPOLEN_ACCECN_PERFIELD * \ 264 TCP_ACCECN_NUMFIELDS) 265 #define TCP_ACCECN_SAFETY_SHIFT 1 /* SAFETY_FACTOR in accecn draft */ 266 267 /* Flags in tp->nonagle */ 268 #define TCP_NAGLE_OFF 1 /* Nagle's algo is disabled */ 269 #define TCP_NAGLE_CORK 2 /* Socket is corked */ 270 #define TCP_NAGLE_PUSH 4 /* Cork is overridden for already queued data */ 271 272 /* TCP thin-stream limits */ 273 #define TCP_THIN_LINEAR_RETRIES 6 /* After 6 linear retries, do exp. backoff */ 274 275 /* TCP initial congestion window as per rfc6928 */ 276 #define TCP_INIT_CWND 10 277 278 /* Bit Flags for sysctl_tcp_fastopen */ 279 #define TFO_CLIENT_ENABLE 1 280 #define TFO_SERVER_ENABLE 2 281 #define TFO_CLIENT_NO_COOKIE 4 /* Data in SYN w/o cookie option */ 282 283 /* Accept SYN data w/o any cookie option */ 284 #define TFO_SERVER_COOKIE_NOT_REQD 0x200 285 286 /* Force enable TFO on all listeners, i.e., not requiring the 287 * TCP_FASTOPEN socket option. 288 */ 289 #define TFO_SERVER_WO_SOCKOPT1 0x400 290 291 292 /* sysctl variables for tcp */ 293 extern int sysctl_tcp_max_orphans; 294 extern long sysctl_tcp_mem[3]; 295 296 #define TCP_RACK_LOSS_DETECTION 0x1 /* Use RACK to detect losses */ 297 #define TCP_RACK_STATIC_REO_WND 0x2 /* Use static RACK reo wnd */ 298 #define TCP_RACK_NO_DUPTHRESH 0x4 /* Do not use DUPACK threshold in RACK */ 299 300 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc); 301 302 extern struct percpu_counter tcp_sockets_allocated; 303 extern unsigned long tcp_memory_pressure; 304 305 /* optimized version of sk_under_memory_pressure() for TCP sockets */ 306 static inline bool tcp_under_memory_pressure(const struct sock *sk) 307 { 308 if (mem_cgroup_sk_enabled(sk) && 309 mem_cgroup_sk_under_memory_pressure(sk)) 310 return true; 311 312 if (sk->sk_bypass_prot_mem) 313 return false; 314 315 return READ_ONCE(tcp_memory_pressure); 316 } 317 /* 318 * The next routines deal with comparing 32 bit unsigned ints 319 * and worry about wraparound (automatic with unsigned arithmetic). 320 */ 321 322 static inline bool before(__u32 seq1, __u32 seq2) 323 { 324 return (__s32)(seq1-seq2) < 0; 325 } 326 #define after(seq2, seq1) before(seq1, seq2) 327 328 /* is s2<=s1<=s3 ? */ 329 static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3) 330 { 331 return seq3 - seq2 >= seq1 - seq2; 332 } 333 334 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) 335 { 336 sk_wmem_queued_add(sk, -skb->truesize); 337 if (!skb_zcopy_pure(skb)) 338 sk_mem_uncharge(sk, skb->truesize); 339 else 340 sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); 341 __kfree_skb(skb); 342 } 343 344 void sk_forced_mem_schedule(struct sock *sk, int size); 345 346 bool tcp_check_oom(const struct sock *sk, int shift); 347 348 349 extern struct proto tcp_prot; 350 351 #define TCP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.tcp_statistics, field) 352 #define __TCP_INC_STATS(net, field) __SNMP_INC_STATS((net)->mib.tcp_statistics, field) 353 #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) 354 #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) 355 356 /* 357 * TCP splice context 358 */ 359 struct tcp_splice_state { 360 struct pipe_inode_info *pipe; 361 size_t len; 362 unsigned int flags; 363 }; 364 365 void tcp_tsq_work_init(void); 366 367 int tcp_v4_err(struct sk_buff *skb, u32); 368 369 void tcp_shutdown(struct sock *sk, int how); 370 371 int tcp_v4_rcv(struct sk_buff *skb); 372 373 void tcp_remove_empty_skb(struct sock *sk); 374 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); 375 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); 376 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, 377 size_t size, struct ubuf_info *uarg); 378 void tcp_splice_eof(struct socket *sock); 379 int tcp_send_mss(struct sock *sk, int *size_goal, int flags); 380 int tcp_wmem_schedule(struct sock *sk, int copy); 381 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, 382 int size_goal); 383 384 void tcp_release_cb(struct sock *sk); 385 386 static inline bool tcp_release_cb_cond(struct sock *sk) 387 { 388 #ifdef CONFIG_INET 389 if (likely(sk->sk_prot->release_cb == tcp_release_cb)) { 390 if (unlikely(smp_load_acquire(&sk->sk_tsq_flags) & TCP_DEFERRED_ALL)) 391 tcp_release_cb(sk); 392 return true; 393 } 394 #endif 395 return false; 396 } 397 398 void tcp_write_timer_handler(struct sock *sk); 399 void tcp_delack_timer_handler(struct sock *sk); 400 int tcp_ioctl(struct sock *sk, int cmd, int *karg); 401 enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb); 402 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); 403 void tcp_rcvbuf_grow(struct sock *sk, u32 newval); 404 void tcp_rcv_space_adjust(struct sock *sk); 405 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); 406 void tcp_twsk_destructor(struct sock *sk); 407 void tcp_twsk_purge(struct list_head *net_exit_list); 408 int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 409 unsigned int offset, size_t len); 410 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, 411 struct pipe_inode_info *pipe, size_t len, 412 unsigned int flags); 413 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp, 414 bool force_schedule); 415 416 static inline void tcp_dec_quickack_mode(struct sock *sk) 417 { 418 struct inet_connection_sock *icsk = inet_csk(sk); 419 420 if (icsk->icsk_ack.quick) { 421 /* How many ACKs S/ACKing new data have we sent? */ 422 const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0; 423 424 if (pkts >= icsk->icsk_ack.quick) { 425 icsk->icsk_ack.quick = 0; 426 /* Leaving quickack mode we deflate ATO. */ 427 icsk->icsk_ack.ato = TCP_ATO_MIN; 428 } else 429 icsk->icsk_ack.quick -= pkts; 430 } 431 } 432 433 #define TCP_ECN_MODE_RFC3168 BIT(0) 434 #define TCP_ECN_QUEUE_CWR BIT(1) 435 #define TCP_ECN_DEMAND_CWR BIT(2) 436 #define TCP_ECN_SEEN BIT(3) 437 #define TCP_ECN_MODE_ACCECN BIT(4) 438 439 #define TCP_ECN_DISABLED 0 440 #define TCP_ECN_MODE_PENDING (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) 441 #define TCP_ECN_MODE_ANY (TCP_ECN_MODE_RFC3168 | TCP_ECN_MODE_ACCECN) 442 443 static inline bool tcp_ecn_mode_any(const struct tcp_sock *tp) 444 { 445 return tp->ecn_flags & TCP_ECN_MODE_ANY; 446 } 447 448 static inline bool tcp_ecn_mode_rfc3168(const struct tcp_sock *tp) 449 { 450 return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_RFC3168; 451 } 452 453 static inline bool tcp_ecn_mode_accecn(const struct tcp_sock *tp) 454 { 455 return (tp->ecn_flags & TCP_ECN_MODE_ANY) == TCP_ECN_MODE_ACCECN; 456 } 457 458 static inline bool tcp_ecn_disabled(const struct tcp_sock *tp) 459 { 460 return !tcp_ecn_mode_any(tp); 461 } 462 463 static inline bool tcp_ecn_mode_pending(const struct tcp_sock *tp) 464 { 465 return (tp->ecn_flags & TCP_ECN_MODE_PENDING) == TCP_ECN_MODE_PENDING; 466 } 467 468 static inline void tcp_ecn_mode_set(struct tcp_sock *tp, u8 mode) 469 { 470 tp->ecn_flags &= ~TCP_ECN_MODE_ANY; 471 tp->ecn_flags |= mode; 472 } 473 474 enum tcp_tw_status { 475 TCP_TW_SUCCESS = 0, 476 TCP_TW_RST = 1, 477 TCP_TW_ACK = 2, 478 TCP_TW_SYN = 3, 479 TCP_TW_ACK_OOW = 4 480 }; 481 482 483 enum tcp_tw_status tcp_timewait_state_process(struct inet_timewait_sock *tw, 484 struct sk_buff *skb, 485 const struct tcphdr *th, 486 u32 *tw_isn, 487 enum skb_drop_reason *drop_reason); 488 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, 489 struct request_sock *req, bool fastopen, 490 bool *lost_race, enum skb_drop_reason *drop_reason); 491 enum skb_drop_reason tcp_child_process(struct sock *parent, struct sock *child, 492 struct sk_buff *skb); 493 void tcp_enter_loss(struct sock *sk); 494 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag); 495 void tcp_clear_retrans(struct tcp_sock *tp); 496 void tcp_update_pacing_rate(struct sock *sk); 497 void tcp_set_rto(struct sock *sk); 498 void tcp_update_metrics(struct sock *sk); 499 void tcp_init_metrics(struct sock *sk); 500 void tcp_metrics_init(void); 501 bool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst); 502 void __tcp_close(struct sock *sk, long timeout); 503 void tcp_close(struct sock *sk, long timeout); 504 void tcp_init_sock(struct sock *sk); 505 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); 506 __poll_t tcp_poll(struct file *file, struct socket *sock, 507 struct poll_table_struct *wait); 508 int do_tcp_getsockopt(struct sock *sk, int level, 509 int optname, sockptr_t optval, sockptr_t optlen); 510 int tcp_getsockopt(struct sock *sk, int level, int optname, 511 char __user *optval, int __user *optlen); 512 bool tcp_bpf_bypass_getsockopt(int level, int optname); 513 int do_tcp_setsockopt(struct sock *sk, int level, int optname, 514 sockptr_t optval, unsigned int optlen); 515 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, 516 unsigned int optlen); 517 void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout); 518 void tcp_set_keepalive(struct sock *sk, int val); 519 void tcp_syn_ack_timeout(const struct request_sock *req); 520 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 521 int flags); 522 int __tcp_set_rcvlowat(struct sock *sk, int val, bool wakeup); 523 int tcp_set_rcvlowat(struct sock *sk, int val); 524 void tcp_set_rcvbuf(struct sock *sk, int val); 525 int tcp_set_window_clamp(struct sock *sk, int val); 526 527 static inline void 528 tcp_update_recv_tstamps(struct sk_buff *skb, 529 struct scm_timestamping_internal *tss) 530 { 531 tss->ts[0] = skb->tstamp; 532 tss->ts[2] = skb_hwtstamps(skb)->hwtstamp; 533 } 534 535 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, 536 struct scm_timestamping_internal *tss); 537 void tcp_data_ready(struct sock *sk); 538 #ifdef CONFIG_MMU 539 int tcp_mmap(struct file *file, struct socket *sock, 540 struct vm_area_struct *vma); 541 #endif 542 void tcp_parse_options(const struct net *net, const struct sk_buff *skb, 543 struct tcp_options_received *opt_rx, 544 int estab, struct tcp_fastopen_cookie *foc); 545 546 /* 547 * BPF SKB-less helpers 548 */ 549 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph, 550 struct tcphdr *th, u32 *cookie); 551 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph, 552 struct tcphdr *th, u32 *cookie); 553 u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss); 554 u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, 555 const struct tcp_request_sock_ops *af_ops, 556 struct sock *sk, struct tcphdr *th); 557 /* 558 * TCP v4 functions exported for the inet6 API 559 */ 560 561 void tcp_v4_mtu_reduced(struct sock *sk); 562 void tcp_req_err(struct sock *sk, u32 seq, bool abort); 563 void tcp_ld_RTO_revert(struct sock *sk, u32 seq); 564 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb); 565 struct sock *tcp_create_openreq_child(const struct sock *sk, 566 struct request_sock *req, 567 struct sk_buff *skb); 568 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst); 569 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, 570 struct request_sock *req, 571 struct dst_entry *dst, 572 struct request_sock *req_unhash, 573 bool *own_req, 574 void (*opt_child_init)(struct sock *newsk, 575 const struct sock *sk)); 576 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); 577 int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); 578 int tcp_connect(struct sock *sk); 579 enum tcp_synack_type { 580 TCP_SYNACK_NORMAL, 581 TCP_SYNACK_FASTOPEN, 582 TCP_SYNACK_COOKIE, 583 TCP_SYNACK_RETRANS, 584 }; 585 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, 586 struct request_sock *req, 587 struct tcp_fastopen_cookie *foc, 588 enum tcp_synack_type synack_type, 589 struct sk_buff *syn_skb); 590 int tcp_disconnect(struct sock *sk, int flags); 591 592 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb); 593 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size); 594 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb); 595 596 /* From syncookies.c */ 597 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, 598 struct request_sock *req, 599 struct dst_entry *dst); 600 int __cookie_v4_check(const struct iphdr *iph, const struct tcphdr *th); 601 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb); 602 struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, 603 struct sock *sk, struct sk_buff *skb, 604 struct tcp_options_received *tcp_opt, 605 int mss, u32 tsoff); 606 607 #if IS_ENABLED(CONFIG_BPF) 608 struct bpf_tcp_req_attrs { 609 u32 rcv_tsval; 610 u32 rcv_tsecr; 611 u16 mss; 612 u8 rcv_wscale; 613 u8 snd_wscale; 614 u8 ecn_ok; 615 u8 wscale_ok; 616 u8 sack_ok; 617 u8 tstamp_ok; 618 u8 usec_ts_ok; 619 u8 reserved[3]; 620 }; 621 #endif 622 623 #ifdef CONFIG_SYN_COOKIES 624 625 /* Syncookies use a monotonic timer which increments every 60 seconds. 626 * This counter is used both as a hash input and partially encoded into 627 * the cookie value. A cookie is only validated further if the delta 628 * between the current counter value and the encoded one is less than this, 629 * i.e. a sent cookie is valid only at most for 2*60 seconds (or less if 630 * the counter advances immediately after a cookie is generated). 631 */ 632 #define MAX_SYNCOOKIE_AGE 2 633 #define TCP_SYNCOOKIE_PERIOD (60 * HZ) 634 #define TCP_SYNCOOKIE_VALID (MAX_SYNCOOKIE_AGE * TCP_SYNCOOKIE_PERIOD) 635 636 /* syncookies: remember time of last synqueue overflow 637 * But do not dirty this field too often (once per second is enough) 638 * It is racy as we do not hold a lock, but race is very minor. 639 */ 640 static inline void tcp_synq_overflow(const struct sock *sk) 641 { 642 unsigned int last_overflow; 643 unsigned int now = jiffies; 644 645 if (sk->sk_reuseport) { 646 struct sock_reuseport *reuse; 647 648 reuse = rcu_dereference(sk->sk_reuseport_cb); 649 if (likely(reuse)) { 650 last_overflow = READ_ONCE(reuse->synq_overflow_ts); 651 if (!time_between32(now, last_overflow, 652 last_overflow + HZ)) 653 WRITE_ONCE(reuse->synq_overflow_ts, now); 654 return; 655 } 656 } 657 658 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 659 if (!time_between32(now, last_overflow, last_overflow + HZ)) 660 WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); 661 } 662 663 /* syncookies: no recent synqueue overflow on this listening socket? */ 664 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) 665 { 666 unsigned int last_overflow; 667 unsigned int now = jiffies; 668 669 if (sk->sk_reuseport) { 670 struct sock_reuseport *reuse; 671 672 reuse = rcu_dereference(sk->sk_reuseport_cb); 673 if (likely(reuse)) { 674 last_overflow = READ_ONCE(reuse->synq_overflow_ts); 675 return !time_between32(now, last_overflow - HZ, 676 last_overflow + 677 TCP_SYNCOOKIE_VALID); 678 } 679 } 680 681 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); 682 683 /* If last_overflow <= jiffies <= last_overflow + TCP_SYNCOOKIE_VALID, 684 * then we're under synflood. However, we have to use 685 * 'last_overflow - HZ' as lower bound. That's because a concurrent 686 * tcp_synq_overflow() could update .ts_recent_stamp after we read 687 * jiffies but before we store .ts_recent_stamp into last_overflow, 688 * which could lead to rejecting a valid syncookie. 689 */ 690 return !time_between32(now, last_overflow - HZ, 691 last_overflow + TCP_SYNCOOKIE_VALID); 692 } 693 694 static inline u32 tcp_cookie_time(void) 695 { 696 u64 val = get_jiffies_64(); 697 698 do_div(val, TCP_SYNCOOKIE_PERIOD); 699 return val; 700 } 701 702 /* Convert one nsec 64bit timestamp to ts (ms or usec resolution) */ 703 static inline u64 tcp_ns_to_ts(bool usec_ts, u64 val) 704 { 705 if (usec_ts) 706 return div_u64(val, NSEC_PER_USEC); 707 708 return div_u64(val, NSEC_PER_MSEC); 709 } 710 711 u32 __cookie_v4_init_sequence(const struct iphdr *iph, const struct tcphdr *th, 712 u16 *mssp); 713 __u32 cookie_v4_init_sequence(const struct sk_buff *skb, __u16 *mss); 714 u64 cookie_init_timestamp(struct request_sock *req, u64 now); 715 bool cookie_timestamp_decode(const struct net *net, 716 struct tcp_options_received *opt); 717 718 static inline bool cookie_ecn_ok(const struct net *net, const struct dst_entry *dst) 719 { 720 return READ_ONCE(net->ipv4.sysctl_tcp_ecn) || 721 dst_feature(dst, RTAX_FEATURE_ECN); 722 } 723 724 #if IS_ENABLED(CONFIG_BPF) 725 static inline bool cookie_bpf_ok(struct sk_buff *skb) 726 { 727 return skb->sk; 728 } 729 730 struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb); 731 #else 732 static inline bool cookie_bpf_ok(struct sk_buff *skb) 733 { 734 return false; 735 } 736 737 static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk, 738 struct sk_buff *skb) 739 { 740 return NULL; 741 } 742 #endif 743 744 /* From net/ipv6/syncookies.c */ 745 int __cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th); 746 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb); 747 748 u32 __cookie_v6_init_sequence(const struct ipv6hdr *iph, 749 const struct tcphdr *th, u16 *mssp); 750 __u32 cookie_v6_init_sequence(const struct sk_buff *skb, __u16 *mss); 751 #endif 752 /* tcp_output.c */ 753 754 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb); 755 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb); 756 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, 757 int nonagle); 758 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 759 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); 760 void tcp_retransmit_timer(struct sock *sk); 761 void tcp_xmit_retransmit_queue(struct sock *); 762 void tcp_simple_retransmit(struct sock *); 763 void tcp_enter_recovery(struct sock *sk, bool ece_ack); 764 int tcp_trim_head(struct sock *, struct sk_buff *, u32); 765 enum tcp_queue { 766 TCP_FRAG_IN_WRITE_QUEUE, 767 TCP_FRAG_IN_RTX_QUEUE, 768 }; 769 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, 770 struct sk_buff *skb, u32 len, 771 unsigned int mss_now, gfp_t gfp); 772 773 void tcp_send_probe0(struct sock *); 774 int tcp_write_wakeup(struct sock *, int mib); 775 void tcp_send_fin(struct sock *sk); 776 void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason); 777 int tcp_send_synack(struct sock *); 778 void tcp_push_one(struct sock *, unsigned int mss_now); 779 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags); 780 void tcp_send_ack(struct sock *sk); 781 void tcp_send_delayed_ack(struct sock *sk); 782 void tcp_send_loss_probe(struct sock *sk); 783 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto); 784 void tcp_skb_collapse_tstamp(struct sk_buff *skb, 785 const struct sk_buff *next_skb); 786 787 /* tcp_input.c */ 788 void tcp_rearm_rto(struct sock *sk); 789 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); 790 void tcp_done_with_error(struct sock *sk, int err); 791 void tcp_reset(struct sock *sk, struct sk_buff *skb); 792 void tcp_fin(struct sock *sk); 793 void __tcp_check_space(struct sock *sk); 794 795 /* Mirror of SOCK_NOSPACE in tcp_sock, maintained by sk_set_nospace() 796 * and sk_clear_nospace(). 797 * 798 * MPTCP subflows share the parent socket, and thus its SOCK_NOSPACE bit. 799 * Keep their mirror always set (see subflow_ulp_init()) so that they 800 * always reach __tcp_check_space() and behave as before. 801 */ 802 static inline void tcp_set_nospace(struct sock *sk) 803 { 804 if (sk_is_tcp(sk)) { 805 /* pairs with smp_mb__before_atomic() in tcp_clear_nospace() */ 806 smp_mb__after_atomic(); 807 /* pairs with smp_mb() in tcp_check_space() */ 808 smp_store_mb(tcp_sk(sk)->tcp_nospace, 1); 809 } 810 } 811 812 static inline void tcp_clear_nospace(struct sock *sk) 813 { 814 if (sk_is_tcp(sk) && !sk_is_mptcp(sk)) { 815 WRITE_ONCE(tcp_sk(sk)->tcp_nospace, 0); 816 /* pairs with smp_mb__after_atomic() in tcp_set_nospace() */ 817 smp_mb__before_atomic(); 818 } 819 } 820 821 static inline void tcp_check_space(struct sock *sk) 822 { 823 /* pairs with tcp_poll() */ 824 smp_mb(); 825 826 if (unlikely(READ_ONCE(tcp_sk(sk)->tcp_nospace))) 827 __tcp_check_space(sk); 828 } 829 830 void tcp_sack_compress_send_ack(struct sock *sk); 831 832 static inline void tcp_cleanup_skb(struct sk_buff *skb) 833 { 834 skb_dst_drop(skb); 835 secpath_reset(skb); 836 } 837 838 static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb) 839 { 840 DEBUG_NET_WARN_ON_ONCE(skb_dst(skb)); 841 DEBUG_NET_WARN_ON_ONCE(secpath_exists(skb)); 842 __skb_queue_tail(&sk->sk_receive_queue, skb); 843 } 844 845 /* tcp_timer.c */ 846 void tcp_init_xmit_timers(struct sock *); 847 static inline void tcp_clear_xmit_timers(struct sock *sk) 848 { 849 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) 850 __sock_put(sk); 851 852 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) 853 __sock_put(sk); 854 855 inet_csk_clear_xmit_timers(sk); 856 } 857 858 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu); 859 unsigned int tcp_current_mss(struct sock *sk); 860 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when); 861 862 u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, 863 int min_tso_segs); 864 865 /* Bound MSS / TSO packet size with the half of the window */ 866 static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) 867 { 868 int cutoff; 869 870 /* When peer uses tiny windows, there is no use in packetizing 871 * to sub-MSS pieces for the sake of SWS or making sure there 872 * are enough packets in the pipe for fast recovery. 873 * 874 * On the other hand, for extremely large MSS devices, handling 875 * smaller than MSS windows in this way does make sense. 876 */ 877 if (tp->max_window > TCP_MSS_DEFAULT) 878 cutoff = (tp->max_window >> 1); 879 else 880 cutoff = tp->max_window; 881 882 if (cutoff && pktsize > cutoff) 883 return max_t(int, cutoff, 68U - tp->tcp_header_len); 884 else 885 return pktsize; 886 } 887 888 /* tcp.c */ 889 void tcp_get_info(struct sock *, struct tcp_info *); 890 void tcp_rate_check_app_limited(struct sock *sk); 891 892 /* Read 'sendfile()'-style from a TCP socket */ 893 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 894 sk_read_actor_t recv_actor); 895 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc, 896 sk_read_actor_t recv_actor, bool noack, 897 u32 *copied_seq); 898 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); 899 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off); 900 void tcp_read_done(struct sock *sk, size_t len); 901 902 void tcp_initialize_rcv_mss(struct sock *sk); 903 904 int tcp_mtu_to_mss(struct sock *sk, int pmtu); 905 int tcp_mss_to_mtu(struct sock *sk, int mss); 906 void tcp_mtup_init(struct sock *sk); 907 908 static inline unsigned int tcp_rto_max(const struct sock *sk) 909 { 910 return READ_ONCE(inet_csk(sk)->icsk_rto_max); 911 } 912 913 static inline void tcp_bound_rto(struct sock *sk) 914 { 915 inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk)); 916 } 917 918 static inline u32 __tcp_set_rto(const struct tcp_sock *tp) 919 { 920 u32 rto = usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us); 921 922 return max(rto, 1U); 923 } 924 925 static inline unsigned long tcp_reqsk_timeout(struct request_sock *req) 926 { 927 u64 timeout = (u64)req->timeout << req->num_timeout; 928 929 return (unsigned long)min_t(u64, timeout, 930 tcp_rto_max(req->rsk_listener)); 931 } 932 933 u32 tcp_delack_max(const struct sock *sk); 934 935 /* Compute the actual rto_min value */ 936 static inline u32 tcp_rto_min(const struct sock *sk) 937 { 938 const struct dst_entry *dst = __sk_dst_get(sk); 939 u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min); 940 941 if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) 942 rto_min = clamp(dst_metric_rtt(dst, RTAX_RTO_MIN), 943 1, TCP_RTO_MAX); 944 return rto_min; 945 } 946 947 static inline u32 tcp_rto_min_us(const struct sock *sk) 948 { 949 return jiffies_to_usecs(tcp_rto_min(sk)); 950 } 951 952 static inline bool tcp_ca_dst_locked(const struct dst_entry *dst) 953 { 954 return dst_metric_locked(dst, RTAX_CC_ALGO); 955 } 956 957 /* Minimum RTT in usec. ~0 means not available. */ 958 static inline u32 tcp_min_rtt(const struct tcp_sock *tp) 959 { 960 return minmax_get(&tp->rtt_min); 961 } 962 963 /* Compute the actual receive window we are currently advertising. 964 * Rcv_nxt can be after the window if our peer push more data 965 * than the offered window. 966 */ 967 static inline u32 tcp_receive_window(const struct tcp_sock *tp) 968 { 969 s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt; 970 971 if (win < 0) 972 win = 0; 973 return (u32) win; 974 } 975 976 /* Compute the maximum receive window we ever advertised. 977 * Rcv_nxt can be after the window if our peer push more data 978 * than the offered window. 979 */ 980 static inline u32 tcp_max_receive_window(const struct tcp_sock *tp) 981 { 982 s32 win = tp->rcv_mwnd_seq - tp->rcv_nxt; 983 984 if (win < 0) 985 win = 0; 986 return (u32) win; 987 } 988 989 /* Check if we need to update the maximum receive window sequence number */ 990 static inline void tcp_update_max_rcv_wnd_seq(struct tcp_sock *tp) 991 { 992 u32 wre = tp->rcv_wup + tp->rcv_wnd; 993 994 if (after(wre, tp->rcv_mwnd_seq)) 995 tp->rcv_mwnd_seq = wre; 996 } 997 998 /* Choose a new window, without checks for shrinking, and without 999 * scaling applied to the result. The caller does these things 1000 * if necessary. This is a "raw" window selection. 1001 */ 1002 u32 __tcp_select_window(struct sock *sk); 1003 1004 void tcp_send_window_probe(struct sock *sk); 1005 1006 /* TCP uses 32bit jiffies to save some space. 1007 * Note that this is different from tcp_time_stamp, which 1008 * historically has been the same until linux-4.13. 1009 */ 1010 #define tcp_jiffies32 ((u32)jiffies) 1011 1012 /* 1013 * Deliver a 32bit value for TCP timestamp option (RFC 7323) 1014 * It is no longer tied to jiffies, but to 1 ms clock. 1015 * Note: double check if you want to use tcp_jiffies32 instead of this. 1016 */ 1017 #define TCP_TS_HZ 1000 1018 1019 static inline u64 tcp_clock_ns(void) 1020 { 1021 return ktime_get_ns(); 1022 } 1023 1024 static inline u64 tcp_clock_us(void) 1025 { 1026 return div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1027 } 1028 1029 static inline u64 tcp_clock_ms(void) 1030 { 1031 return div_u64(tcp_clock_ns(), NSEC_PER_MSEC); 1032 } 1033 1034 /* TCP Timestamp included in TS option (RFC 1323) can either use ms 1035 * or usec resolution. Each socket carries a flag to select one or other 1036 * resolution, as the route attribute could change anytime. 1037 * Each flow must stick to initial resolution. 1038 */ 1039 static inline u32 tcp_clock_ts(bool usec_ts) 1040 { 1041 return usec_ts ? tcp_clock_us() : tcp_clock_ms(); 1042 } 1043 1044 static inline u32 tcp_time_stamp_ms(const struct tcp_sock *tp) 1045 { 1046 return div_u64(tp->tcp_mstamp, USEC_PER_MSEC); 1047 } 1048 1049 static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp) 1050 { 1051 if (tp->tcp_usec_ts) 1052 return tp->tcp_mstamp; 1053 return tcp_time_stamp_ms(tp); 1054 } 1055 1056 /* Refresh clocks of a TCP socket, 1057 * ensuring monotically increasing values. 1058 */ 1059 static inline void tcp_mstamp_refresh_inline(struct tcp_sock *tp) 1060 { 1061 u64 val = tcp_clock_ns(); 1062 1063 tp->tcp_clock_cache = val; 1064 tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC); 1065 } 1066 void tcp_mstamp_refresh(struct tcp_sock *tp); 1067 1068 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) 1069 { 1070 return max_t(s64, t1 - t0, 0); 1071 } 1072 1073 /* provide the departure time in us unit */ 1074 static inline u64 tcp_skb_timestamp_us(const struct sk_buff *skb) 1075 { 1076 return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC); 1077 } 1078 1079 /* Provide skb TSval in usec or ms unit */ 1080 static inline u32 tcp_skb_timestamp_ts(bool usec_ts, const struct sk_buff *skb) 1081 { 1082 if (usec_ts) 1083 return tcp_skb_timestamp_us(skb); 1084 1085 return div_u64(skb->skb_mstamp_ns, NSEC_PER_MSEC); 1086 } 1087 1088 static inline u32 tcp_tw_tsval(const struct tcp_timewait_sock *tcptw) 1089 { 1090 return tcp_clock_ts(tcptw->tw_sk.tw_usec_ts) + tcptw->tw_ts_offset; 1091 } 1092 1093 static inline u32 tcp_rsk_tsval(const struct tcp_request_sock *treq) 1094 { 1095 return tcp_clock_ts(treq->req_usec_ts) + treq->ts_off; 1096 } 1097 1098 #define tcp_flag_byte(th) (((u_int8_t *)th)[13]) 1099 1100 #define TCPHDR_FIN BIT(0) 1101 #define TCPHDR_SYN BIT(1) 1102 #define TCPHDR_RST BIT(2) 1103 #define TCPHDR_PSH BIT(3) 1104 #define TCPHDR_ACK BIT(4) 1105 #define TCPHDR_URG BIT(5) 1106 #define TCPHDR_ECE BIT(6) 1107 #define TCPHDR_CWR BIT(7) 1108 #define TCPHDR_AE BIT(8) 1109 #define TCPHDR_FLAGS_MASK (TCPHDR_FIN | TCPHDR_SYN | TCPHDR_RST | \ 1110 TCPHDR_PSH | TCPHDR_ACK | TCPHDR_URG | \ 1111 TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE) 1112 #define tcp_flags_ntohs(th) (ntohs(*(__be16 *)&tcp_flag_word(th)) & \ 1113 TCPHDR_FLAGS_MASK) 1114 1115 #define TCPHDR_ACE (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE) 1116 #define TCPHDR_SYN_ECN (TCPHDR_SYN | TCPHDR_ECE | TCPHDR_CWR) 1117 #define TCPHDR_SYNACK_ACCECN (TCPHDR_SYN | TCPHDR_ACK | TCPHDR_CWR) 1118 1119 #define TCP_ACCECN_CEP_ACE_MASK 0x7 1120 #define TCP_ACCECN_ACE_MAX_DELTA 6 1121 1122 /* To avoid/detect middlebox interference, not all counters start at 0. 1123 * See draft-ietf-tcpm-accurate-ecn for the latest values. 1124 */ 1125 #define TCP_ACCECN_CEP_INIT_OFFSET 5 1126 #define TCP_ACCECN_E1B_INIT_OFFSET 1 1127 #define TCP_ACCECN_E0B_INIT_OFFSET 1 1128 #define TCP_ACCECN_CEB_INIT_OFFSET 0 1129 1130 /* State flags for sacked in struct tcp_skb_cb */ 1131 enum tcp_skb_cb_sacked_flags { 1132 TCPCB_SACKED_ACKED = (1 << 0), /* SKB ACK'd by a SACK block */ 1133 TCPCB_SACKED_RETRANS = (1 << 1), /* SKB retransmitted */ 1134 TCPCB_LOST = (1 << 2), /* SKB is lost */ 1135 TCPCB_TAGBITS = (TCPCB_SACKED_ACKED | TCPCB_SACKED_RETRANS | 1136 TCPCB_LOST), /* All tag bits */ 1137 TCPCB_REPAIRED = (1 << 4), /* SKB repaired (no skb_mstamp_ns) */ 1138 TCPCB_EVER_RETRANS = (1 << 7), /* Ever retransmitted frame */ 1139 TCPCB_RETRANS = (TCPCB_SACKED_RETRANS | TCPCB_EVER_RETRANS | 1140 TCPCB_REPAIRED), 1141 }; 1142 1143 /* This is what the send packet queuing engine uses to pass 1144 * TCP per-packet control information to the transmission code. 1145 * We also store the host-order sequence numbers in here too. 1146 * This is 44 bytes if IPV6 is enabled. 1147 * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately. 1148 */ 1149 struct tcp_skb_cb { 1150 __u32 seq; /* Starting sequence number */ 1151 __u32 end_seq; /* SEQ + FIN + SYN + datalen */ 1152 union { 1153 /* Notes : 1154 * tcp_tw_isn is used in input path only 1155 * (isn chosen by tcp_timewait_state_process()) 1156 * tcp_gso_segs/size are used in write queue only, 1157 * cf tcp_skb_pcount()/tcp_skb_mss() 1158 */ 1159 u32 tcp_tw_isn; 1160 struct { 1161 u16 tcp_gso_segs; 1162 u16 tcp_gso_size; 1163 }; 1164 }; 1165 __u16 tcp_flags; /* TCP header flags (tcp[12-13])*/ 1166 1167 __u8 sacked; /* State flags for SACK. */ 1168 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */ 1169 #define TSTAMP_ACK_SK 0x1 1170 #define TSTAMP_ACK_BPF 0x2 1171 __u8 txstamp_ack:2, /* Record TX timestamp for ack? */ 1172 eor:1, /* Is skb MSG_EOR marked? */ 1173 has_rxtstamp:1, /* SKB has a RX timestamp */ 1174 unused:4; 1175 __u32 ack_seq; /* Sequence number ACK'd */ 1176 union { 1177 struct { 1178 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1) 1179 /* There is space for up to 24 bytes */ 1180 __u32 is_app_limited:1, /* cwnd not fully used? */ 1181 delivered_ce:20, 1182 unused:11; 1183 /* pkts S/ACKed so far upon tx of skb, incl retrans: */ 1184 __u32 delivered; 1185 /* start of send pipeline phase */ 1186 u64 first_tx_mstamp; 1187 /* when we reached the "delivered" count */ 1188 u64 delivered_mstamp; 1189 } tx; /* only used for outgoing skbs */ 1190 union { 1191 struct inet_skb_parm h4; 1192 #if IS_ENABLED(CONFIG_IPV6) 1193 struct inet6_skb_parm h6; 1194 #endif 1195 } header; /* For incoming skbs */ 1196 }; 1197 }; 1198 1199 #define TCP_SKB_CB(__skb) ((struct tcp_skb_cb *)&((__skb)->cb[0])) 1200 1201 extern const struct inet_connection_sock_af_ops ipv4_specific; 1202 1203 #if IS_ENABLED(CONFIG_IPV6) 1204 /* This is the variant of inet6_iif() that must be used by TCP, 1205 * as TCP moves IP6CB into a different location in skb->cb[] 1206 */ 1207 static inline int tcp_v6_iif(const struct sk_buff *skb) 1208 { 1209 return TCP_SKB_CB(skb)->header.h6.iif; 1210 } 1211 1212 static inline int tcp_v6_iif_l3_slave(const struct sk_buff *skb) 1213 { 1214 bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); 1215 1216 return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; 1217 } 1218 1219 /* TCP_SKB_CB reference means this can not be used from early demux */ 1220 static inline int tcp_v6_sdif(const struct sk_buff *skb) 1221 { 1222 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 1223 if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags)) 1224 return TCP_SKB_CB(skb)->header.h6.iif; 1225 #endif 1226 return 0; 1227 } 1228 1229 extern const struct inet_connection_sock_af_ops ipv6_specific; 1230 1231 INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); 1232 1233 #endif 1234 1235 /* TCP_SKB_CB reference means this can not be used from early demux */ 1236 static inline int tcp_v4_sdif(struct sk_buff *skb) 1237 { 1238 #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) 1239 if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) 1240 return TCP_SKB_CB(skb)->header.h4.iif; 1241 #endif 1242 return 0; 1243 } 1244 1245 /* Due to TSO, an SKB can be composed of multiple actual 1246 * packets. To keep these tracked properly, we use this. 1247 */ 1248 static inline int tcp_skb_pcount(const struct sk_buff *skb) 1249 { 1250 return TCP_SKB_CB(skb)->tcp_gso_segs; 1251 } 1252 1253 static inline void tcp_skb_pcount_set(struct sk_buff *skb, int segs) 1254 { 1255 TCP_SKB_CB(skb)->tcp_gso_segs = segs; 1256 } 1257 1258 static inline void tcp_skb_pcount_add(struct sk_buff *skb, int segs) 1259 { 1260 TCP_SKB_CB(skb)->tcp_gso_segs += segs; 1261 } 1262 1263 /* This is valid iff skb is in write queue and tcp_skb_pcount() > 1. */ 1264 static inline int tcp_skb_mss(const struct sk_buff *skb) 1265 { 1266 return TCP_SKB_CB(skb)->tcp_gso_size; 1267 } 1268 1269 static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) 1270 { 1271 return likely(!TCP_SKB_CB(skb)->eor); 1272 } 1273 1274 static inline bool tcp_skb_can_collapse(const struct sk_buff *to, 1275 const struct sk_buff *from) 1276 { 1277 return likely(tcp_skb_can_collapse_to(to) && 1278 mptcp_skb_can_collapse(to, from) && 1279 !skb_cmp_decrypted(to, from) && 1280 skb_pure_zcopy_same(to, from) && 1281 skb_frags_readable(to) == skb_frags_readable(from)); 1282 } 1283 1284 static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to, 1285 const struct sk_buff *from) 1286 { 1287 return likely(mptcp_skb_can_collapse(to, from) && 1288 !skb_cmp_decrypted(to, from)); 1289 } 1290 1291 /* Events passed to congestion control interface */ 1292 enum tcp_ca_event { 1293 CA_EVENT_TX_START, /* first transmit when no packets in flight */ 1294 CA_EVENT_CWND_RESTART, /* congestion window restart */ 1295 CA_EVENT_COMPLETE_CWR, /* end of congestion recovery */ 1296 CA_EVENT_LOSS, /* loss timeout */ 1297 CA_EVENT_ECN_NO_CE, /* ECT set, but not CE marked */ 1298 CA_EVENT_ECN_IS_CE, /* received CE marked IP packet */ 1299 }; 1300 1301 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */ 1302 enum tcp_ca_ack_event_flags { 1303 CA_ACK_SLOWPATH = (1 << 0), /* In slow path processing */ 1304 CA_ACK_WIN_UPDATE = (1 << 1), /* ACK updated window */ 1305 CA_ACK_ECE = (1 << 2), /* ECE bit is set on ack */ 1306 }; 1307 1308 /* 1309 * Interface for adding new TCP congestion control handlers 1310 */ 1311 #define TCP_CA_NAME_MAX 16 1312 #define TCP_CA_MAX 128 1313 #define TCP_CA_BUF_MAX (TCP_CA_NAME_MAX*TCP_CA_MAX) 1314 1315 #define TCP_CA_UNSPEC 0 1316 1317 /* Algorithm can be set on socket without CAP_NET_ADMIN privileges */ 1318 #define TCP_CONG_NON_RESTRICTED BIT(0) 1319 /* Requires ECN/ECT set on all packets */ 1320 #define TCP_CONG_NEEDS_ECN BIT(1) 1321 /* Require successfully negotiated AccECN capability */ 1322 #define TCP_CONG_NEEDS_ACCECN BIT(2) 1323 /* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ 1324 #define TCP_CONG_ECT_1_NEGOTIATION BIT(3) 1325 /* Cannot fallback to RFC3168 during AccECN negotiation */ 1326 #define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) 1327 #define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ 1328 TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ 1329 TCP_CONG_NO_FALLBACK_RFC3168) 1330 1331 union tcp_cc_info; 1332 1333 struct ack_sample { 1334 u32 pkts_acked; 1335 s32 rtt_us; 1336 u32 in_flight; 1337 }; 1338 1339 /* A rate sample measures the number of (original/retransmitted) data 1340 * packets delivered "delivered" over an interval of time "interval_us". 1341 * The tcp_rate.c code fills in the rate sample, and congestion 1342 * control modules that define a cong_control function to run at the end 1343 * of ACK processing can optionally chose to consult this sample when 1344 * setting cwnd and pacing rate. 1345 * A sample is invalid if "delivered" or "interval_us" is negative. 1346 */ 1347 struct rate_sample { 1348 u64 prior_mstamp; /* starting timestamp for interval */ 1349 u32 prior_delivered; /* tp->delivered at "prior_mstamp" */ 1350 u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */ 1351 s32 delivered; /* number of packets delivered over interval */ 1352 s32 delivered_ce; /* number of packets delivered w/ CE marks*/ 1353 long interval_us; /* time for tp->delivered to incr "delivered" */ 1354 u32 snd_interval_us; /* snd interval for delivered packets */ 1355 u32 rcv_interval_us; /* rcv interval for delivered packets */ 1356 long rtt_us; /* RTT of last (S)ACKed packet (or -1) */ 1357 int losses; /* number of packets marked lost upon ACK */ 1358 u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */ 1359 u32 prior_in_flight; /* in flight before this ACK */ 1360 u32 last_end_seq; /* end_seq of most recently ACKed packet */ 1361 bool is_app_limited; /* is sample from packet with bubble in pipe? */ 1362 bool is_retrans; /* is sample from retransmission? */ 1363 bool is_ack_delayed; /* is this (likely) a delayed ACK? */ 1364 }; 1365 1366 struct tcp_congestion_ops { 1367 /* fast path fields are put first to fill one cache line */ 1368 1369 /* A congestion control (CC) must provide one of either: 1370 * 1371 * (a) a cong_avoid function, if the CC wants to use the core TCP 1372 * stack's default functionality to implement a "classic" 1373 * (Reno/CUBIC-style) response to packet loss, RFC3168 ECN, 1374 * idle periods, pacing rate computations, etc. 1375 * 1376 * (b) a cong_control function, if the CC wants custom behavior and 1377 * complete control of all congestion control behaviors. 1378 */ 1379 /* (a) "classic" response: calculate new cwnd. 1380 */ 1381 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); 1382 /* (b) "custom" response: call when packets are delivered to update 1383 * cwnd and pacing rate, after all the ca_state processing. 1384 */ 1385 void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs); 1386 1387 /* return slow start threshold (required) */ 1388 u32 (*ssthresh)(struct sock *sk); 1389 1390 /* call before changing ca_state (optional) */ 1391 void (*set_state)(struct sock *sk, u8 new_state); 1392 1393 /* call when cwnd event occurs (optional) */ 1394 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); 1395 1396 /* call when CA_EVENT_TX_START cwnd event occurs (optional) */ 1397 void (*cwnd_event_tx_start)(struct sock *sk); 1398 1399 /* call when ack arrives (optional) */ 1400 void (*in_ack_event)(struct sock *sk, u32 flags); 1401 1402 /* hook for packet ack accounting (optional) */ 1403 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); 1404 1405 /* Override tcp_tso_autosize() (optional) 1406 * 1407 * If provided, this callback supplies the TSO segment target count 1408 * instead of using tcp_tso_autosize(). The returned value is 1409 * subsequently clamped to [1, sk->sk_gso_max_segs] by the caller. 1410 * 1411 * For the kernel callback path, mss_now originates from 1412 * tcp_current_mss() and should never be zero. 1413 */ 1414 u32 (*tso_segs)(struct sock *sk, u32 mss_now); 1415 1416 /* new value of cwnd after loss (required) */ 1417 u32 (*undo_cwnd)(struct sock *sk); 1418 /* returns the multiplier used in tcp_sndbuf_expand (optional) */ 1419 u32 (*sndbuf_expand)(struct sock *sk); 1420 1421 /* control/slow paths put last */ 1422 /* get info for inet_diag (optional) */ 1423 size_t (*get_info)(struct sock *sk, u32 ext, int *attr, 1424 union tcp_cc_info *info); 1425 1426 char name[TCP_CA_NAME_MAX]; 1427 struct module *owner; 1428 struct list_head list; 1429 u32 key; 1430 u32 flags; 1431 1432 /* initialize private data (optional) */ 1433 void (*init)(struct sock *sk); 1434 /* cleanup private data (optional) */ 1435 void (*release)(struct sock *sk); 1436 } ____cacheline_aligned_in_smp; 1437 1438 int tcp_register_congestion_control(struct tcp_congestion_ops *type); 1439 void tcp_unregister_congestion_control(struct tcp_congestion_ops *type); 1440 int tcp_update_congestion_control(struct tcp_congestion_ops *type, 1441 struct tcp_congestion_ops *old_type); 1442 int tcp_validate_congestion_control(struct tcp_congestion_ops *ca); 1443 1444 void tcp_assign_congestion_control(struct sock *sk); 1445 void tcp_init_congestion_control(struct sock *sk); 1446 void tcp_cleanup_congestion_control(struct sock *sk); 1447 int tcp_set_default_congestion_control(struct net *net, const char *name); 1448 void tcp_get_default_congestion_control(struct net *net, char *name); 1449 void tcp_get_available_congestion_control(char *buf, size_t len); 1450 void tcp_get_allowed_congestion_control(char *buf, size_t len); 1451 int tcp_set_allowed_congestion_control(char *allowed); 1452 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, 1453 bool cap_net_admin); 1454 u32 tcp_slow_start(struct tcp_sock *tp, u32 acked); 1455 void tcp_cong_avoid_ai(struct tcp_sock *tp, u32 w, u32 acked); 1456 1457 u32 tcp_reno_ssthresh(struct sock *sk); 1458 u32 tcp_reno_undo_cwnd(struct sock *sk); 1459 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked); 1460 extern struct tcp_congestion_ops tcp_reno; 1461 1462 struct tcp_congestion_ops *tcp_ca_find(const char *name); 1463 struct tcp_congestion_ops *tcp_ca_find_key(u32 key); 1464 u32 tcp_ca_get_key_by_name(const char *name, bool *ecn_ca); 1465 #ifdef CONFIG_INET 1466 char *tcp_ca_get_name_by_key(u32 key, char *buffer); 1467 #else 1468 static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer) 1469 { 1470 return NULL; 1471 } 1472 #endif 1473 1474 static inline bool tcp_ca_needs_ecn(const struct sock *sk) 1475 { 1476 const struct inet_connection_sock *icsk = inet_csk(sk); 1477 1478 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; 1479 } 1480 1481 static inline bool tcp_ca_needs_accecn(const struct sock *sk) 1482 { 1483 const struct inet_connection_sock *icsk = inet_csk(sk); 1484 1485 return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; 1486 } 1487 1488 static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) 1489 { 1490 const struct inet_connection_sock *icsk = inet_csk(sk); 1491 1492 return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; 1493 } 1494 1495 static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) 1496 { 1497 const struct inet_connection_sock *icsk = inet_csk(sk); 1498 1499 return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; 1500 } 1501 1502 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) 1503 { 1504 const struct inet_connection_sock *icsk = inet_csk(sk); 1505 1506 if (event == CA_EVENT_TX_START) { 1507 if (icsk->icsk_ca_ops->cwnd_event_tx_start) 1508 icsk->icsk_ca_ops->cwnd_event_tx_start(sk); 1509 return; 1510 } 1511 if (icsk->icsk_ca_ops->cwnd_event) 1512 icsk->icsk_ca_ops->cwnd_event(sk, event); 1513 } 1514 1515 /* From tcp_cong.c */ 1516 void tcp_set_ca_state(struct sock *sk, const u8 ca_state); 1517 1518 1519 static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) 1520 { 1521 return t1 > t2 || (t1 == t2 && after(seq1, seq2)); 1522 } 1523 1524 /* These functions determine how the current flow behaves in respect of SACK 1525 * handling. SACK is negotiated with the peer, and therefore it can vary 1526 * between different flows. 1527 * 1528 * tcp_is_sack - SACK enabled 1529 * tcp_is_reno - No SACK 1530 */ 1531 static inline int tcp_is_sack(const struct tcp_sock *tp) 1532 { 1533 return likely(tp->rx_opt.sack_ok); 1534 } 1535 1536 static inline bool tcp_is_reno(const struct tcp_sock *tp) 1537 { 1538 return !tcp_is_sack(tp); 1539 } 1540 1541 static inline unsigned int tcp_left_out(const struct tcp_sock *tp) 1542 { 1543 return tp->sacked_out + tp->lost_out; 1544 } 1545 1546 /* This determines how many packets are "in the network" to the best 1547 * of our knowledge. In many cases it is conservative, but where 1548 * detailed information is available from the receiver (via SACK 1549 * blocks etc.) we can make more aggressive calculations. 1550 * 1551 * Use this for decisions involving congestion control, use just 1552 * tp->packets_out to determine if the send queue is empty or not. 1553 * 1554 * Read this equation as: 1555 * 1556 * "Packets sent once on transmission queue" MINUS 1557 * "Packets left network, but not honestly ACKed yet" PLUS 1558 * "Packets fast retransmitted" 1559 */ 1560 static inline unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 1561 { 1562 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 1563 } 1564 1565 #define TCP_INFINITE_SSTHRESH 0x7fffffff 1566 1567 /* Default and maximum snd_cwnd_clamp, in packets. 1568 * A peer can not advertise more than 65535 << 14 bytes, so a larger 1569 * cwnd is useless with any MSS of 1024 bytes or more. 1570 * Congestion control modules multiply cwnd by u32 factors up to 2047 1571 * (cwnd * (BICTCP_BETA_SCALE + beta) in CUBIC), and CUBIC's 1572 * bictcp_update() expects cwnd to stay below about one million packets. 1573 */ 1574 #define TCP_MAX_SND_CWND (1U << 20) 1575 1576 static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) 1577 { 1578 return tp->snd_cwnd; 1579 } 1580 1581 static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) 1582 { 1583 WARN_ON_ONCE((int)val <= 0); 1584 WRITE_ONCE(tp->snd_cwnd, val); 1585 } 1586 1587 static inline bool tcp_in_slow_start(const struct tcp_sock *tp) 1588 { 1589 return tcp_snd_cwnd(tp) < tp->snd_ssthresh; 1590 } 1591 1592 static inline bool tcp_in_initial_slowstart(const struct tcp_sock *tp) 1593 { 1594 return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH; 1595 } 1596 1597 static inline bool tcp_in_cwnd_reduction(const struct sock *sk) 1598 { 1599 return (TCPF_CA_CWR | TCPF_CA_Recovery) & 1600 (1 << inet_csk(sk)->icsk_ca_state); 1601 } 1602 1603 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd. 1604 * The exception is cwnd reduction phase, when cwnd is decreasing towards 1605 * ssthresh. 1606 */ 1607 static inline __u32 tcp_current_ssthresh(const struct sock *sk) 1608 { 1609 const struct tcp_sock *tp = tcp_sk(sk); 1610 1611 if (tcp_in_cwnd_reduction(sk)) 1612 return tp->snd_ssthresh; 1613 else 1614 return max(tp->snd_ssthresh, 1615 ((tcp_snd_cwnd(tp) >> 1) + 1616 (tcp_snd_cwnd(tp) >> 2))); 1617 } 1618 1619 /* Use define here intentionally to get WARN_ON location shown at the caller */ 1620 #define tcp_verify_left_out(tp) WARN_ON(tcp_left_out(tp) > tp->packets_out) 1621 1622 void tcp_enter_cwr(struct sock *sk); 1623 __u32 tcp_init_cwnd(const struct tcp_sock *tp, const struct dst_entry *dst); 1624 1625 /* The maximum number of MSS of available cwnd for which TSO defers 1626 * sending if not using sysctl_tcp_tso_win_divisor. 1627 */ 1628 static inline __u32 tcp_max_tso_deferred_mss(const struct tcp_sock *tp) 1629 { 1630 return 3; 1631 } 1632 1633 /* Returns end sequence number of the receiver's advertised window */ 1634 static inline u32 tcp_wnd_end(const struct tcp_sock *tp) 1635 { 1636 return tp->snd_una + tp->snd_wnd; 1637 } 1638 1639 /* We follow the spirit of RFC2861 to validate cwnd but implement a more 1640 * flexible approach. The RFC suggests cwnd should not be raised unless 1641 * it was fully used previously. And that's exactly what we do in 1642 * congestion avoidance mode. But in slow start we allow cwnd to grow 1643 * as long as the application has used half the cwnd. 1644 * Example : 1645 * cwnd is 10 (IW10), but application sends 9 frames. 1646 * We allow cwnd to reach 18 when all frames are ACKed. 1647 * This check is safe because it's as aggressive as slow start which already 1648 * risks 100% overshoot. The advantage is that we discourage application to 1649 * either send more filler packets or data to artificially blow up the cwnd 1650 * usage, and allow application-limited process to probe bw more aggressively. 1651 */ 1652 static inline bool tcp_is_cwnd_limited(const struct sock *sk) 1653 { 1654 const struct tcp_sock *tp = tcp_sk(sk); 1655 1656 if (tp->is_cwnd_limited) 1657 return true; 1658 1659 /* If in slow start, ensure cwnd grows to twice what was ACKed. */ 1660 if (tcp_in_slow_start(tp)) 1661 return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; 1662 1663 return false; 1664 } 1665 1666 /* BBR congestion control needs pacing. 1667 * Same remark for SO_MAX_PACING_RATE. 1668 * sch_fq packet scheduler is efficiently handling pacing, 1669 * but is not always installed/used. 1670 * Return true if TCP stack should pace packets itself. 1671 */ 1672 static inline bool tcp_needs_internal_pacing(const struct sock *sk) 1673 { 1674 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; 1675 } 1676 1677 /* Estimates in how many jiffies next packet for this flow can be sent. 1678 * Scheduling a retransmit timer too early would be silly. 1679 */ 1680 static inline unsigned long tcp_pacing_delay(const struct sock *sk) 1681 { 1682 s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; 1683 1684 return delay > 0 ? nsecs_to_jiffies(delay) : 0; 1685 } 1686 1687 static inline void tcp_reset_xmit_timer(struct sock *sk, 1688 const int what, 1689 unsigned long when, 1690 bool pace_delay) 1691 { 1692 if (pace_delay) 1693 when += tcp_pacing_delay(sk); 1694 inet_csk_reset_xmit_timer(sk, what, when, 1695 tcp_rto_max(sk)); 1696 } 1697 1698 /* Something is really bad, we could not queue an additional packet, 1699 * because qdisc is full or receiver sent a 0 window, or we are paced. 1700 * We do not want to add fuel to the fire, or abort too early, 1701 * so make sure the timer we arm now is at least 200ms in the future, 1702 * regardless of current icsk_rto value (as it could be ~2ms) 1703 */ 1704 static inline unsigned long tcp_probe0_base(const struct sock *sk) 1705 { 1706 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); 1707 } 1708 1709 /* Variant of inet_csk_rto_backoff() used for zero window probes */ 1710 static inline unsigned long tcp_probe0_when(const struct sock *sk, 1711 unsigned long max_when) 1712 { 1713 u8 backoff = min_t(u8, ilog2(TCP_RTO_MAX / TCP_RTO_MIN) + 1, 1714 inet_csk(sk)->icsk_backoff); 1715 u64 when = (u64)tcp_probe0_base(sk) << backoff; 1716 1717 return (unsigned long)min_t(u64, when, max_when); 1718 } 1719 1720 static inline void tcp_check_probe_timer(struct sock *sk) 1721 { 1722 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) 1723 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, 1724 tcp_probe0_base(sk), true); 1725 } 1726 1727 static inline void tcp_init_wl(struct tcp_sock *tp, u32 seq) 1728 { 1729 tp->snd_wl1 = seq; 1730 } 1731 1732 static inline void tcp_update_wl(struct tcp_sock *tp, u32 seq) 1733 { 1734 tp->snd_wl1 = seq; 1735 } 1736 1737 /* 1738 * Calculate(/check) TCP checksum 1739 */ 1740 static inline __sum16 tcp_v4_check(int len, __be32 saddr, 1741 __be32 daddr, __wsum base) 1742 { 1743 return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_TCP, base); 1744 } 1745 1746 static inline bool tcp_checksum_complete(struct sk_buff *skb) 1747 { 1748 return !skb_csum_unnecessary(skb) && 1749 __skb_checksum_complete(skb); 1750 } 1751 1752 enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb); 1753 1754 static inline enum skb_drop_reason 1755 tcp_filter(struct sock *sk, struct sk_buff *skb) 1756 { 1757 const struct tcphdr *th = (const struct tcphdr *)skb->data; 1758 1759 return sk_filter_trim_cap(sk, skb, __tcp_hdrlen(th)); 1760 } 1761 1762 void tcp_set_state(struct sock *sk, int state); 1763 void tcp_done(struct sock *sk); 1764 int tcp_abort(struct sock *sk, int err); 1765 1766 static inline void tcp_sack_reset(struct tcp_options_received *rx_opt) 1767 { 1768 rx_opt->dsack = 0; 1769 rx_opt->num_sacks = 0; 1770 } 1771 1772 void tcp_cwnd_restart(struct sock *sk, s32 delta); 1773 1774 static inline void tcp_slow_start_after_idle_check(struct sock *sk) 1775 { 1776 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; 1777 struct tcp_sock *tp = tcp_sk(sk); 1778 s32 delta; 1779 1780 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || 1781 tp->packets_out || ca_ops->cong_control) 1782 return; 1783 delta = tcp_jiffies32 - tp->lsndtime; 1784 if (delta > inet_csk(sk)->icsk_rto) 1785 tcp_cwnd_restart(sk, delta); 1786 } 1787 1788 /* Determine a window scaling and initial window to offer. */ 1789 void tcp_select_initial_window(const struct sock *sk, int __space, 1790 __u32 mss, __u32 *rcv_wnd, 1791 __u32 *window_clamp, int wscale_ok, 1792 __u8 *rcv_wscale, __u32 init_rcv_wnd); 1793 1794 static inline int __tcp_win_from_space(u8 scaling_ratio, int space) 1795 { 1796 s64 scaled_space = (s64)space * scaling_ratio; 1797 1798 return scaled_space >> TCP_RMEM_TO_WIN_SCALE; 1799 } 1800 1801 static inline int tcp_win_from_space(const struct sock *sk, int space) 1802 { 1803 return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); 1804 } 1805 1806 /* inverse of __tcp_win_from_space() */ 1807 static inline int __tcp_space_from_win(u8 scaling_ratio, int win) 1808 { 1809 u64 val = (u64)win << TCP_RMEM_TO_WIN_SCALE; 1810 1811 do_div(val, scaling_ratio); 1812 return val; 1813 } 1814 1815 static inline int tcp_space_from_win(const struct sock *sk, int win) 1816 { 1817 return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); 1818 } 1819 1820 /* Assume a 50% default for skb->len/skb->truesize ratio. 1821 * This may be adjusted later in tcp_measure_rcv_mss(). 1822 */ 1823 #define TCP_DEFAULT_SCALING_RATIO (1 << (TCP_RMEM_TO_WIN_SCALE - 1)) 1824 1825 static inline void tcp_scaling_ratio_init(struct sock *sk) 1826 { 1827 tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; 1828 } 1829 1830 /* Note: caller must be prepared to deal with negative returns */ 1831 static inline int tcp_space(const struct sock *sk) 1832 { 1833 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - 1834 READ_ONCE(sk->sk_backlog.len) - 1835 atomic_read(&sk->sk_rmem_alloc)); 1836 } 1837 1838 static inline int tcp_full_space(const struct sock *sk) 1839 { 1840 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); 1841 } 1842 1843 static inline u32 tcp_dst_advmss(const struct dst_entry *dst) 1844 { 1845 return max_t(u32, dst_metric_advmss(dst), TCP_MIN_MSS); 1846 } 1847 1848 static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) 1849 { 1850 int unused_mem = sk_unused_reserved_mem(sk); 1851 struct tcp_sock *tp = tcp_sk(sk); 1852 1853 tp->rcv_ssthresh = min(tp->rcv_ssthresh, new_ssthresh); 1854 if (unused_mem) 1855 tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh, 1856 tcp_win_from_space(sk, unused_mem)); 1857 } 1858 1859 static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) 1860 { 1861 __tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss); 1862 } 1863 1864 void tcp_cleanup_rbuf(struct sock *sk, int copied); 1865 void __tcp_cleanup_rbuf(struct sock *sk, int copied); 1866 1867 /* Is the socket in a state in which TCP may send an ACK? */ 1868 static inline bool tcp_can_send_ack(const struct sock *sk) 1869 { 1870 return !((1 << inet_sk_state_load(sk)) & 1871 (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 1872 } 1873 1874 /* We provision sk_rcvbuf around 200% of sk_rcvlowat. 1875 * If 87.5 % (7/8) of the space has been consumed, we want to override 1876 * SO_RCVLOWAT constraint, since we are receiving skbs with too small 1877 * len/truesize ratio. 1878 */ 1879 static inline bool tcp_rmem_pressure(const struct sock *sk) 1880 { 1881 int rcvbuf, threshold; 1882 1883 if (tcp_under_memory_pressure(sk)) 1884 return true; 1885 1886 rcvbuf = READ_ONCE(sk->sk_rcvbuf); 1887 threshold = rcvbuf - (rcvbuf >> 3); 1888 1889 return atomic_read(&sk->sk_rmem_alloc) > threshold; 1890 } 1891 1892 static inline bool tcp_epollin_ready(const struct sock *sk, int target) 1893 { 1894 const struct tcp_sock *tp = tcp_sk(sk); 1895 int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq); 1896 1897 if (avail <= 0) 1898 return false; 1899 1900 return (avail >= target) || tcp_rmem_pressure(sk) || 1901 (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); 1902 } 1903 1904 extern void tcp_openreq_init_rwin(struct request_sock *req, 1905 const struct sock *sk_listener, 1906 const struct dst_entry *dst); 1907 1908 void tcp_enter_memory_pressure(struct sock *sk); 1909 void tcp_leave_memory_pressure(struct sock *sk); 1910 1911 static inline int keepalive_intvl_when(const struct tcp_sock *tp) 1912 { 1913 struct net *net = sock_net((struct sock *)tp); 1914 int val; 1915 1916 /* Paired with WRITE_ONCE() in tcp_sock_set_keepintvl() 1917 * and do_tcp_setsockopt(). 1918 */ 1919 val = READ_ONCE(tp->keepalive_intvl); 1920 1921 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl); 1922 } 1923 1924 static inline int keepalive_time_when(const struct tcp_sock *tp) 1925 { 1926 struct net *net = sock_net((struct sock *)tp); 1927 int val; 1928 1929 /* Paired with WRITE_ONCE() in tcp_sock_set_keepidle_locked() */ 1930 val = READ_ONCE(tp->keepalive_time); 1931 1932 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); 1933 } 1934 1935 static inline int keepalive_probes(const struct tcp_sock *tp) 1936 { 1937 struct net *net = sock_net((struct sock *)tp); 1938 int val; 1939 1940 /* Paired with WRITE_ONCE() in tcp_sock_set_keepcnt() 1941 * and do_tcp_setsockopt(). 1942 */ 1943 val = READ_ONCE(tp->keepalive_probes); 1944 1945 return val ? : READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes); 1946 } 1947 1948 static inline u32 keepalive_time_elapsed(const struct tcp_sock *tp) 1949 { 1950 const struct inet_connection_sock *icsk = &tp->inet_conn; 1951 1952 return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime, 1953 tcp_jiffies32 - tp->rcv_tstamp); 1954 } 1955 1956 static inline int tcp_fin_time(const struct sock *sk) 1957 { 1958 int fin_timeout = tcp_sk(sk)->linger2 ? : 1959 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); 1960 const int rto = inet_csk(sk)->icsk_rto; 1961 1962 if (fin_timeout < (rto << 2) - (rto >> 1)) 1963 fin_timeout = (rto << 2) - (rto >> 1); 1964 1965 return fin_timeout; 1966 } 1967 1968 static inline bool tcp_paws_check(const struct tcp_options_received *rx_opt, 1969 int paws_win) 1970 { 1971 if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win) 1972 return true; 1973 if (unlikely(!time_before32(ktime_get_seconds(), 1974 rx_opt->ts_recent_stamp + TCP_PAWS_WRAP))) 1975 return true; 1976 /* 1977 * Some OSes send SYN and SYNACK messages with tsval=0 tsecr=0, 1978 * then following tcp messages have valid values. Ignore 0 value, 1979 * or else 'negative' tsval might forbid us to accept their packets. 1980 */ 1981 if (!rx_opt->ts_recent) 1982 return true; 1983 return false; 1984 } 1985 1986 static inline bool tcp_paws_reject(const struct tcp_options_received *rx_opt, 1987 int rst) 1988 { 1989 if (tcp_paws_check(rx_opt, 0)) 1990 return false; 1991 1992 /* RST segments are not recommended to carry timestamp, 1993 and, if they do, it is recommended to ignore PAWS because 1994 "their cleanup function should take precedence over timestamps." 1995 Certainly, it is mistake. It is necessary to understand the reasons 1996 of this constraint to relax it: if peer reboots, clock may go 1997 out-of-sync and half-open connections will not be reset. 1998 Actually, the problem would be not existing if all 1999 the implementations followed draft about maintaining clock 2000 via reboots. Linux-2.2 DOES NOT! 2001 2002 However, we can relax time bounds for RST segments to MSL. 2003 */ 2004 if (rst && !time_before32(ktime_get_seconds(), 2005 rx_opt->ts_recent_stamp + TCP_PAWS_MSL)) 2006 return false; 2007 return true; 2008 } 2009 2010 static inline void __tcp_fast_path_on(struct tcp_sock *tp, u32 snd_wnd) 2011 { 2012 u32 ace; 2013 2014 /* mptcp hooks are only on the slow path */ 2015 if (sk_is_mptcp((struct sock *)tp)) 2016 return; 2017 2018 ace = tcp_ecn_mode_accecn(tp) ? 2019 ((tp->delivered_ce + TCP_ACCECN_CEP_INIT_OFFSET) & 2020 TCP_ACCECN_CEP_ACE_MASK) : 0; 2021 2022 tp->pred_flags = htonl((tp->tcp_header_len << 26) | 2023 (ace << 22) | 2024 ntohl(TCP_FLAG_ACK) | 2025 snd_wnd); 2026 } 2027 2028 static inline void tcp_fast_path_on(struct tcp_sock *tp) 2029 { 2030 __tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale); 2031 } 2032 2033 static inline void tcp_fast_path_check(struct sock *sk) 2034 { 2035 struct tcp_sock *tp = tcp_sk(sk); 2036 2037 if (RB_EMPTY_ROOT(&tp->out_of_order_queue) && 2038 tp->rcv_wnd && 2039 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && 2040 !tp->urg_data) 2041 tcp_fast_path_on(tp); 2042 } 2043 2044 bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, 2045 int mib_idx, u32 *last_oow_ack_time); 2046 void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb, 2047 struct request_sock *req); 2048 2049 static inline void tcp_mib_init(struct net *net) 2050 { 2051 /* See RFC 2012 */ 2052 TCP_ADD_STATS(net, TCP_MIB_RTOALGORITHM, 1); 2053 TCP_ADD_STATS(net, TCP_MIB_RTOMIN, TCP_RTO_MIN*1000/HZ); 2054 TCP_ADD_STATS(net, TCP_MIB_RTOMAX, TCP_RTO_MAX*1000/HZ); 2055 TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1); 2056 } 2057 2058 /* from STCP */ 2059 static inline void tcp_clear_all_retrans_hints(struct tcp_sock *tp) 2060 { 2061 tp->retransmit_skb_hint = NULL; 2062 } 2063 2064 #define tcp_md5_addr tcp_ao_addr 2065 2066 /* - key database */ 2067 struct tcp_md5sig_key { 2068 struct hlist_node node; 2069 u8 keylen; 2070 u8 family; /* AF_INET or AF_INET6 */ 2071 u8 prefixlen; 2072 u8 flags; 2073 union tcp_md5_addr addr; 2074 int l3index; /* set if key added with L3 scope */ 2075 u8 key[TCP_MD5SIG_MAXKEYLEN]; 2076 struct rcu_head rcu; 2077 }; 2078 2079 /* - sock block */ 2080 struct tcp_md5sig_info { 2081 struct hlist_head head; 2082 struct rcu_head rcu; 2083 }; 2084 2085 /* - pseudo header */ 2086 struct tcp4_pseudohdr { 2087 __be32 saddr; 2088 __be32 daddr; 2089 __u8 pad; 2090 __u8 protocol; 2091 __be16 len; 2092 }; 2093 2094 struct tcp6_pseudohdr { 2095 struct in6_addr saddr; 2096 struct in6_addr daddr; 2097 __be32 len; 2098 __be32 protocol; /* including padding */ 2099 }; 2100 2101 void tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, 2102 const struct sock *sk, const struct sk_buff *skb); 2103 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, 2104 int family, u8 prefixlen, int l3index, u8 flags, 2105 const u8 *newkey, u8 newkeylen); 2106 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr, 2107 int family, u8 prefixlen, int l3index, 2108 struct tcp_md5sig_key *key); 2109 2110 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, 2111 int family, u8 prefixlen, int l3index, u8 flags); 2112 void tcp_clear_md5_list(struct sock *sk); 2113 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk, 2114 const struct sock *addr_sk); 2115 2116 #ifdef CONFIG_TCP_MD5SIG 2117 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index, 2118 const union tcp_md5_addr *addr, 2119 int family, bool any_l3index); 2120 static inline struct tcp_md5sig_key * 2121 tcp_md5_do_lookup(const struct sock *sk, int l3index, 2122 const union tcp_md5_addr *addr, int family) 2123 { 2124 if (!static_branch_unlikely(&tcp_md5_needed.key)) 2125 return NULL; 2126 return __tcp_md5_do_lookup(sk, l3index, addr, family, false); 2127 } 2128 2129 static inline struct tcp_md5sig_key * 2130 tcp_md5_do_lookup_any_l3index(const struct sock *sk, 2131 const union tcp_md5_addr *addr, int family) 2132 { 2133 if (!static_branch_unlikely(&tcp_md5_needed.key)) 2134 return NULL; 2135 return __tcp_md5_do_lookup(sk, 0, addr, family, true); 2136 } 2137 2138 #define tcp_twsk_md5_key(twsk) ((twsk)->tw_md5_key) 2139 void tcp_md5_destruct_sock(struct sock *sk); 2140 #else 2141 static inline struct tcp_md5sig_key * 2142 tcp_md5_do_lookup(const struct sock *sk, int l3index, 2143 const union tcp_md5_addr *addr, int family) 2144 { 2145 return NULL; 2146 } 2147 2148 static inline struct tcp_md5sig_key * 2149 tcp_md5_do_lookup_any_l3index(const struct sock *sk, 2150 const union tcp_md5_addr *addr, int family) 2151 { 2152 return NULL; 2153 } 2154 2155 #define tcp_twsk_md5_key(twsk) NULL 2156 static inline void tcp_md5_destruct_sock(struct sock *sk) 2157 { 2158 } 2159 #endif 2160 2161 struct md5_ctx; 2162 void tcp_md5_hash_skb_data(struct md5_ctx *ctx, const struct sk_buff *skb, 2163 unsigned int header_len); 2164 void tcp_md5_hash_key(struct md5_ctx *ctx, const struct tcp_md5sig_key *key); 2165 2166 /* From tcp_fastopen.c */ 2167 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 2168 struct tcp_fastopen_cookie *cookie); 2169 void tcp_fastopen_cache_set(struct sock *sk, u16 mss, 2170 struct tcp_fastopen_cookie *cookie, bool syn_lost, 2171 u16 try_exp); 2172 struct tcp_fastopen_request { 2173 /* Fast Open cookie. Size 0 means a cookie request */ 2174 struct tcp_fastopen_cookie cookie; 2175 struct msghdr *data; /* data in MSG_FASTOPEN */ 2176 size_t size; 2177 int copied; /* queued in tcp_connect() */ 2178 struct ubuf_info *uarg; 2179 }; 2180 void tcp_free_fastopen_req(struct tcp_sock *tp); 2181 void tcp_fastopen_destroy_cipher(struct sock *sk); 2182 void tcp_fastopen_ctx_destroy(struct net *net); 2183 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk, 2184 void *primary_key, void *backup_key); 2185 int tcp_fastopen_get_cipher(struct net *net, struct inet_connection_sock *icsk, 2186 u64 *key); 2187 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb); 2188 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb, 2189 struct request_sock *req, 2190 struct tcp_fastopen_cookie *foc, 2191 const struct dst_entry *dst); 2192 void tcp_fastopen_init_key_once(struct net *net); 2193 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss, 2194 struct tcp_fastopen_cookie *cookie); 2195 bool tcp_fastopen_defer_connect(struct sock *sk, int *err); 2196 #define TCP_FASTOPEN_KEY_LENGTH sizeof(siphash_key_t) 2197 #define TCP_FASTOPEN_KEY_MAX 2 2198 #define TCP_FASTOPEN_KEY_BUF_LENGTH \ 2199 (TCP_FASTOPEN_KEY_LENGTH * TCP_FASTOPEN_KEY_MAX) 2200 2201 /* Fastopen key context */ 2202 struct tcp_fastopen_context { 2203 siphash_key_t key[TCP_FASTOPEN_KEY_MAX]; 2204 int num; 2205 struct rcu_head rcu; 2206 }; 2207 2208 void tcp_fastopen_active_disable(struct sock *sk); 2209 bool tcp_fastopen_active_should_disable(struct sock *sk); 2210 void tcp_fastopen_active_disable_ofo_check(struct sock *sk); 2211 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired); 2212 2213 /* Caller needs to wrap with rcu_read_(un)lock() */ 2214 static inline 2215 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) 2216 { 2217 struct tcp_fastopen_context *ctx; 2218 2219 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); 2220 if (!ctx) 2221 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); 2222 return ctx; 2223 } 2224 2225 static inline 2226 bool tcp_fastopen_cookie_match(const struct tcp_fastopen_cookie *foc, 2227 const struct tcp_fastopen_cookie *orig) 2228 { 2229 if (orig->len == TCP_FASTOPEN_COOKIE_SIZE && 2230 orig->len == foc->len && 2231 !memcmp(orig->val, foc->val, foc->len)) 2232 return true; 2233 return false; 2234 } 2235 2236 static inline 2237 int tcp_fastopen_context_len(const struct tcp_fastopen_context *ctx) 2238 { 2239 return ctx->num; 2240 } 2241 2242 /* Latencies incurred by various limits for a sender. They are 2243 * chronograph-like stats that are mutually exclusive. 2244 */ 2245 enum tcp_chrono { 2246 TCP_CHRONO_UNSPEC, 2247 TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */ 2248 TCP_CHRONO_RWND_LIMITED, /* Stalled by insufficient receive window */ 2249 TCP_CHRONO_SNDBUF_LIMITED, /* Stalled by insufficient send buffer */ 2250 __TCP_CHRONO_MAX, 2251 }; 2252 2253 static inline void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new) 2254 { 2255 const u32 now = tcp_jiffies32; 2256 enum tcp_chrono old = tp->chrono_type; 2257 2258 /* Following WRITE_ONCE()s pair with READ_ONCE()s in 2259 * tcp_get_info_chrono_stats(). 2260 */ 2261 if (old > TCP_CHRONO_UNSPEC) 2262 WRITE_ONCE(tp->chrono_stat[old - 1], 2263 tp->chrono_stat[old - 1] + now - tp->chrono_start); 2264 WRITE_ONCE(tp->chrono_start, now); 2265 WRITE_ONCE(tp->chrono_type, new); 2266 } 2267 2268 static inline void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type) 2269 { 2270 struct tcp_sock *tp = tcp_sk(sk); 2271 2272 /* If there are multiple conditions worthy of tracking in a 2273 * chronograph then the highest priority enum takes precedence 2274 * over the other conditions. So that if something "more interesting" 2275 * starts happening, stop the previous chrono and start a new one. 2276 */ 2277 if (type > tp->chrono_type) 2278 tcp_chrono_set(tp, type); 2279 } 2280 2281 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); 2282 2283 /* This helper is needed, because skb->tcp_tsorted_anchor uses 2284 * the same memory storage than skb->destructor/_skb_refdst 2285 */ 2286 static inline void tcp_skb_tsorted_anchor_cleanup(struct sk_buff *skb) 2287 { 2288 skb->destructor = NULL; 2289 skb->_skb_refdst = 0UL; 2290 } 2291 2292 #define tcp_skb_tsorted_save(skb) { \ 2293 unsigned long _save = skb->_skb_refdst; \ 2294 skb->_skb_refdst = 0UL; 2295 2296 #define tcp_skb_tsorted_restore(skb) \ 2297 skb->_skb_refdst = _save; \ 2298 } 2299 2300 void tcp_write_queue_purge(struct sock *sk); 2301 2302 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) 2303 { 2304 return skb_rb_first(&sk->tcp_rtx_queue); 2305 } 2306 2307 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) 2308 { 2309 return skb_rb_last(&sk->tcp_rtx_queue); 2310 } 2311 2312 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) 2313 { 2314 return skb_peek_tail(&sk->sk_write_queue); 2315 } 2316 2317 #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ 2318 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp) 2319 2320 static inline struct sk_buff *tcp_send_head(const struct sock *sk) 2321 { 2322 return skb_peek(&sk->sk_write_queue); 2323 } 2324 2325 static inline bool tcp_skb_is_last(const struct sock *sk, 2326 const struct sk_buff *skb) 2327 { 2328 return skb_queue_is_last(&sk->sk_write_queue, skb); 2329 } 2330 2331 /** 2332 * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue 2333 * @sk: socket 2334 * 2335 * Since the write queue can have a temporary empty skb in it, 2336 * we must not use "return skb_queue_empty(&sk->sk_write_queue)" 2337 */ 2338 static inline bool tcp_write_queue_empty(const struct sock *sk) 2339 { 2340 const struct tcp_sock *tp = tcp_sk(sk); 2341 2342 return tp->write_seq == tp->snd_nxt; 2343 } 2344 2345 static inline bool tcp_rtx_queue_empty(const struct sock *sk) 2346 { 2347 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); 2348 } 2349 2350 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) 2351 { 2352 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); 2353 } 2354 2355 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) 2356 { 2357 __skb_queue_tail(&sk->sk_write_queue, skb); 2358 2359 /* Queue it, remembering where we must start sending. */ 2360 if (sk->sk_write_queue.next == skb) 2361 tcp_chrono_start(sk, TCP_CHRONO_BUSY); 2362 } 2363 2364 /* Insert new before skb on the write queue of sk. */ 2365 static inline void tcp_insert_write_queue_before(struct sk_buff *new, 2366 struct sk_buff *skb, 2367 struct sock *sk) 2368 { 2369 __skb_queue_before(&sk->sk_write_queue, skb, new); 2370 } 2371 2372 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) 2373 { 2374 tcp_skb_tsorted_anchor_cleanup(skb); 2375 __skb_unlink(skb, &sk->sk_write_queue); 2376 } 2377 2378 void tcp_rbtree_insert(struct rb_root *root, struct sk_buff *skb); 2379 2380 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) 2381 { 2382 tcp_skb_tsorted_anchor_cleanup(skb); 2383 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); 2384 } 2385 2386 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) 2387 { 2388 list_del(&skb->tcp_tsorted_anchor); 2389 tcp_rtx_queue_unlink(skb, sk); 2390 tcp_wmem_free_skb(sk, skb); 2391 } 2392 2393 static inline void tcp_write_collapse_fence(struct sock *sk) 2394 { 2395 struct sk_buff *skb = tcp_write_queue_tail(sk) ?: tcp_rtx_queue_tail(sk); 2396 2397 if (skb) 2398 TCP_SKB_CB(skb)->eor = 1; 2399 } 2400 2401 static inline void tcp_push_pending_frames(struct sock *sk) 2402 { 2403 if (tcp_send_head(sk)) { 2404 struct tcp_sock *tp = tcp_sk(sk); 2405 2406 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); 2407 } 2408 } 2409 2410 /* Start sequence of the skb just after the highest skb with SACKed 2411 * bit, valid only if sacked_out > 0 or when the caller has ensured 2412 * validity by itself. 2413 */ 2414 static inline u32 tcp_highest_sack_seq(struct tcp_sock *tp) 2415 { 2416 if (!tp->sacked_out) 2417 return tp->snd_una; 2418 2419 if (tp->highest_sack == NULL) 2420 return tp->snd_nxt; 2421 2422 return TCP_SKB_CB(tp->highest_sack)->seq; 2423 } 2424 2425 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) 2426 { 2427 tcp_sk(sk)->highest_sack = skb_rb_next(skb); 2428 } 2429 2430 static inline struct sk_buff *tcp_highest_sack(struct sock *sk) 2431 { 2432 return tcp_sk(sk)->highest_sack; 2433 } 2434 2435 static inline void tcp_highest_sack_reset(struct sock *sk) 2436 { 2437 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); 2438 } 2439 2440 /* Called when old skb is about to be deleted and replaced by new skb */ 2441 static inline void tcp_highest_sack_replace(struct sock *sk, 2442 struct sk_buff *old, 2443 struct sk_buff *new) 2444 { 2445 if (old == tcp_highest_sack(sk)) 2446 tcp_sk(sk)->highest_sack = new; 2447 } 2448 2449 /* This helper checks if socket has IP_TRANSPARENT set */ 2450 static inline bool inet_sk_transparent(const struct sock *sk) 2451 { 2452 switch (sk->sk_state) { 2453 case TCP_TIME_WAIT: 2454 return inet_twsk(sk)->tw_transparent; 2455 case TCP_NEW_SYN_RECV: 2456 return inet_rsk(inet_reqsk(sk))->no_srccheck; 2457 } 2458 return inet_test_bit(TRANSPARENT, sk); 2459 } 2460 2461 /* Determines whether this is a thin stream (which may suffer from 2462 * increased latency). Used to trigger latency-reducing mechanisms. 2463 */ 2464 static inline bool tcp_stream_is_thin(struct tcp_sock *tp) 2465 { 2466 return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp); 2467 } 2468 2469 /* /proc */ 2470 enum tcp_seq_states { 2471 TCP_SEQ_STATE_LISTENING, 2472 TCP_SEQ_STATE_ESTABLISHED, 2473 }; 2474 2475 void *tcp_seq_start(struct seq_file *seq, loff_t *pos); 2476 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos); 2477 void tcp_seq_stop(struct seq_file *seq, void *v); 2478 2479 struct tcp_seq_afinfo { 2480 sa_family_t family; 2481 }; 2482 2483 struct tcp_iter_state { 2484 struct seq_net_private p; 2485 enum tcp_seq_states state; 2486 struct sock *syn_wait_sk; 2487 int bucket, offset, sbucket, num; 2488 loff_t last_pos; 2489 }; 2490 2491 extern struct request_sock_ops tcp_request_sock_ops; 2492 extern struct request_sock_ops tcp6_request_sock_ops; 2493 2494 void tcp_v4_destroy_sock(struct sock *sk); 2495 2496 struct sk_buff *tcp_gso_segment(struct sk_buff *skb, 2497 netdev_features_t features); 2498 struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th); 2499 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb, 2500 struct tcphdr *th); 2501 INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); 2502 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); 2503 #ifdef CONFIG_INET 2504 void tcp_gro_complete(struct sk_buff *skb); 2505 #else 2506 static inline void tcp_gro_complete(struct sk_buff *skb) { } 2507 #endif 2508 2509 static inline void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, 2510 __be32 daddr) 2511 { 2512 struct tcphdr *th = tcp_hdr(skb); 2513 2514 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0); 2515 skb->csum_start = skb_transport_header(skb) - skb->head; 2516 skb->csum_offset = offsetof(struct tcphdr, check); 2517 } 2518 2519 static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) 2520 { 2521 struct net *net = sock_net((struct sock *)tp); 2522 u32 val; 2523 2524 val = READ_ONCE(tp->notsent_lowat); 2525 2526 return val ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat); 2527 } 2528 2529 bool tcp_stream_memory_free(const struct sock *sk, int wake); 2530 2531 #ifdef CONFIG_PROC_FS 2532 int tcp4_proc_init(void); 2533 void tcp4_proc_exit(void); 2534 #endif 2535 2536 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req); 2537 int tcp_conn_request(struct request_sock_ops *rsk_ops, 2538 const struct tcp_request_sock_ops *af_ops, 2539 struct sock *sk, struct sk_buff *skb); 2540 2541 /* TCP af-specific functions */ 2542 struct tcp_sock_af_ops { 2543 #ifdef CONFIG_TCP_MD5SIG 2544 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk, 2545 const struct sock *addr_sk); 2546 void (*calc_md5_hash)(char *location, 2547 const struct tcp_md5sig_key *md5, 2548 const struct sock *sk, 2549 const struct sk_buff *skb); 2550 int (*md5_parse)(struct sock *sk, 2551 int optname, 2552 sockptr_t optval, 2553 int optlen); 2554 #endif 2555 #ifdef CONFIG_TCP_AO 2556 int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen); 2557 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, 2558 struct sock *addr_sk, 2559 int sndid, int rcvid); 2560 void (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key, 2561 const struct sock *sk, 2562 __be32 sisn, __be32 disn, bool send); 2563 int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao, 2564 const struct sock *sk, const struct sk_buff *skb, 2565 const u8 *tkey, int hash_offset, u32 sne); 2566 #endif 2567 }; 2568 2569 struct tcp_request_sock_ops { 2570 u16 mss_clamp; 2571 #ifdef CONFIG_TCP_MD5SIG 2572 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk, 2573 const struct sock *addr_sk); 2574 void (*calc_md5_hash) (char *location, 2575 const struct tcp_md5sig_key *md5, 2576 const struct sock *sk, 2577 const struct sk_buff *skb); 2578 #endif 2579 #ifdef CONFIG_TCP_AO 2580 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, 2581 struct request_sock *req, 2582 int sndid, int rcvid); 2583 void (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk); 2584 int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt, 2585 struct request_sock *req, const struct sk_buff *skb, 2586 int hash_offset, u32 sne); 2587 #endif 2588 #ifdef CONFIG_SYN_COOKIES 2589 __u32 (*cookie_init_seq)(const struct sk_buff *skb, 2590 __u16 *mss); 2591 #endif 2592 struct dst_entry *(*route_req)(const struct sock *sk, 2593 struct sk_buff *skb, 2594 struct flowi *fl, 2595 struct request_sock *req, 2596 u32 tw_isn); 2597 union tcp_seq_and_ts_off (*init_seq_and_ts_off)( 2598 const struct net *net, 2599 const struct sk_buff *skb); 2600 int (*send_synack)(const struct sock *sk, struct dst_entry *dst, 2601 struct flowi *fl, struct request_sock *req, 2602 struct tcp_fastopen_cookie *foc, 2603 enum tcp_synack_type synack_type, 2604 struct sk_buff *syn_skb); 2605 }; 2606 2607 extern const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops; 2608 #if IS_ENABLED(CONFIG_IPV6) 2609 extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; 2610 #endif 2611 2612 #ifdef CONFIG_SYN_COOKIES 2613 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 2614 struct sk_buff *skb, __u16 *mss) 2615 { 2616 return ops->cookie_init_seq(skb, mss); 2617 } 2618 #else 2619 static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, 2620 struct sk_buff *skb, __u16 *mss) 2621 { 2622 return 0; 2623 } 2624 #endif 2625 2626 #ifdef CONFIG_SYN_COOKIES 2627 static inline void cookie_record_sent(const struct sock *sk) 2628 { 2629 tcp_synq_overflow(sk); 2630 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); 2631 } 2632 #else 2633 static inline void cookie_record_sent(const struct sock *sk) 2634 { 2635 } 2636 #endif 2637 2638 struct tcp_key { 2639 union { 2640 struct { 2641 struct tcp_ao_key *ao_key; 2642 char *traffic_key; 2643 u32 sne; 2644 u8 rcv_next; 2645 }; 2646 struct tcp_md5sig_key *md5_key; 2647 }; 2648 enum { 2649 TCP_KEY_NONE = 0, 2650 TCP_KEY_MD5, 2651 TCP_KEY_AO, 2652 } type; 2653 }; 2654 2655 static inline void tcp_get_current_key(const struct sock *sk, 2656 struct tcp_key *out) 2657 { 2658 #if defined(CONFIG_TCP_AO) || defined(CONFIG_TCP_MD5SIG) 2659 const struct tcp_sock *tp = tcp_sk(sk); 2660 #endif 2661 2662 #ifdef CONFIG_TCP_AO 2663 if (static_branch_unlikely(&tcp_ao_needed.key)) { 2664 struct tcp_ao_info *ao; 2665 2666 ao = rcu_dereference_protected(tp->ao_info, 2667 lockdep_sock_is_held(sk)); 2668 if (ao) { 2669 out->ao_key = READ_ONCE(ao->current_key); 2670 out->type = TCP_KEY_AO; 2671 return; 2672 } 2673 } 2674 #endif 2675 #ifdef CONFIG_TCP_MD5SIG 2676 if (static_branch_unlikely(&tcp_md5_needed.key) && 2677 rcu_access_pointer(tp->md5sig_info)) { 2678 out->md5_key = tp->af_specific->md5_lookup(sk, sk); 2679 if (out->md5_key) { 2680 out->type = TCP_KEY_MD5; 2681 return; 2682 } 2683 } 2684 #endif 2685 out->type = TCP_KEY_NONE; 2686 } 2687 2688 static inline bool tcp_key_is_md5(const struct tcp_key *key) 2689 { 2690 if (static_branch_tcp_md5()) 2691 return key->type == TCP_KEY_MD5; 2692 return false; 2693 } 2694 2695 static inline bool tcp_key_is_ao(const struct tcp_key *key) 2696 { 2697 if (static_branch_tcp_ao()) 2698 return key->type == TCP_KEY_AO; 2699 return false; 2700 } 2701 2702 int tcpv4_offload_init(void); 2703 2704 void tcp_v4_init(void); 2705 void tcp_init(void); 2706 2707 /* tcp_recovery.c */ 2708 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb); 2709 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); 2710 extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, 2711 u32 reo_wnd); 2712 extern bool tcp_rack_mark_lost(struct sock *sk); 2713 extern void tcp_rack_reo_timeout(struct sock *sk); 2714 2715 /* tcp_plb.c */ 2716 2717 /* 2718 * Scaling factor for fractions in PLB. For example, tcp_plb_update_state 2719 * expects cong_ratio which represents fraction of traffic that experienced 2720 * congestion over a single RTT. In order to avoid floating point operations, 2721 * this fraction should be mapped to (1 << TCP_PLB_SCALE) and passed in. 2722 */ 2723 #define TCP_PLB_SCALE 8 2724 2725 /* State for PLB (Protective Load Balancing) for a single TCP connection. */ 2726 struct tcp_plb_state { 2727 u8 consec_cong_rounds:5, /* consecutive congested rounds */ 2728 unused:3; 2729 u32 pause_until; /* jiffies32 when PLB can resume rerouting */ 2730 }; 2731 2732 static inline void tcp_plb_init(const struct sock *sk, 2733 struct tcp_plb_state *plb) 2734 { 2735 plb->consec_cong_rounds = 0; 2736 plb->pause_until = 0; 2737 } 2738 void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb, 2739 const int cong_ratio); 2740 void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb); 2741 void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb); 2742 2743 static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str) 2744 { 2745 WARN_ONCE(cond, 2746 "%scwn:%u out:%u sacked:%u lost:%u retrans:%u tlp_high_seq:%u sk_state:%u ca_state:%u advmss:%u mss_cache:%u pmtu:%u\n", 2747 str, 2748 tcp_snd_cwnd(tcp_sk(sk)), 2749 tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out, 2750 tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out, 2751 tcp_sk(sk)->tlp_high_seq, sk->sk_state, 2752 inet_csk(sk)->icsk_ca_state, 2753 tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache, 2754 inet_csk(sk)->icsk_pmtu_cookie); 2755 } 2756 2757 /* At how many usecs into the future should the RTO fire? */ 2758 static inline s64 tcp_rto_delta_us(const struct sock *sk) 2759 { 2760 const struct sk_buff *skb = tcp_rtx_queue_head(sk); 2761 u32 rto = inet_csk(sk)->icsk_rto; 2762 2763 if (likely(skb)) { 2764 u64 rto_time_stamp_us = tcp_skb_timestamp_us(skb) + jiffies_to_usecs(rto); 2765 2766 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; 2767 } else { 2768 tcp_warn_once(sk, 1, "rtx queue empty: "); 2769 return jiffies_to_usecs(rto); 2770 } 2771 2772 } 2773 2774 /* 2775 * Save and compile IPv4 options, return a pointer to it 2776 */ 2777 static inline struct ip_options_rcu *tcp_v4_save_options(struct net *net, 2778 struct sk_buff *skb) 2779 { 2780 const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt; 2781 struct ip_options_rcu *dopt = NULL; 2782 2783 if (opt->optlen) { 2784 int opt_size = sizeof(*dopt) + opt->optlen; 2785 2786 dopt = kmalloc(opt_size, GFP_ATOMIC); 2787 if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) { 2788 kfree(dopt); 2789 dopt = NULL; 2790 } 2791 } 2792 return dopt; 2793 } 2794 2795 /* locally generated TCP pure ACKs have skb->truesize == 2 2796 * (check tcp_send_ack() in net/ipv4/tcp_output.c ) 2797 * This is much faster than dissecting the packet to find out. 2798 * (Think of GRE encapsulations, IPv4, IPv6, ...) 2799 */ 2800 static inline bool skb_is_tcp_pure_ack(const struct sk_buff *skb) 2801 { 2802 return skb->truesize == 2; 2803 } 2804 2805 static inline void skb_set_tcp_pure_ack(struct sk_buff *skb) 2806 { 2807 skb->truesize = 2; 2808 } 2809 2810 static inline int tcp_inq(struct sock *sk) 2811 { 2812 struct tcp_sock *tp = tcp_sk(sk); 2813 int answ; 2814 2815 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { 2816 answ = 0; 2817 } else if (sock_flag(sk, SOCK_URGINLINE) || 2818 !tp->urg_data || 2819 before(tp->urg_seq, tp->copied_seq) || 2820 !before(tp->urg_seq, tp->rcv_nxt)) { 2821 2822 answ = tp->rcv_nxt - tp->copied_seq; 2823 2824 /* Subtract 1, if FIN was received */ 2825 if (answ && sock_flag(sk, SOCK_DONE)) 2826 answ--; 2827 } else { 2828 answ = tp->urg_seq - tp->copied_seq; 2829 } 2830 2831 return answ; 2832 } 2833 2834 int tcp_peek_len(struct socket *sock); 2835 2836 static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) 2837 { 2838 u16 segs_in; 2839 2840 segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); 2841 2842 /* We update these fields while other threads might 2843 * read them from tcp_get_info() 2844 */ 2845 WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); 2846 if (skb->len > tcp_hdrlen(skb)) 2847 WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); 2848 } 2849 2850 /* 2851 * TCP listen path runs lockless. 2852 * We forced "struct sock" to be const qualified to make sure 2853 * we don't modify one of its field by mistake. 2854 * Here, we increment sk_drops which is an atomic_t, so we can safely 2855 * make sock writable again. 2856 */ 2857 static inline void tcp_listendrop(const struct sock *sk) 2858 { 2859 sk_drops_inc((struct sock *)sk); 2860 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); 2861 } 2862 2863 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); 2864 2865 /* 2866 * Interface for adding Upper Level Protocols over TCP 2867 */ 2868 2869 #define TCP_ULP_NAME_MAX 16 2870 #define TCP_ULP_MAX 128 2871 #define TCP_ULP_BUF_MAX (TCP_ULP_NAME_MAX*TCP_ULP_MAX) 2872 2873 struct tcp_ulp_ops { 2874 struct list_head list; 2875 2876 /* initialize ulp */ 2877 int (*init)(struct sock *sk); 2878 /* update ulp */ 2879 void (*update)(struct sock *sk, struct proto *p, 2880 void (*write_space)(struct sock *sk)); 2881 /* cleanup ulp */ 2882 void (*release)(struct sock *sk); 2883 /* diagnostic */ 2884 int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin); 2885 size_t (*get_info_size)(const struct sock *sk, bool net_admin); 2886 /* clone ulp */ 2887 void (*clone)(const struct request_sock *req, struct sock *newsk, 2888 const gfp_t priority); 2889 2890 char name[TCP_ULP_NAME_MAX]; 2891 struct module *owner; 2892 }; 2893 int tcp_register_ulp(struct tcp_ulp_ops *type); 2894 void tcp_unregister_ulp(struct tcp_ulp_ops *type); 2895 int tcp_set_ulp(struct sock *sk, const char *name); 2896 void tcp_get_available_ulp(char *buf, size_t len); 2897 void tcp_cleanup_ulp(struct sock *sk); 2898 void tcp_update_ulp(struct sock *sk, struct proto *p, 2899 void (*write_space)(struct sock *sk)); 2900 2901 #define MODULE_ALIAS_TCP_ULP(name) \ 2902 MODULE_INFO(alias, name); \ 2903 MODULE_INFO(alias, "tcp-ulp-" name) 2904 2905 #ifdef CONFIG_NET_SOCK_MSG 2906 struct sk_msg; 2907 struct sk_psock; 2908 2909 #ifdef CONFIG_BPF_SYSCALL 2910 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); 2911 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk); 2912 #ifdef CONFIG_BPF_STREAM_PARSER 2913 struct strparser; 2914 int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc, 2915 sk_read_actor_t recv_actor); 2916 #endif /* CONFIG_BPF_STREAM_PARSER */ 2917 #endif /* CONFIG_BPF_SYSCALL */ 2918 2919 #ifdef CONFIG_INET 2920 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb); 2921 #else 2922 static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) 2923 { 2924 } 2925 #endif 2926 2927 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress, 2928 struct sk_msg *msg, u32 bytes, int flags); 2929 #endif /* CONFIG_NET_SOCK_MSG */ 2930 2931 #if !defined(CONFIG_BPF_SYSCALL) || !defined(CONFIG_NET_SOCK_MSG) 2932 static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) 2933 { 2934 } 2935 #endif 2936 2937 #ifdef CONFIG_CGROUP_BPF 2938 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 2939 struct sk_buff *skb, 2940 unsigned int end_offset) 2941 { 2942 skops->skb = skb; 2943 skops->skb_data_end = skb->data + end_offset; 2944 } 2945 #else 2946 static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, 2947 struct sk_buff *skb, 2948 unsigned int end_offset) 2949 { 2950 } 2951 #endif 2952 2953 /* Call BPF_SOCK_OPS program that returns an int. If the return value 2954 * is < 0, then the BPF op failed (for example if the loaded BPF 2955 * program does not support the chosen operation or there is no BPF 2956 * program loaded). 2957 */ 2958 #ifdef CONFIG_BPF 2959 static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) 2960 { 2961 struct bpf_sock_ops_kern sock_ops; 2962 int ret; 2963 2964 memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp)); 2965 if (sk_fullsock(sk)) { 2966 sock_ops.is_fullsock = 1; 2967 sock_ops.is_locked_tcp_sock = 1; 2968 sock_owned_by_me(sk); 2969 } 2970 2971 sock_ops.sk = sk; 2972 sock_ops.op = op; 2973 if (nargs > 0) 2974 memcpy(sock_ops.args, args, nargs * sizeof(*args)); 2975 2976 ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops); 2977 if (ret == 0) 2978 ret = sock_ops.reply; 2979 else 2980 ret = -1; 2981 return ret; 2982 } 2983 2984 #define tcp_call_bpf(sk, op) \ 2985 ({ \ 2986 int __ret = 0; \ 2987 if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) { \ 2988 __ret = __tcp_call_bpf(sk, op, 0, NULL); \ 2989 } \ 2990 __ret; \ 2991 }) 2992 2993 #define tcp_call_bpf_flag(sk, op, ...) \ 2994 do { \ 2995 if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && \ 2996 BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) { \ 2997 u32 __args[] = { __VA_ARGS__ }; \ 2998 __tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args); \ 2999 } \ 3000 } while (0) 3001 3002 static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) 3003 { 3004 tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; 3005 WRITE_ONCE(tcp_sk(sk)->bpf_tcp_ops_flags, 0); 3006 } 3007 3008 #else 3009 static inline int tcp_call_bpf(struct sock *sk, int op) 3010 { 3011 return -EPERM; 3012 } 3013 3014 #define tcp_call_bpf_flag(sk, op, ...) do { } while (0) 3015 3016 static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) 3017 { 3018 } 3019 3020 #endif 3021 3022 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_CGROUP_BPF) 3023 3024 struct bpf_tcp_ops { 3025 /* Should return the initial SYN (active open) or SYN-ACK (passive open) 3026 * retransmission timeout. Return the timeout in jiffies, or <= 0 for 3027 * the kernel default. 3028 * 3029 * @req: request_sock on the passive (synack) path; NULL otherwise. 3030 */ 3031 int (*timeout_init)(struct sock *sk, struct request_sock *req); 3032 3033 /* Should return the initial advertised receive window, in packets, 3034 * or < 0 for the kernel default. @req as in timeout_init(). 3035 */ 3036 int (*rwnd_init)(struct sock *sk, struct request_sock *req); 3037 3038 /* Called when an active connection becomes established. 3039 * @skb is the SYNACK that completed the 3WHS, or NULL for a 3040 * TCP_REPAIR socket (tcp_finish_connect() with no skb). 3041 */ 3042 void (*active_established)(struct sock *sk, struct sk_buff *skb__nullable); 3043 3044 /* Called when a passive connection becomes established. 3045 * @skb is the ACK that completed the 3WHS. 3046 */ 3047 void (*passive_established)(struct sock *sk, struct sk_buff *skb); 3048 3049 /* Called when the retransmission timer fires. */ 3050 void (*rto)(struct sock *sk); 3051 3052 /* Called on every RTT sample if BPF_TCP_OPS_FLAG_RTT is enabled. 3053 * @mrtt: the measured RTT, in microseconds. 3054 * @srtt: the updated smoothed RTT. 3055 */ 3056 void (*rtt)(struct sock *sk, long mrtt, u32 srtt); 3057 3058 /* Called when the connection changes TCP state. 3059 * @state: the new state (one of the TCP_* states). 3060 */ 3061 void (*set_state)(struct sock *sk, int state); 3062 3063 /* Called when an skb is retransmitted. 3064 * @skb: the retransmitted skb. 3065 * @err: tcp_transmit_skb() return value (0 on success). 3066 */ 3067 void (*retrans)(struct sock *sk, struct sk_buff *skb, int err); 3068 3069 /* Called right before an active connection is initialized. */ 3070 void (*connect)(struct sock *sk); 3071 3072 /* Called on listen(2), right after the socket enters TCP_LISTEN. */ 3073 void (*listen)(struct sock *sk); 3074 3075 /* 3076 * Parse the TCP header options of an incoming skb received on an 3077 * established connection. Use bpf_dynptr_from_skb()/bpf_skb_load_bytes() 3078 * to access the options. 3079 * 3080 * Called if BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_ALL is enabled, or if 3081 * BPF_TCP_OPS_FLAG_PARSE_HDR_OPT_UNKNOWN is enabled and an unknown 3082 * option is received. 3083 */ 3084 void (*parse_hdr)(struct sock *sk, struct sk_buff *skb); 3085 3086 /* 3087 * Reserve space in the outgoing TCP header for options to be written 3088 * later by write_hdr_opt(). Call bpf_reserve_hdr_opt() to reserve bytes. 3089 * 3090 * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. 3091 * 3092 * @skb: outgoing packet. NULL when called from tcp_current_mss() 3093 * (MSS sizing). 3094 * @req: request_sock on the synack path; NULL otherwise. 3095 * @syn_skb: incoming SYN on the synack path; NULL otherwise. 3096 * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. 3097 * @remaining: pointer to the size of space still available; cast it 3098 * using bpf_rdonly_cast() before dereferencing. 3099 */ 3100 void (*hdr_opt_len)(struct sock *sk, struct sk_buff *skb, 3101 struct request_sock *req, struct sk_buff *syn_skb, 3102 enum tcp_synack_type synack_type, 3103 unsigned int *remaining); 3104 3105 /* 3106 * Write header options into the space reserved earlier by hdr_opt_len(). 3107 * Use bpf_store_hdr_opt() to write; it appends within the reserved window 3108 * shared with legacy SOCKOPS. 3109 * 3110 * Called if BPF_TCP_OPS_FLAG_WRITE_HDR_OPT is enabled. 3111 * 3112 * @skb: outgoing packet. 3113 * @req: request_sock on the synack path; NULL otherwise. 3114 * @syn_skb: incoming SYN on the synack path; NULL otherwise. 3115 * @synack_type: TCP_SYNACK_COOKIE indicates a stateless syncookie. 3116 * @opt_off: offset in the outgoing @skb's TCP header where the 3117 * bpf_tcp_ops portion of the reserved window begins, i.e. after 3118 * the kernel and legacy options. 3119 */ 3120 void (*write_hdr_opt)(struct sock *sk, struct sk_buff *skb, 3121 struct request_sock *req, struct sk_buff *syn_skb, 3122 enum tcp_synack_type synack_type, 3123 u32 opt_off); 3124 3125 /* 3126 * Called when an incoming skb is enqueued to sk->sk_receive_queue 3127 * if BPF_TCP_OPS_FLAG_RCVQ is enabled. 3128 */ 3129 void (*enqueue_rcvq)(struct sock *sk, struct sk_buff *skb); 3130 3131 /* 3132 * Called after data is dequeued from sk->sk_receive_queue 3133 * if BPF_TCP_OPS_FLAG_RCVQ is enabled. 3134 */ 3135 void (*dequeue_rcvq)(struct sock *sk); 3136 }; 3137 3138 #define __bpf_tcp_ops_call(op, sk, ...) \ 3139 do { \ 3140 const struct bpf_prog_array_item *item; \ 3141 const struct bpf_tcp_ops *tcp_ops; \ 3142 struct cgroup *cgrp; \ 3143 \ 3144 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); \ 3145 rcu_read_lock_dont_migrate(); \ 3146 bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ 3147 CGROUP_TCP_SOCK_OPS) { \ 3148 if (tcp_ops->op) \ 3149 tcp_ops->op(sk, ##__VA_ARGS__); \ 3150 } \ 3151 rcu_read_unlock_migrate(); \ 3152 } while (0) 3153 3154 #define bpf_tcp_ops_call(op, sk, ...) \ 3155 do { \ 3156 if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) \ 3157 __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ 3158 } while (0) 3159 3160 #define bpf_tcp_ops_call_flag(op, flag, sk, ...) \ 3161 do { \ 3162 if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS) && \ 3163 BPF_TCP_OPS_TEST_FLAG(tcp_sk(sk), flag)) \ 3164 __bpf_tcp_ops_call(op, sk, ##__VA_ARGS__); \ 3165 } while (0) 3166 3167 #define bpf_tcp_ops_call_int(op, init_retval, sk, ...) \ 3168 ({ \ 3169 int __retval = (init_retval); \ 3170 if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS)) { \ 3171 const struct bpf_prog_array_item *item; \ 3172 const struct bpf_tcp_ops *tcp_ops; \ 3173 struct bpf_tramp_run_ctx run_ctx; \ 3174 struct bpf_run_ctx *old_run_ctx; \ 3175 struct sock *__sk = sk_to_full_sk(sk); \ 3176 struct request_sock *req = NULL; \ 3177 struct cgroup *cgrp; \ 3178 \ 3179 if (__sk) { \ 3180 run_ctx.retval = (init_retval); \ 3181 cgrp = sock_cgroup_ptr(&__sk->sk_cgrp_data); \ 3182 if (!sk_fullsock(sk)) \ 3183 req = (struct request_sock *)sk; \ 3184 rcu_read_lock_dont_migrate(); \ 3185 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);\ 3186 bpf_cgroup_struct_ops_foreach(tcp_ops, item, cgrp, \ 3187 CGROUP_TCP_SOCK_OPS) { \ 3188 if (tcp_ops->op) \ 3189 run_ctx.retval = tcp_ops->op(__sk, req, ##__VA_ARGS__); \ 3190 } \ 3191 bpf_reset_run_ctx(old_run_ctx); \ 3192 rcu_read_unlock_migrate(); \ 3193 __retval = run_ctx.retval; \ 3194 } \ 3195 } \ 3196 __retval; \ 3197 }) 3198 3199 #else 3200 #define __bpf_tcp_ops_call(op, sk, ...) do { } while (0) 3201 #define bpf_tcp_ops_call(op, sk, ...) do { } while (0) 3202 #define bpf_tcp_ops_call_flag(op, flag, sk, ...) do { } while (0) 3203 #define bpf_tcp_ops_call_int(op, init_retval, sk, ...) (init_retval) 3204 #endif 3205 3206 static inline u32 tcp_timeout_init(struct sock *sk) 3207 { 3208 int timeout; 3209 3210 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT); 3211 timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk); 3212 if (timeout <= 0) 3213 timeout = TCP_TIMEOUT_INIT; 3214 return min_t(int, timeout, TCP_RTO_MAX); 3215 } 3216 3217 static inline u32 tcp_rwnd_init_bpf(struct sock *sk) 3218 { 3219 int rwnd; 3220 3221 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT); 3222 rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk); 3223 if (rwnd < 0) 3224 rwnd = 0; 3225 return rwnd; 3226 } 3227 3228 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) 3229 { 3230 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1); 3231 } 3232 3233 static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) 3234 { 3235 tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); 3236 bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt); 3237 } 3238 3239 #if IS_ENABLED(CONFIG_SMC) 3240 extern struct static_key_false tcp_have_smc; 3241 #endif 3242 3243 #if IS_ENABLED(CONFIG_TLS_DEVICE) 3244 void clean_acked_data_enable(struct tcp_sock *tp, 3245 void (*cad)(struct sock *sk, u32 ack_seq)); 3246 void clean_acked_data_disable(struct tcp_sock *tp); 3247 void clean_acked_data_flush(void); 3248 #endif 3249 3250 DECLARE_STATIC_KEY_FALSE(tcp_tx_delay_enabled); 3251 static inline void tcp_add_tx_delay(struct sk_buff *skb, 3252 const struct tcp_sock *tp) 3253 { 3254 if (static_branch_unlikely(&tcp_tx_delay_enabled)) 3255 skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC; 3256 } 3257 3258 /* Compute Earliest Departure Time for some control packets 3259 * like ACK or RST for TIME_WAIT or non ESTABLISHED sockets. 3260 */ 3261 static inline u64 tcp_transmit_time(const struct sock *sk) 3262 { 3263 if (static_branch_unlikely(&tcp_tx_delay_enabled)) { 3264 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? 3265 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; 3266 3267 return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC; 3268 } 3269 return 0; 3270 } 3271 3272 static inline int tcp_parse_auth_options(const struct tcphdr *th, 3273 const u8 **md5_hash, const struct tcp_ao_hdr **aoh) 3274 { 3275 const u8 *md5_tmp, *ao_tmp; 3276 int ret; 3277 3278 ret = tcp_do_parse_auth_options(th, &md5_tmp, &ao_tmp); 3279 if (ret) 3280 return ret; 3281 3282 if (md5_hash) 3283 *md5_hash = md5_tmp; 3284 3285 if (aoh) { 3286 if (!ao_tmp) 3287 *aoh = NULL; 3288 else 3289 *aoh = (struct tcp_ao_hdr *)(ao_tmp - 2); 3290 } 3291 3292 return 0; 3293 } 3294 3295 static inline bool tcp_ao_required(struct sock *sk, const void *saddr, 3296 int family, int l3index, bool stat_inc) 3297 { 3298 #ifdef CONFIG_TCP_AO 3299 struct tcp_ao_info *ao_info; 3300 struct tcp_ao_key *ao_key; 3301 3302 if (!static_branch_unlikely(&tcp_ao_needed.key)) 3303 return false; 3304 3305 ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info, 3306 lockdep_sock_is_held(sk)); 3307 if (!ao_info) 3308 return false; 3309 3310 ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1); 3311 if (ao_info->ao_required || ao_key) { 3312 if (stat_inc) { 3313 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED); 3314 atomic64_inc(&ao_info->counters.ao_required); 3315 } 3316 return true; 3317 } 3318 #endif 3319 return false; 3320 } 3321 3322 enum skb_drop_reason tcp_inbound_hash(struct sock *sk, 3323 const struct request_sock *req, const struct sk_buff *skb, 3324 const void *saddr, const void *daddr, 3325 int family, int dif, int sdif); 3326 3327 static inline int tcp_recv_should_stop(struct sock *sk) 3328 { 3329 return READ_ONCE(sk->sk_err) || 3330 sk->sk_state == TCP_CLOSE || 3331 (sk->sk_shutdown & RCV_SHUTDOWN) || 3332 signal_pending(current); 3333 } 3334 3335 INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off 3336 tcp_v4_init_seq_and_ts_off(const struct net *net, 3337 const struct sk_buff *skb)); 3338 INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off 3339 tcp_v6_init_seq_and_ts_off(const struct net *net, 3340 const struct sk_buff *skb)); 3341 #endif /* _TCP_H */ 3342