389 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)  in tcp_compute_delivery_rate()  argument
391 	u32 rate = READ_ONCE(tp->rate_delivered);  in tcp_compute_delivery_rate()
392 	u32 intv = READ_ONCE(tp->rate_interval_us);  in tcp_compute_delivery_rate()
396 		rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;  in tcp_compute_delivery_rate()
405 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_md5_destruct_sock()  local
407 	if (tp->md5sig_info) {  in tcp_md5_destruct_sock()
410 		kfree(rcu_replace_pointer(tp->md5sig_info, NULL, 1));  in tcp_md5_destruct_sock()
424 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_init_sock()  local
427 	tp->out_of_order_queue = RB_ROOT;  in tcp_init_sock()
430 	INIT_LIST_HEAD(&tp->tsq_node);  in tcp_init_sock()
431 	INIT_LIST_HEAD(&tp->tsorted_sent_queue);  in tcp_init_sock()
441 	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);  in tcp_init_sock()
442 	minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);  in tcp_init_sock()
449 	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);  in tcp_init_sock()
452 	tp->app_limited = ~0U;  in tcp_init_sock()
453 	tp->rate_app_limited = 1;  in tcp_init_sock()
458 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;  in tcp_init_sock()
459 	tp->snd_cwnd_clamp = ~0;  in tcp_init_sock()
460 	tp->mss_cache = TCP_MSS_DEFAULT;  in tcp_init_sock()
462 	tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);  in tcp_init_sock()
465 	tp->tsoffset = 0;  in tcp_init_sock()
466 	tp->rack.reo_wnd_steps = 1;  in tcp_init_sock()
512 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_stream_memory_free()  local
513 	u32 notsent_bytes = READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt);  in tcp_stream_memory_free()
515 	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);  in tcp_stream_memory_free()
537 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_poll()  local
589 	    (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {  in tcp_poll()
591 		u16 urg_data = READ_ONCE(tp->urg_data);  in tcp_poll()
594 		    READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&  in tcp_poll()
642 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_ioctl()  local
656 		answ = READ_ONCE(tp->urg_data) &&  in tcp_ioctl()
657 		       READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);  in tcp_ioctl()
666 			answ = READ_ONCE(tp->write_seq) - tp->snd_una;  in tcp_ioctl()
675 			answ = READ_ONCE(tp->write_seq) -  in tcp_ioctl()
676 			       READ_ONCE(tp->snd_nxt);  in tcp_ioctl()
686 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)  in tcp_mark_push()  argument
689 	tp->pushed_seq = tp->write_seq;  in tcp_mark_push()
692 static inline bool forced_push(const struct tcp_sock *tp)  in forced_push()  argument
694 	return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));  in forced_push()
699 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_skb_entail()  local
702 	tcb->seq     = tcb->end_seq = tp->write_seq;  in tcp_skb_entail()
709 	if (tp->nonagle & TCP_NAGLE_PUSH)  in tcp_skb_entail()
710 		tp->nonagle &= ~TCP_NAGLE_PUSH;  in tcp_skb_entail()
715 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)  in tcp_mark_urg()  argument
718 		tp->snd_up = tp->write_seq;  in tcp_mark_urg()
744 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_push()  local
750 	if (!(flags & MSG_MORE) || forced_push(tp))  in tcp_push()
751 		tcp_mark_push(tp, skb);  in tcp_push()
753 	tcp_mark_urg(tp, flags);  in tcp_push()
961 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_xmit_size_goal()  local
968 	new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size);  in tcp_xmit_size_goal()
971 	size_goal = tp->gso_segs * mss_now;  in tcp_xmit_size_goal()
974 		tp->gso_segs = min_t(u16, new_size_goal / mss_now,  in tcp_xmit_size_goal()
976 		size_goal = tp->gso_segs * mss_now;  in tcp_xmit_size_goal()
1044 void tcp_free_fastopen_req(struct tcp_sock *tp)  in tcp_free_fastopen_req()  argument
1046 	if (tp->fastopen_req) {  in tcp_free_fastopen_req()
1047 		kfree(tp->fastopen_req);  in tcp_free_fastopen_req()
1048 		tp->fastopen_req = NULL;  in tcp_free_fastopen_req()
1055 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_sendmsg_fastopen()  local
1065 	if (tp->fastopen_req)  in tcp_sendmsg_fastopen()
1068 	tp->fastopen_req = kzalloc_obj(struct tcp_fastopen_request,  in tcp_sendmsg_fastopen()
1070 	if (unlikely(!tp->fastopen_req))  in tcp_sendmsg_fastopen()
1072 	tp->fastopen_req->data = msg;  in tcp_sendmsg_fastopen()
1073 	tp->fastopen_req->size = size;  in tcp_sendmsg_fastopen()
1074 	tp->fastopen_req->uarg = uarg;  in tcp_sendmsg_fastopen()
1091 	if (tp->fastopen_req) {  in tcp_sendmsg_fastopen()
1092 		*copied = tp->fastopen_req->copied;  in tcp_sendmsg_fastopen()
1093 		tcp_free_fastopen_req(tp);  in tcp_sendmsg_fastopen()
1102 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_rate_check_app_limited()  local
1105 	    tp->write_seq - tp->snd_nxt < tp->mss_cache &&  in tcp_rate_check_app_limited()
1109 	    tcp_packets_in_flight(tp) < tcp_snd_cwnd(tp) &&  in tcp_rate_check_app_limited()
1111 	    tp->lost_out <= tp->retrans_out)  in tcp_rate_check_app_limited()
1112 		tp->app_limited =  in tcp_rate_check_app_limited()
1113 			(tp->delivered + tcp_packets_in_flight(tp)) ? : 1;  in tcp_rate_check_app_limited()
1120 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_sendmsg_locked()  local
1181 	    !tp->repair) {  in tcp_sendmsg_locked()
1204 	if (unlikely(tp->repair)) {  in tcp_sendmsg_locked()
1205 		if (tp->repair_queue == TCP_RECV_QUEUE) {  in tcp_sendmsg_locked()
1211 		if (tp->repair_queue == TCP_NO_QUEUE)  in tcp_sendmsg_locked()
1274 			if (tp->repair)  in tcp_sendmsg_locked()
1293 					tcp_mark_push(tp, skb);  in tcp_sendmsg_locked()
1343 				tcp_mark_push(tp, skb);  in tcp_sendmsg_locked()
1360 					tcp_mark_push(tp, skb);  in tcp_sendmsg_locked()
1377 		WRITE_ONCE(tp->write_seq, tp->write_seq + copy);  in tcp_sendmsg_locked()
1388 		if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))  in tcp_sendmsg_locked()
1391 		if (forced_push(tp)) {  in tcp_sendmsg_locked()
1392 			tcp_mark_push(tp, skb);  in tcp_sendmsg_locked()
1415 		tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);  in tcp_sendmsg_locked()
1462 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_splice_eof()  local
1470 	tcp_push(sk, 0, mss_now, tp->nonagle, size_goal);  in tcp_splice_eof()
1481 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_recv_urg()  local
1484 	if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||  in tcp_recv_urg()
1485 	    tp->urg_data == TCP_URG_READ)  in tcp_recv_urg()
1491 	if (tp->urg_data & TCP_URG_VALID) {  in tcp_recv_urg()
1493 		char c = tp->urg_data;  in tcp_recv_urg()
1496 			WRITE_ONCE(tp->urg_data, TCP_URG_READ);  in tcp_recv_urg()
1554 	struct tcp_sock *tp = tcp_sk(sk);  in __tcp_cleanup_rbuf()  local
1561 		    tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||  in __tcp_cleanup_rbuf()
1583 		__u32 rcv_window_now = tcp_receive_window(tp);  in __tcp_cleanup_rbuf()
1586 		if (2*rcv_window_now <= tp->window_clamp) {  in __tcp_cleanup_rbuf()
1599 		tcp_mstamp_refresh(tp);  in __tcp_cleanup_rbuf()
1607 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_cleanup_rbuf()  local
1609 	WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),  in tcp_cleanup_rbuf()
1611 	     tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);  in tcp_cleanup_rbuf()
1668 	struct tcp_sock *tp = tcp_sk(sk);  in __tcp_read_sock()  local
1682 			if (unlikely(tp->urg_data)) {  in __tcp_read_sock()
1683 				u32 urg_offset = tp->urg_seq - seq;  in __tcp_read_sock()
1787 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_read_done()  local
1788 	u32 seq = tp->copied_seq;  in tcp_read_done()
1814 	WRITE_ONCE(tp->copied_seq, seq);  in tcp_read_done()
1832 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_set_rcvlowat()  local
1852 		if (tp->window_clamp && tp->window_clamp < val)  in tcp_set_rcvlowat()
1853 			WRITE_ONCE(tp->window_clamp, val);  in tcp_set_rcvlowat()
2204 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_zerocopy_receive()  local
2209 	u32 seq = tp->copied_seq;  in tcp_zerocopy_receive()
2328 		WRITE_ONCE(tp->copied_seq, seq);  in tcp_zerocopy_receive()
2416 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_inq_hint()  local
2417 	u32 copied_seq = READ_ONCE(tp->copied_seq);  in tcp_inq_hint()
2418 	u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);  in tcp_inq_hint()
2422 	if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {  in tcp_inq_hint()
2424 		inq = tp->rcv_nxt - tp->copied_seq;  in tcp_inq_hint()
2660 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_recvmsg_locked()  local
2677 	if (tp->recvmsg_inq)  in tcp_recvmsg_locked()
2685 	if (unlikely(tp->repair)) {  in tcp_recvmsg_locked()
2690 		if (tp->repair_queue == TCP_SEND_QUEUE)  in tcp_recvmsg_locked()
2694 		if (tp->repair_queue == TCP_NO_QUEUE)  in tcp_recvmsg_locked()
2700 	seq = &tp->copied_seq;  in tcp_recvmsg_locked()
2703 		peek_seq = tp->copied_seq + peek_offset;  in tcp_recvmsg_locked()
2713 		if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {  in tcp_recvmsg_locked()
2732 				 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,  in tcp_recvmsg_locked()
2747 			     *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);  in tcp_recvmsg_locked()
2803 		    (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) {  in tcp_recvmsg_locked()
2807 			peek_seq = tp->copied_seq + peek_offset;  in tcp_recvmsg_locked()
2818 		if (unlikely(tp->urg_data)) {  in tcp_recvmsg_locked()
2819 			u32 urg_offset = tp->urg_seq - *seq;  in tcp_recvmsg_locked()
2884 		if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {  in tcp_recvmsg_locked()
2885 			WRITE_ONCE(tp->urg_data, 0);  in tcp_recvmsg_locked()
3258 		struct tcp_sock *tp = tcp_sk(sk);  in __tcp_close()  local
3259 		if (READ_ONCE(tp->linger2) < 0) {  in __tcp_close()
3367 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_disconnect()  local
3378 	} else if (unlikely(tp->repair)) {  in tcp_disconnect()
3383 	} else if (tp->snd_nxt != tp->write_seq &&  in tcp_disconnect()
3396 	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);  in tcp_disconnect()
3397 	WRITE_ONCE(tp->urg_data, 0);  in tcp_disconnect()
3401 	skb_rbtree_purge(&tp->out_of_order_queue);  in tcp_disconnect()
3409 	tp->srtt_us = 0;  in tcp_disconnect()
3410 	tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);  in tcp_disconnect()
3411 	tp->rcv_rtt_last_tsecr = 0;  in tcp_disconnect()
3413 	seq = tp->write_seq + tp->max_window + 2;  in tcp_disconnect()
3416 	WRITE_ONCE(tp->write_seq, seq);  in tcp_disconnect()
3424 	WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH);  in tcp_disconnect()
3425 	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);  in tcp_disconnect()
3426 	tp->snd_cwnd_cnt = 0;  in tcp_disconnect()
3427 	tp->is_cwnd_limited = 0;  in tcp_disconnect()
3428 	tp->max_packets_out = 0;  in tcp_disconnect()
3429 	tp->window_clamp = 0;  in tcp_disconnect()
3430 	tp->delivered = 0;  in tcp_disconnect()
3431 	tp->delivered_ce = 0;  in tcp_disconnect()
3432 	tp->accecn_fail_mode = 0;  in tcp_disconnect()
3433 	tp->saw_accecn_opt = TCP_ACCECN_OPT_NOT_SEEN;  in tcp_disconnect()
3434 	tcp_accecn_init_counters(tp);  in tcp_disconnect()
3435 	tp->prev_ecnfield = 0;  in tcp_disconnect()
3436 	tp->accecn_opt_tstamp = 0;  in tcp_disconnect()
3437 	tp->pkts_acked_ewma = 0;  in tcp_disconnect()
3443 	tp->is_sack_reneg = 0;  in tcp_disconnect()
3444 	tcp_clear_retrans(tp);  in tcp_disconnect()
3445 	tp->total_retrans = 0;  in tcp_disconnect()
3451 	memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));  in tcp_disconnect()
3454 	tcp_saved_syn_free(tp);  in tcp_disconnect()
3455 	tp->compressed_ack = 0;  in tcp_disconnect()
3456 	tp->segs_in = 0;  in tcp_disconnect()
3457 	tp->segs_out = 0;  in tcp_disconnect()
3458 	tp->bytes_sent = 0;  in tcp_disconnect()
3459 	tp->bytes_acked = 0;  in tcp_disconnect()
3460 	tp->bytes_received = 0;  in tcp_disconnect()
3461 	tp->bytes_retrans = 0;  in tcp_disconnect()
3462 	tp->data_segs_in = 0;  in tcp_disconnect()
3463 	tp->data_segs_out = 0;  in tcp_disconnect()
3464 	tp->duplicate_sack[0].start_seq = 0;  in tcp_disconnect()
3465 	tp->duplicate_sack[0].end_seq = 0;  in tcp_disconnect()
3466 	tp->dsack_dups = 0;  in tcp_disconnect()
3467 	tp->reord_seen = 0;  in tcp_disconnect()
3468 	tp->retrans_out = 0;  in tcp_disconnect()
3469 	tp->sacked_out = 0;  in tcp_disconnect()
3470 	tp->tlp_high_seq = 0;  in tcp_disconnect()
3471 	tp->last_oow_ack_time = 0;  in tcp_disconnect()
3472 	tp->plb_rehash = 0;  in tcp_disconnect()
3474 	tp->app_limited = ~0U;  in tcp_disconnect()
3475 	tp->rate_app_limited = 1;  in tcp_disconnect()
3476 	tp->rack.mstamp = 0;  in tcp_disconnect()
3477 	tp->rack.advanced = 0;  in tcp_disconnect()
3478 	tp->rack.reo_wnd_steps = 1;  in tcp_disconnect()
3479 	tp->rack.last_delivered = 0;  in tcp_disconnect()
3480 	tp->rack.reo_wnd_persist = 0;  in tcp_disconnect()
3481 	tp->rack.dsack_seen = 0;  in tcp_disconnect()
3482 	tp->syn_data_acked = 0;  in tcp_disconnect()
3483 	tp->syn_fastopen_child = 0;  in tcp_disconnect()
3484 	tp->rx_opt.saw_tstamp = 0;  in tcp_disconnect()
3485 	tp->rx_opt.dsack = 0;  in tcp_disconnect()
3486 	tp->rx_opt.num_sacks = 0;  in tcp_disconnect()
3487 	tp->rcv_ooopack = 0;  in tcp_disconnect()
3491 	req = rcu_dereference_protected(tp->fastopen_rsk,  in tcp_disconnect()
3495 	tcp_free_fastopen_req(tp);  in tcp_disconnect()
3497 	tp->fastopen_client_fail = 0;  in tcp_disconnect()
3517 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)  in tcp_repair_set_window()  argument
3521 	if (!tp->repair)  in tcp_repair_set_window()
3533 	if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))  in tcp_repair_set_window()
3536 	if (after(opt.rcv_wup, tp->rcv_nxt))  in tcp_repair_set_window()
3539 	tp->snd_wl1	= opt.snd_wl1;  in tcp_repair_set_window()
3540 	tp->snd_wnd	= opt.snd_wnd;  in tcp_repair_set_window()
3541 	tp->max_window	= opt.max_window;  in tcp_repair_set_window()
3543 	tp->rcv_wnd	= opt.rcv_wnd;  in tcp_repair_set_window()
3544 	tp->rcv_wup	= opt.rcv_wup;  in tcp_repair_set_window()
3545 	tp->rcv_mwnd_seq = opt.rcv_wup + opt.rcv_wnd;  in tcp_repair_set_window()
3553 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_repair_options_est()  local
3566 			tp->rx_opt.mss_clamp = opt.opt_val;  in tcp_repair_options_est()
3577 				tp->rx_opt.snd_wscale = snd_wscale;  in tcp_repair_options_est()
3578 				tp->rx_opt.rcv_wscale = rcv_wscale;  in tcp_repair_options_est()
3579 				tp->rx_opt.wscale_ok = 1;  in tcp_repair_options_est()
3586 			tp->rx_opt.sack_ok |= TCP_SACK_SEEN;  in tcp_repair_options_est()
3592 			tp->rx_opt.tstamp_ok = 1;  in tcp_repair_options_est()
3604 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_enable_tx_delay()  local
3605 	s32 delta = (val - tp->tcp_tx_delay) << 3;  in tcp_enable_tx_delay()
3620 		s64 srtt = (s64)tp->srtt_us + delta;  in tcp_enable_tx_delay()
3622 		WRITE_ONCE(tp->srtt_us,  in tcp_enable_tx_delay()
3628 		minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);  in tcp_enable_tx_delay()
3645 	struct tcp_sock *tp = tcp_sk(sk);  in __tcp_sock_set_cork()  local
3648 		tp->nonagle |= TCP_NAGLE_CORK;  in __tcp_sock_set_cork()
3650 		tp->nonagle &= ~TCP_NAGLE_CORK;  in __tcp_sock_set_cork()
3651 		if (tp->nonagle & TCP_NAGLE_OFF)  in __tcp_sock_set_cork()
3652 			tp->nonagle |= TCP_NAGLE_PUSH;  in __tcp_sock_set_cork()
3739 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_sock_set_keepidle_locked()  local
3745 	WRITE_ONCE(tp->keepalive_time, val * HZ);  in tcp_sock_set_keepidle_locked()
3748 		u32 elapsed = keepalive_time_elapsed(tp);  in tcp_sock_set_keepidle_locked()
3750 		if (tp->keepalive_time > elapsed)  in tcp_sock_set_keepidle_locked()
3751 			elapsed = tp->keepalive_time - elapsed;  in tcp_sock_set_keepidle_locked()
3795 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_set_window_clamp()  local
3800 		WRITE_ONCE(tp->window_clamp, 0);  in tcp_set_window_clamp()
3804 	old_window_clamp = tp->window_clamp;  in tcp_set_window_clamp()
3810 	WRITE_ONCE(tp->window_clamp, new_window_clamp);  in tcp_set_window_clamp()
3818 		new_rcv_ssthresh = min(tp->rcv_wnd, new_window_clamp);  in tcp_set_window_clamp()
3819 		tp->rcv_ssthresh = max(new_rcv_ssthresh, tp->rcv_ssthresh);  in tcp_set_window_clamp()
3843 	struct tcp_sock *tp = tcp_sk(sk);  in do_tcp_setsockopt()  local
3930 			WRITE_ONCE(tp->linger2, -1);  in do_tcp_setsockopt()
3932 			WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX);  in do_tcp_setsockopt()
3934 			WRITE_ONCE(tp->linger2, val * HZ);  in do_tcp_setsockopt()
3978 			tp->thin_lto = val;  in do_tcp_setsockopt()
3990 			tp->repair = 1;  in do_tcp_setsockopt()
3992 			tp->repair_queue = TCP_NO_QUEUE;  in do_tcp_setsockopt()
3994 			tp->repair = 0;  in do_tcp_setsockopt()
3998 			tp->repair = 0;  in do_tcp_setsockopt()
4006 		if (!tp->repair)  in do_tcp_setsockopt()
4009 			tp->repair_queue = val;  in do_tcp_setsockopt()
4017 		} else if (tp->repair_queue == TCP_SEND_QUEUE) {  in do_tcp_setsockopt()
4021 				WRITE_ONCE(tp->write_seq, val);  in do_tcp_setsockopt()
4022 		} else if (tp->repair_queue == TCP_RECV_QUEUE) {  in do_tcp_setsockopt()
4023 			if (tp->rcv_nxt != tp->copied_seq) {  in do_tcp_setsockopt()
4026 				WRITE_ONCE(tp->rcv_nxt, val);  in do_tcp_setsockopt()
4027 				WRITE_ONCE(tp->copied_seq, val);  in do_tcp_setsockopt()
4035 		if (!tp->repair)  in do_tcp_setsockopt()
4037 		else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)  in do_tcp_setsockopt()
4055 			tp->save_syn = val;  in do_tcp_setsockopt()
4086 		if (tp->repair)  in do_tcp_setsockopt()
4091 		err = tp->af_specific->ao_parse(sk, optname, optval, optlen);  in do_tcp_setsockopt()
4098 		err = tp->af_specific->md5_parse(sk, optname, optval, optlen);  in do_tcp_setsockopt()
4117 				tp->fastopen_connect = val;  in do_tcp_setsockopt()
4130 			tp->fastopen_no_cookie = val;  in do_tcp_setsockopt()
4133 		if (!tp->repair) {  in do_tcp_setsockopt()
4141 		tp->tcp_usec_ts = val & 1;  in do_tcp_setsockopt()
4142 		WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts));  in do_tcp_setsockopt()
4145 		err = tcp_repair_set_window(tp, optval, optlen);  in do_tcp_setsockopt()
4148 		WRITE_ONCE(tp->notsent_lowat, val);  in do_tcp_setsockopt()
4155 			tp->recvmsg_inq = val;  in do_tcp_setsockopt()
4164 		WRITE_ONCE(tp->tcp_tx_delay, val);  in do_tcp_setsockopt()
4187 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,  in tcp_get_info_chrono_stats()  argument
4198 	cur = READ_ONCE(tp->chrono_type);  in tcp_get_info_chrono_stats()
4200 		stats[i] = READ_ONCE(tp->chrono_stat[i - 1]);  in tcp_get_info_chrono_stats()
4202 			stats[i] += tcp_jiffies32 - READ_ONCE(tp->chrono_start);  in tcp_get_info_chrono_stats()
4215 	const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */  in tcp_get_info()  local
4240 	info->tcpi_reordering = tp->reordering;  in tcp_get_info()
4241 	info->tcpi_snd_cwnd = tcp_snd_cwnd(tp);  in tcp_get_info()
4260 	if (tp->rx_opt.tstamp_ok)  in tcp_get_info()
4262 	if (tcp_is_sack(tp))  in tcp_get_info()
4264 	if (tp->rx_opt.wscale_ok) {  in tcp_get_info()
4266 		info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;  in tcp_get_info()
4267 		info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;  in tcp_get_info()
4270 	if (tcp_ecn_mode_any(tp))  in tcp_get_info()
4272 	if (tp->ecn_flags & TCP_ECN_SEEN)  in tcp_get_info()
4274 	if (tp->syn_data_acked)  in tcp_get_info()
4276 	if (tp->tcp_usec_ts)  in tcp_get_info()
4278 	if (tp->syn_fastopen_child)  in tcp_get_info()
4284 	info->tcpi_snd_mss = tp->mss_cache;  in tcp_get_info()
4287 	info->tcpi_unacked = tp->packets_out;  in tcp_get_info()
4288 	info->tcpi_sacked = tp->sacked_out;  in tcp_get_info()
4290 	info->tcpi_lost = tp->lost_out;  in tcp_get_info()
4291 	info->tcpi_retrans = tp->retrans_out;  in tcp_get_info()
4294 	info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);  in tcp_get_info()
4296 	info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);  in tcp_get_info()
4299 	info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;  in tcp_get_info()
4300 	info->tcpi_rtt = tp->srtt_us >> 3;  in tcp_get_info()
4301 	info->tcpi_rttvar = tp->mdev_us >> 2;  in tcp_get_info()
4302 	info->tcpi_snd_ssthresh = tp->snd_ssthresh;  in tcp_get_info()
4303 	info->tcpi_advmss = tp->advmss;  in tcp_get_info()
4305 	info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;  in tcp_get_info()
4306 	info->tcpi_rcv_space = tp->rcvq_space.space;  in tcp_get_info()
4308 	info->tcpi_total_retrans = tp->total_retrans;  in tcp_get_info()
4310 	info->tcpi_bytes_acked = tp->bytes_acked;  in tcp_get_info()
4311 	info->tcpi_bytes_received = tp->bytes_received;  in tcp_get_info()
4312 	info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);  in tcp_get_info()
4313 	tcp_get_info_chrono_stats(tp, info);  in tcp_get_info()
4315 	info->tcpi_segs_out = tp->segs_out;  in tcp_get_info()
4318 	info->tcpi_segs_in = READ_ONCE(tp->segs_in);  in tcp_get_info()
4319 	info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in);  in tcp_get_info()
4321 	info->tcpi_min_rtt = tcp_min_rtt(tp);  in tcp_get_info()
4322 	info->tcpi_data_segs_out = tp->data_segs_out;  in tcp_get_info()
4324 	info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;  in tcp_get_info()
4325 	rate64 = tcp_compute_delivery_rate(tp);  in tcp_get_info()
4328 	info->tcpi_delivered = tp->delivered;  in tcp_get_info()
4329 	info->tcpi_delivered_ce = tp->delivered_ce;  in tcp_get_info()
4330 	info->tcpi_bytes_sent = tp->bytes_sent;  in tcp_get_info()
4331 	info->tcpi_bytes_retrans = tp->bytes_retrans;  in tcp_get_info()
4332 	info->tcpi_dsack_dups = tp->dsack_dups;  in tcp_get_info()
4333 	info->tcpi_reord_seen = tp->reord_seen;  in tcp_get_info()
4334 	info->tcpi_rcv_ooopack = tp->rcv_ooopack;  in tcp_get_info()
4335 	info->tcpi_snd_wnd = tp->snd_wnd;  in tcp_get_info()
4336 	info->tcpi_rcv_wnd = tp->rcv_wnd;  in tcp_get_info()
4337 	info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash;  in tcp_get_info()
4338 	info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;  in tcp_get_info()
4340 	info->tcpi_total_rto = tp->total_rto;  in tcp_get_info()
4341 	info->tcpi_total_rto_recoveries = tp->total_rto_recoveries;  in tcp_get_info()
4342 	info->tcpi_total_rto_time = tp->total_rto_time;  in tcp_get_info()
4343 	if (tp->rto_stamp)  in tcp_get_info()
4344 		info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp;  in tcp_get_info()
4346 	if (tcp_ecn_disabled(tp))  in tcp_get_info()
4348 	else if (tcp_ecn_mode_rfc3168(tp))  in tcp_get_info()
4350 	else if (tcp_ecn_mode_accecn(tp))  in tcp_get_info()
4352 	else if (tcp_ecn_mode_pending(tp))  in tcp_get_info()
4354 	info->tcpi_accecn_fail_mode = tp->accecn_fail_mode;  in tcp_get_info()
4355 	info->tcpi_accecn_opt_seen = tp->saw_accecn_opt;  in tcp_get_info()
4356 	info->tcpi_received_ce = tp->received_ce;  in tcp_get_info()
4357 	info->tcpi_delivered_e1_bytes = tp->delivered_ecn_bytes[ect1_idx];  in tcp_get_info()
4358 	info->tcpi_delivered_e0_bytes = tp->delivered_ecn_bytes[ect0_idx];  in tcp_get_info()
4359 	info->tcpi_delivered_ce_bytes = tp->delivered_ecn_bytes[ce_idx];  in tcp_get_info()
4360 	info->tcpi_received_e1_bytes = tp->received_ecn_bytes[ect1_idx];  in tcp_get_info()
4361 	info->tcpi_received_e0_bytes = tp->received_ecn_bytes[ect0_idx];  in tcp_get_info()
4362 	info->tcpi_received_ce_bytes = tp->received_ecn_bytes[ce_idx];  in tcp_get_info()
4416 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_get_timestamping_opt_stats()  local
4426 	tcp_get_info_chrono_stats(tp, &info);  in tcp_get_timestamping_opt_stats()
4434 			  READ_ONCE(tp->data_segs_out), TCP_NLA_PAD);  in tcp_get_timestamping_opt_stats()
4436 			  READ_ONCE(tp->total_retrans), TCP_NLA_PAD);  in tcp_get_timestamping_opt_stats()
4442 	rate64 = tcp_compute_delivery_rate(tp);  in tcp_get_timestamping_opt_stats()
4445 	nla_put_u32(stats, TCP_NLA_SND_CWND, READ_ONCE(tp->snd_cwnd));  in tcp_get_timestamping_opt_stats()
4446 	nla_put_u32(stats, TCP_NLA_REORDERING, READ_ONCE(tp->reordering));  in tcp_get_timestamping_opt_stats()
4447 	nla_put_u32(stats, TCP_NLA_MIN_RTT, data_race(tcp_min_rtt(tp)));  in tcp_get_timestamping_opt_stats()
4451 	nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, data_race(!!tp->rate_app_limited));  in tcp_get_timestamping_opt_stats()
4452 	nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, READ_ONCE(tp->snd_ssthresh));  in tcp_get_timestamping_opt_stats()
4453 	nla_put_u32(stats, TCP_NLA_DELIVERED, READ_ONCE(tp->delivered));  in tcp_get_timestamping_opt_stats()
4454 	nla_put_u32(stats, TCP_NLA_DELIVERED_CE, READ_ONCE(tp->delivered_ce));  in tcp_get_timestamping_opt_stats()
4458 			  READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_una)));  in tcp_get_timestamping_opt_stats()
4461 	nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, READ_ONCE(tp->bytes_sent),  in tcp_get_timestamping_opt_stats()
4464 			  READ_ONCE(tp->bytes_retrans), TCP_NLA_PAD);  in tcp_get_timestamping_opt_stats()
4465 	nla_put_u32(stats, TCP_NLA_DSACK_DUPS, READ_ONCE(tp->dsack_dups));  in tcp_get_timestamping_opt_stats()
4466 	nla_put_u32(stats, TCP_NLA_REORD_SEEN, READ_ONCE(tp->reord_seen));  in tcp_get_timestamping_opt_stats()
4467 	nla_put_u32(stats, TCP_NLA_SRTT, READ_ONCE(tp->srtt_us) >> 3);  in tcp_get_timestamping_opt_stats()
4469 		    READ_ONCE(tp->timeout_rehash));  in tcp_get_timestamping_opt_stats()
4472 			  READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt)));  in tcp_get_timestamping_opt_stats()
4480 		    READ_ONCE(tp->plb_rehash) + READ_ONCE(tp->timeout_rehash));  in tcp_get_timestamping_opt_stats()
4488 	struct tcp_sock *tp = tcp_sk(sk);  in do_tcp_getsockopt()  local
4503 		val = tp->mss_cache;  in do_tcp_getsockopt()
4504 		user_mss = READ_ONCE(tp->rx_opt.user_mss);  in do_tcp_getsockopt()
4508 		if (tp->repair)  in do_tcp_getsockopt()
4509 			val = tp->rx_opt.mss_clamp;  in do_tcp_getsockopt()
4512 		val = !!(tp->nonagle&TCP_NAGLE_OFF);  in do_tcp_getsockopt()
4515 		val = !!(tp->nonagle&TCP_NAGLE_CORK);  in do_tcp_getsockopt()
4518 		val = keepalive_time_when(tp) / HZ;  in do_tcp_getsockopt()
4521 		val = keepalive_intvl_when(tp) / HZ;  in do_tcp_getsockopt()
4524 		val = keepalive_probes(tp);  in do_tcp_getsockopt()
4531 		val = READ_ONCE(tp->linger2);  in do_tcp_getsockopt()
4541 		val = READ_ONCE(tp->window_clamp);  in do_tcp_getsockopt()
4625 		val = tp->thin_lto;  in do_tcp_getsockopt()
4633 		val = tp->repair;  in do_tcp_getsockopt()
4637 		if (tp->repair)  in do_tcp_getsockopt()
4638 			val = tp->repair_queue;  in do_tcp_getsockopt()
4652 		if (!tp->repair)  in do_tcp_getsockopt()
4655 		opt.snd_wl1	= tp->snd_wl1;  in do_tcp_getsockopt()
4656 		opt.snd_wnd	= tp->snd_wnd;  in do_tcp_getsockopt()
4657 		opt.max_window	= tp->max_window;  in do_tcp_getsockopt()
4658 		opt.rcv_wnd	= tp->rcv_wnd;  in do_tcp_getsockopt()
4659 		opt.rcv_wup	= tp->rcv_wup;  in do_tcp_getsockopt()
4666 		if (tp->repair_queue == TCP_SEND_QUEUE)  in do_tcp_getsockopt()
4667 			val = tp->write_seq;  in do_tcp_getsockopt()
4668 		else if (tp->repair_queue == TCP_RECV_QUEUE)  in do_tcp_getsockopt()
4669 			val = tp->rcv_nxt;  in do_tcp_getsockopt()
4683 		val = tp->fastopen_connect;  in do_tcp_getsockopt()
4687 		val = tp->fastopen_no_cookie;  in do_tcp_getsockopt()
4691 		val = READ_ONCE(tp->tcp_tx_delay);  in do_tcp_getsockopt()
4695 		val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset);  in do_tcp_getsockopt()
4696 		if (tp->tcp_usec_ts)  in do_tcp_getsockopt()
4702 		val = READ_ONCE(tp->notsent_lowat);  in do_tcp_getsockopt()
4705 		val = tp->recvmsg_inq;  in do_tcp_getsockopt()
4708 		val = tp->save_syn;  in do_tcp_getsockopt()
4715 		if (tp->saved_syn) {  in do_tcp_getsockopt()
4716 			if (len < tcp_saved_syn_len(tp->saved_syn)) {  in do_tcp_getsockopt()
4717 				len = tcp_saved_syn_len(tp->saved_syn);  in do_tcp_getsockopt()
4725 			len = tcp_saved_syn_len(tp->saved_syn);  in do_tcp_getsockopt()
4730 			if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {  in do_tcp_getsockopt()
4734 			tcp_saved_syn_free(tp);  in do_tcp_getsockopt()
4925 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_inbound_md5_hash()  local
4943 		tp->af_specific->calc_md5_hash(newhash, key, NULL, skb);  in tcp_inbound_md5_hash()