1 // SPDX-License-Identifier: GPL-2.0-only
21  *    send_rate = #pkts_delivered/(last_snd_time - first_snd_time)
22  *    ack_rate  = #pkts_delivered/(last_ack_time - first_ack_time)
28  * deliberately avoids using the inter-packet spacing approach because that
31  * TCP flows can often be application-limited in request/response workloads.
32  * The estimator marks a bandwidth sample as application-limited if there
46 	  * the full time the network needs to deliver all in-flight  in tcp_rate_skb_sent()
53 	  * because the latter is a guess based on RTO and loss-marking  in tcp_rate_skb_sent()
58 	if (!tp->packets_out) {  in tcp_rate_skb_sent()
61 		tp->first_tx_mstamp  = tstamp_us;  in tcp_rate_skb_sent()
62 		tp->delivered_mstamp = tstamp_us;  in tcp_rate_skb_sent()
65 	TCP_SKB_CB(skb)->tx.first_tx_mstamp	= tp->first_tx_mstamp;  in tcp_rate_skb_sent()
66 	TCP_SKB_CB(skb)->tx.delivered_mstamp	= tp->delivered_mstamp;  in tcp_rate_skb_sent()
67 	TCP_SKB_CB(skb)->tx.delivered		= tp->delivered;  in tcp_rate_skb_sent()
68 	TCP_SKB_CB(skb)->tx.delivered_ce	= tp->delivered_ce;  in tcp_rate_skb_sent()
69 	TCP_SKB_CB(skb)->tx.is_app_limited	= tp->app_limited ? 1 : 0;  in tcp_rate_skb_sent()
75  * If an ACK (s)acks multiple skbs (e.g., stretched-acks), this function is
81 			    struct rate_sample *rs)  in tcp_rate_skb_delivered()  argument
87 	if (!scb->tx.delivered_mstamp)  in tcp_rate_skb_delivered()
91 	if (!rs->prior_delivered ||  in tcp_rate_skb_delivered()
92 	    tcp_skb_sent_after(tx_tstamp, tp->first_tx_mstamp,  in tcp_rate_skb_delivered()
93 			       scb->end_seq, rs->last_end_seq)) {  in tcp_rate_skb_delivered()
94 		rs->prior_delivered_ce  = scb->tx.delivered_ce;  in tcp_rate_skb_delivered()
95 		rs->prior_delivered  = scb->tx.delivered;  in tcp_rate_skb_delivered()
96 		rs->prior_mstamp     = scb->tx.delivered_mstamp;  in tcp_rate_skb_delivered()
97 		rs->is_app_limited   = scb->tx.is_app_limited;  in tcp_rate_skb_delivered()
98 		rs->is_retrans	     = scb->sacked & TCPCB_RETRANS;  in tcp_rate_skb_delivered()
99 		rs->last_end_seq     = scb->end_seq;  in tcp_rate_skb_delivered()
102 		tp->first_tx_mstamp  = tx_tstamp;  in tcp_rate_skb_delivered()
104 		rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,  in tcp_rate_skb_delivered()
105 						     scb->tx.first_tx_mstamp);  in tcp_rate_skb_delivered()
112 	if (scb->sacked & TCPCB_SACKED_ACKED)  in tcp_rate_skb_delivered()
113 		scb->tx.delivered_mstamp = 0;  in tcp_rate_skb_delivered()
118 		  bool is_sack_reneg, struct rate_sample *rs)  in tcp_rate_gen()  argument
124 	if (tp->app_limited && after(tp->delivered, tp->app_limited))  in tcp_rate_gen()
125 		tp->app_limited = 0;  in tcp_rate_gen()
132 		tp->delivered_mstamp = tp->tcp_mstamp;  in tcp_rate_gen()
134 	rs->acked_sacked = delivered;	/* freshly ACKed or SACKed */  in tcp_rate_gen()
135 	rs->losses = lost;		/* freshly marked lost */  in tcp_rate_gen()
141 	if (!rs->prior_mstamp || is_sack_reneg) {  in tcp_rate_gen()
142 		rs->delivered = -1;  in tcp_rate_gen()
143 		rs->interval_us = -1;  in tcp_rate_gen()
146 	rs->delivered   = tp->delivered - rs->prior_delivered;  in tcp_rate_gen()
148 	rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce;  in tcp_rate_gen()
150 	rs->delivered_ce &= TCPCB_DELIVERED_CE_MASK;  in tcp_rate_gen()
157 	snd_us = rs->interval_us;				/* send phase */  in tcp_rate_gen()
158 	ack_us = tcp_stamp_us_delta(tp->tcp_mstamp,  in tcp_rate_gen()
159 				    rs->prior_mstamp); /* ack phase */  in tcp_rate_gen()
160 	rs->interval_us = max(snd_us, ack_us);  in tcp_rate_gen()
163 	rs->snd_interval_us = snd_us;  in tcp_rate_gen()
164 	rs->rcv_interval_us = ack_us;  in tcp_rate_gen()
166 	/* Normally we expect interval_us >= min-rtt.  in tcp_rate_gen()
167 	 * Note that rate may still be over-estimated when a spuriously  in tcp_rate_gen()
169 	 * is under-estimated (up to an RTT). However continuously  in tcp_rate_gen()
173 	if (unlikely(rs->interval_us < tcp_min_rtt(tp))) {  in tcp_rate_gen()
174 		if (!rs->is_retrans)  in tcp_rate_gen()
176 				 rs->interval_us, rs->delivered,  in tcp_rate_gen()
177 				 inet_csk(sk)->icsk_ca_state,  in tcp_rate_gen()
178 				 tp->rx_opt.sack_ok, tcp_min_rtt(tp));  in tcp_rate_gen()
179 		rs->interval_us = -1;  in tcp_rate_gen()
183 	/* Record the last non-app-limited or the highest app-limited bw */  in tcp_rate_gen()
184 	if (!rs->is_app_limited ||  in tcp_rate_gen()
185 	    ((u64)rs->delivered * tp->rate_interval_us >=  in tcp_rate_gen()
186 	     (u64)tp->rate_delivered * rs->interval_us)) {  in tcp_rate_gen()
187 		tp->rate_delivered = rs->delivered;  in tcp_rate_gen()
188 		tp->rate_interval_us = rs->interval_us;  in tcp_rate_gen()
189 		tp->rate_app_limited = rs->is_app_limited;  in tcp_rate_gen()
193 /* If a gap is detected between sends, mark the socket application-limited. */
199 	    tp->write_seq - tp->snd_nxt < tp->mss_cache &&  in tcp_rate_check_app_limited()
205 	    tp->lost_out <= tp->retrans_out)  in tcp_rate_check_app_limited()
206 		tp->app_limited =  in tcp_rate_check_app_limited()
207 			(tp->delivered + tcp_packets_in_flight(tp)) ? : 1;  in tcp_rate_check_app_limited()