1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995
5 * The Regents of the University of California.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 /*-
34 *
35 * NRL grants permission for redistribution and use in source and binary
36 * forms, with or without modification, of the software and documentation
37 * created at NRL provided that the following conditions are met:
38 *
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 * must display the following acknowledgements:
46 * This product includes software developed by the University of
47 * California, Berkeley and its contributors.
48 * This product includes software developed at the Information
49 * Technology Division, US Naval Research Laboratory.
50 * 4. Neither the name of the NRL nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THE SOFTWARE PROVIDED BY NRL IS PROVIDED BY NRL AND CONTRIBUTORS ``AS
55 * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
57 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NRL OR
58 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
59 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
60 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
61 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
62 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
63 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
64 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
65 *
66 * The views and conclusions contained in the software and documentation
67 * are those of the authors and should not be interpreted as representing
68 * official policies, either expressed or implied, of the US Naval
69 * Research Laboratory (NRL).
70 */
71
72 #include "opt_inet.h"
73 #include "opt_inet6.h"
74
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/kernel.h>
78 #include <sys/sysctl.h>
79 #include <sys/malloc.h>
80 #include <sys/mbuf.h>
81 #include <sys/proc.h> /* for proc0 declaration */
82 #include <sys/protosw.h>
83 #include <sys/socket.h>
84 #include <sys/socketvar.h>
85 #include <sys/syslog.h>
86 #include <sys/systm.h>
87
88 #include <machine/cpu.h> /* before tcp_seq.h, for tcp_random18() */
89
90 #include <vm/uma.h>
91
92 #include <net/if.h>
93 #include <net/if_var.h>
94 #include <net/route.h>
95 #include <net/vnet.h>
96
97 #include <netinet/in.h>
98 #include <netinet/in_systm.h>
99 #include <netinet/ip.h>
100 #include <netinet/in_var.h>
101 #include <netinet/in_pcb.h>
102 #include <netinet/ip_var.h>
103 #include <netinet/ip6.h>
104 #include <netinet/icmp6.h>
105 #include <netinet6/nd6.h>
106 #include <netinet6/ip6_var.h>
107 #include <netinet6/in6_pcb.h>
108 #include <netinet/tcp.h>
109 #include <netinet/tcp_fsm.h>
110 #include <netinet/tcp_seq.h>
111 #include <netinet/tcp_timer.h>
112 #include <netinet/tcp_var.h>
113 #include <netinet/tcpip.h>
114 #include <netinet/cc/cc.h>
115
116 #include <machine/in_cksum.h>
117
118 VNET_DECLARE(struct uma_zone *, sack_hole_zone);
119 #define V_sack_hole_zone VNET(sack_hole_zone)
120
121 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
122 "TCP SACK");
123
124 VNET_DEFINE(int, tcp_do_sack) = 1;
125 SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW,
126 &VNET_NAME(tcp_do_sack), 0,
127 "Enable/Disable TCP SACK support");
128
129 VNET_DEFINE(int, tcp_do_lrd) = 1;
130 SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, lrd, CTLFLAG_VNET | CTLFLAG_RW,
131 &VNET_NAME(tcp_do_lrd), 1,
132 "Perform Lost Retransmission Detection");
133
134 VNET_DEFINE(int, tcp_sack_tso) = 0;
135 SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, tso, CTLFLAG_VNET | CTLFLAG_RW,
136 &VNET_NAME(tcp_sack_tso), 0,
137 "Allow TSO during SACK loss recovery");
138
139 VNET_DEFINE(int, tcp_sack_maxholes) = 128;
140 SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, maxholes, CTLFLAG_VNET | CTLFLAG_RW,
141 &VNET_NAME(tcp_sack_maxholes), 0,
142 "Maximum number of TCP SACK holes allowed per connection");
143
144 VNET_DEFINE(int, tcp_sack_globalmaxholes) = 65536;
145 SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, globalmaxholes, CTLFLAG_VNET | CTLFLAG_RW,
146 &VNET_NAME(tcp_sack_globalmaxholes), 0,
147 "Global maximum number of TCP SACK holes");
148
149 VNET_DEFINE(int, tcp_sack_globalholes) = 0;
150 SYSCTL_INT(_net_inet_tcp_sack, OID_AUTO, globalholes, CTLFLAG_VNET | CTLFLAG_RD,
151 &VNET_NAME(tcp_sack_globalholes), 0,
152 "Global number of TCP SACK holes currently allocated");
153
154 int
tcp_dsack_block_exists(struct tcpcb * tp)155 tcp_dsack_block_exists(struct tcpcb *tp)
156 {
157 /* Return true if a DSACK block exists */
158 if (tp->rcv_numsacks == 0)
159 return (0);
160 if (SEQ_LEQ(tp->sackblks[0].end, tp->rcv_nxt))
161 return(1);
162 return (0);
163 }
164
165 /*
166 * This function will find overlaps with the currently stored sackblocks
167 * and add any overlap as a dsack block upfront
168 */
169 void
tcp_update_dsack_list(struct tcpcb * tp,tcp_seq rcv_start,tcp_seq rcv_end)170 tcp_update_dsack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)
171 {
172 struct sackblk head_blk,mid_blk,saved_blks[MAX_SACK_BLKS];
173 int i, j, n, identical;
174 tcp_seq start, end;
175
176 INP_WLOCK_ASSERT(tptoinpcb(tp));
177
178 KASSERT(SEQ_LT(rcv_start, rcv_end), ("rcv_start < rcv_end"));
179
180 if (SEQ_LT(rcv_end, tp->rcv_nxt) ||
181 ((rcv_end == tp->rcv_nxt) &&
182 (tp->rcv_numsacks > 0 ) &&
183 (tp->sackblks[0].end == tp->rcv_nxt))) {
184 saved_blks[0].start = rcv_start;
185 saved_blks[0].end = rcv_end;
186 } else {
187 saved_blks[0].start = saved_blks[0].end = 0;
188 }
189
190 head_blk.start = head_blk.end = 0;
191 mid_blk.start = rcv_start;
192 mid_blk.end = rcv_end;
193 identical = 0;
194
195 for (i = 0; i < tp->rcv_numsacks; i++) {
196 start = tp->sackblks[i].start;
197 end = tp->sackblks[i].end;
198 if (SEQ_LT(rcv_end, start)) {
199 /* pkt left to sack blk */
200 continue;
201 }
202 if (SEQ_GT(rcv_start, end)) {
203 /* pkt right to sack blk */
204 continue;
205 }
206 if (SEQ_GT(tp->rcv_nxt, end)) {
207 if ((SEQ_MAX(rcv_start, start) != SEQ_MIN(rcv_end, end)) &&
208 (SEQ_GT(head_blk.start, SEQ_MAX(rcv_start, start)) ||
209 (head_blk.start == head_blk.end))) {
210 head_blk.start = SEQ_MAX(rcv_start, start);
211 head_blk.end = SEQ_MIN(rcv_end, end);
212 }
213 continue;
214 }
215 if (((head_blk.start == head_blk.end) ||
216 SEQ_LT(start, head_blk.start)) &&
217 (SEQ_GT(end, rcv_start) &&
218 SEQ_LEQ(start, rcv_end))) {
219 head_blk.start = start;
220 head_blk.end = end;
221 }
222 mid_blk.start = SEQ_MIN(mid_blk.start, start);
223 mid_blk.end = SEQ_MAX(mid_blk.end, end);
224 if ((mid_blk.start == start) &&
225 (mid_blk.end == end))
226 identical = 1;
227 }
228 if (SEQ_LT(head_blk.start, head_blk.end)) {
229 /* store overlapping range */
230 saved_blks[0].start = SEQ_MAX(rcv_start, head_blk.start);
231 saved_blks[0].end = SEQ_MIN(rcv_end, head_blk.end);
232 }
233 n = 1;
234 /*
235 * Second, if not ACKed, store the SACK block that
236 * overlaps with the DSACK block unless it is identical
237 */
238 if ((SEQ_LT(tp->rcv_nxt, mid_blk.end) &&
239 !((mid_blk.start == saved_blks[0].start) &&
240 (mid_blk.end == saved_blks[0].end))) ||
241 identical == 1) {
242 saved_blks[n].start = mid_blk.start;
243 saved_blks[n++].end = mid_blk.end;
244 }
245 for (j = 0; (j < tp->rcv_numsacks) && (n < MAX_SACK_BLKS); j++) {
246 if (((SEQ_LT(tp->sackblks[j].end, mid_blk.start) ||
247 SEQ_GT(tp->sackblks[j].start, mid_blk.end)) &&
248 (SEQ_GT(tp->sackblks[j].start, tp->rcv_nxt))))
249 saved_blks[n++] = tp->sackblks[j];
250 }
251 j = 0;
252 for (i = 0; i < n; i++) {
253 /* we can end up with a stale initial entry */
254 if (SEQ_LT(saved_blks[i].start, saved_blks[i].end)) {
255 tp->sackblks[j++] = saved_blks[i];
256 }
257 }
258 tp->rcv_numsacks = j;
259 }
260
261 /*
262 * This function is called upon receipt of new valid data (while not in
263 * header prediction mode), and it updates the ordered list of sacks.
264 */
265 void
tcp_update_sack_list(struct tcpcb * tp,tcp_seq rcv_start,tcp_seq rcv_end)266 tcp_update_sack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)
267 {
268 /*
269 * First reported block MUST be the most recent one. Subsequent
270 * blocks SHOULD be in the order in which they arrived at the
271 * receiver. These two conditions make the implementation fully
272 * compliant with RFC 2018.
273 */
274 struct sackblk head_blk, saved_blks[MAX_SACK_BLKS];
275 int num_head, num_saved, i;
276
277 INP_WLOCK_ASSERT(tptoinpcb(tp));
278
279 /* Check arguments. */
280 KASSERT(SEQ_LEQ(rcv_start, rcv_end), ("SEG_GT(rcv_start, rcv_end)"));
281
282 if ((rcv_start == rcv_end) &&
283 (tp->rcv_numsacks >= 1) &&
284 (rcv_end == tp->sackblks[0].end)) {
285 /* retaining DSACK block below rcv_nxt (todrop) */
286 head_blk = tp->sackblks[0];
287 } else {
288 /* SACK block for the received segment. */
289 head_blk.start = rcv_start;
290 head_blk.end = rcv_end;
291 }
292
293 /*
294 * Merge updated SACK blocks into head_blk, and save unchanged SACK
295 * blocks into saved_blks[]. num_saved will have the number of the
296 * saved SACK blocks.
297 */
298 num_saved = 0;
299 for (i = 0; i < tp->rcv_numsacks; i++) {
300 tcp_seq start = tp->sackblks[i].start;
301 tcp_seq end = tp->sackblks[i].end;
302 if (SEQ_GEQ(start, end) || SEQ_LEQ(start, tp->rcv_nxt)) {
303 /*
304 * Discard this SACK block.
305 */
306 } else if (SEQ_LEQ(head_blk.start, end) &&
307 SEQ_GEQ(head_blk.end, start)) {
308 /*
309 * Merge this SACK block into head_blk. This SACK
310 * block itself will be discarded.
311 */
312 /*
313 * |-|
314 * |---| merge
315 *
316 * |-|
317 * |---| merge
318 *
319 * |-----|
320 * |-| DSACK smaller
321 *
322 * |-|
323 * |-----| DSACK smaller
324 */
325 if (head_blk.start == end)
326 head_blk.start = start;
327 else if (head_blk.end == start)
328 head_blk.end = end;
329 else {
330 if (SEQ_LT(head_blk.start, start)) {
331 tcp_seq temp = start;
332 start = head_blk.start;
333 head_blk.start = temp;
334 }
335 if (SEQ_GT(head_blk.end, end)) {
336 tcp_seq temp = end;
337 end = head_blk.end;
338 head_blk.end = temp;
339 }
340 if ((head_blk.start != start) ||
341 (head_blk.end != end)) {
342 if ((num_saved >= 1) &&
343 SEQ_GEQ(saved_blks[num_saved-1].start, start) &&
344 SEQ_LEQ(saved_blks[num_saved-1].end, end))
345 num_saved--;
346 saved_blks[num_saved].start = start;
347 saved_blks[num_saved].end = end;
348 num_saved++;
349 }
350 }
351 } else {
352 /*
353 * This block supercedes the prior block
354 */
355 if ((num_saved >= 1) &&
356 SEQ_GEQ(saved_blks[num_saved-1].start, start) &&
357 SEQ_LEQ(saved_blks[num_saved-1].end, end))
358 num_saved--;
359 /*
360 * Save this SACK block.
361 */
362 saved_blks[num_saved].start = start;
363 saved_blks[num_saved].end = end;
364 num_saved++;
365 }
366 }
367
368 /*
369 * Update SACK list in tp->sackblks[].
370 */
371 num_head = 0;
372 if (SEQ_LT(rcv_start, rcv_end)) {
373 /*
374 * The received data segment is an out-of-order segment. Put
375 * head_blk at the top of SACK list.
376 */
377 tp->sackblks[0] = head_blk;
378 num_head = 1;
379 /*
380 * If the number of saved SACK blocks exceeds its limit,
381 * discard the last SACK block.
382 */
383 if (num_saved >= MAX_SACK_BLKS)
384 num_saved--;
385 }
386 if ((rcv_start == rcv_end) &&
387 (rcv_start == tp->sackblks[0].end)) {
388 num_head = 1;
389 }
390 if (num_saved > 0) {
391 /*
392 * Copy the saved SACK blocks back.
393 */
394 bcopy(saved_blks, &tp->sackblks[num_head],
395 sizeof(struct sackblk) * num_saved);
396 }
397
398 /* Save the number of SACK blocks. */
399 tp->rcv_numsacks = num_head + num_saved;
400 }
401
402 void
tcp_clean_dsack_blocks(struct tcpcb * tp)403 tcp_clean_dsack_blocks(struct tcpcb *tp)
404 {
405 struct sackblk saved_blks[MAX_SACK_BLKS];
406 int num_saved, i;
407
408 INP_WLOCK_ASSERT(tptoinpcb(tp));
409 /*
410 * Clean up any DSACK blocks that
411 * are in our queue of sack blocks.
412 *
413 */
414 num_saved = 0;
415 for (i = 0; i < tp->rcv_numsacks; i++) {
416 tcp_seq start = tp->sackblks[i].start;
417 tcp_seq end = tp->sackblks[i].end;
418 if (SEQ_GEQ(start, end) || SEQ_LEQ(start, tp->rcv_nxt)) {
419 /*
420 * Discard this D-SACK block.
421 */
422 continue;
423 }
424 /*
425 * Save this SACK block.
426 */
427 saved_blks[num_saved].start = start;
428 saved_blks[num_saved].end = end;
429 num_saved++;
430 }
431 if (num_saved > 0) {
432 /*
433 * Copy the saved SACK blocks back.
434 */
435 bcopy(saved_blks, &tp->sackblks[0],
436 sizeof(struct sackblk) * num_saved);
437 }
438 tp->rcv_numsacks = num_saved;
439 }
440
441 /*
442 * Delete all receiver-side SACK information.
443 */
444 void
tcp_clean_sackreport(struct tcpcb * tp)445 tcp_clean_sackreport(struct tcpcb *tp)
446 {
447 int i;
448
449 INP_WLOCK_ASSERT(tptoinpcb(tp));
450 tp->rcv_numsacks = 0;
451 for (i = 0; i < MAX_SACK_BLKS; i++)
452 tp->sackblks[i].start = tp->sackblks[i].end=0;
453 }
454
455 /*
456 * Allocate struct sackhole.
457 */
458 static struct sackhole *
tcp_sackhole_alloc(struct tcpcb * tp,tcp_seq start,tcp_seq end)459 tcp_sackhole_alloc(struct tcpcb *tp, tcp_seq start, tcp_seq end)
460 {
461 struct sackhole *hole;
462
463 if (tp->snd_numholes >= V_tcp_sack_maxholes ||
464 V_tcp_sack_globalholes >= V_tcp_sack_globalmaxholes) {
465 TCPSTAT_INC(tcps_sack_sboverflow);
466 return NULL;
467 }
468
469 hole = (struct sackhole *)uma_zalloc(V_sack_hole_zone, M_NOWAIT);
470 if (hole == NULL)
471 return NULL;
472
473 hole->start = start;
474 hole->end = end;
475 hole->rxmit = start;
476
477 tp->snd_numholes++;
478 atomic_add_int(&V_tcp_sack_globalholes, 1);
479
480 return hole;
481 }
482
483 /*
484 * Free struct sackhole.
485 */
486 static void
tcp_sackhole_free(struct tcpcb * tp,struct sackhole * hole)487 tcp_sackhole_free(struct tcpcb *tp, struct sackhole *hole)
488 {
489
490 uma_zfree(V_sack_hole_zone, hole);
491
492 tp->snd_numholes--;
493 atomic_subtract_int(&V_tcp_sack_globalholes, 1);
494
495 KASSERT(tp->snd_numholes >= 0, ("tp->snd_numholes < 0"));
496 KASSERT(V_tcp_sack_globalholes >= 0, ("tcp_sack_globalholes < 0"));
497 }
498
499 /*
500 * Insert new SACK hole into scoreboard.
501 */
502 static struct sackhole *
tcp_sackhole_insert(struct tcpcb * tp,tcp_seq start,tcp_seq end,struct sackhole * after)503 tcp_sackhole_insert(struct tcpcb *tp, tcp_seq start, tcp_seq end,
504 struct sackhole *after)
505 {
506 struct sackhole *hole;
507
508 /* Allocate a new SACK hole. */
509 hole = tcp_sackhole_alloc(tp, start, end);
510 if (hole == NULL)
511 return NULL;
512
513 /* Insert the new SACK hole into scoreboard. */
514 if (after != NULL)
515 TAILQ_INSERT_AFTER(&tp->snd_holes, after, hole, scblink);
516 else
517 TAILQ_INSERT_TAIL(&tp->snd_holes, hole, scblink);
518
519 /* Update SACK hint. */
520 if (tp->sackhint.nexthole == NULL)
521 tp->sackhint.nexthole = hole;
522
523 return hole;
524 }
525
526 /*
527 * Remove SACK hole from scoreboard.
528 */
529 static void
tcp_sackhole_remove(struct tcpcb * tp,struct sackhole * hole)530 tcp_sackhole_remove(struct tcpcb *tp, struct sackhole *hole)
531 {
532
533 /* Update SACK hint. */
534 if (tp->sackhint.nexthole == hole)
535 tp->sackhint.nexthole = TAILQ_NEXT(hole, scblink);
536
537 /* Remove this SACK hole. */
538 TAILQ_REMOVE(&tp->snd_holes, hole, scblink);
539
540 /* Free this SACK hole. */
541 tcp_sackhole_free(tp, hole);
542 }
543
544 /*
545 * Process cumulative ACK and the TCP SACK option to update the scoreboard.
546 * tp->snd_holes is an ordered list of holes (oldest to newest, in terms of
547 * the sequence space).
548 * Returns SACK_NEWLOSS if incoming ACK indicates ongoing loss (hole split, new hole),
549 * SACK_CHANGE if incoming ACK has previously unknown SACK information,
550 * SACK_NOCHANGE otherwise.
551 */
552 sackstatus_t
tcp_sack_doack(struct tcpcb * tp,struct tcpopt * to,tcp_seq th_ack)553 tcp_sack_doack(struct tcpcb *tp, struct tcpopt *to, tcp_seq th_ack)
554 {
555 struct sackhole *cur, *temp;
556 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1], *sblkp;
557 int i, j, num_sack_blks;
558 sackstatus_t sack_changed;
559 int delivered_data, left_edge_delta;
560 int maxseg = tp->t_maxseg - MAX_TCPOPTLEN;
561
562 tcp_seq loss_hiack = 0;
563 int loss_thresh = 0;
564 int loss_sblks = 0;
565 int notlost_bytes = 0;
566
567 INP_WLOCK_ASSERT(tptoinpcb(tp));
568
569 num_sack_blks = 0;
570 sack_changed = SACK_NOCHANGE;
571 delivered_data = 0;
572 left_edge_delta = 0;
573 /*
574 * If SND.UNA will be advanced by SEG.ACK, and if SACK holes exist,
575 * treat [SND.UNA, SEG.ACK) as if it is a SACK block.
576 * Account changes to SND.UNA always in delivered data.
577 */
578 if (SEQ_LT(tp->snd_una, th_ack) && !TAILQ_EMPTY(&tp->snd_holes)) {
579 left_edge_delta = th_ack - tp->snd_una;
580 delivered_data += left_edge_delta;
581 sack_blocks[num_sack_blks].start = tp->snd_una;
582 sack_blocks[num_sack_blks++].end = th_ack;
583 /*
584 * Pulling snd_fack forward if we got here
585 * due to DSACK blocks
586 */
587 if (SEQ_LT(tp->snd_fack, th_ack)) {
588 tp->snd_fack = th_ack;
589 sack_changed = SACK_CHANGE;
590 }
591 }
592 /*
593 * Append received valid SACK blocks to sack_blocks[], but only if we
594 * received new blocks from the other side.
595 */
596 if (to->to_flags & TOF_SACK) {
597 for (i = 0; i < to->to_nsacks; i++) {
598 bcopy((to->to_sacks + i * TCPOLEN_SACK),
599 &sack, sizeof(sack));
600 sack.start = ntohl(sack.start);
601 sack.end = ntohl(sack.end);
602 if (SEQ_GT(sack.end, sack.start) &&
603 SEQ_GT(sack.start, tp->snd_una) &&
604 SEQ_GT(sack.start, th_ack) &&
605 SEQ_LT(sack.start, tp->snd_max) &&
606 SEQ_GT(sack.end, tp->snd_una) &&
607 SEQ_LEQ(sack.end, tp->snd_max) &&
608 ((sack.end - sack.start) >= maxseg ||
609 SEQ_GEQ(sack.end, tp->snd_max))) {
610 sack_blocks[num_sack_blks++] = sack;
611 } else if (SEQ_LEQ(sack.start, th_ack) &&
612 SEQ_LEQ(sack.end, th_ack)) {
613 /*
614 * Its a D-SACK block.
615 */
616 tcp_record_dsack(tp, sack.start, sack.end, 0);
617 }
618 }
619 }
620 /*
621 * Return if SND.UNA is not advanced and no valid SACK block is
622 * received.
623 */
624 if (num_sack_blks == 0)
625 return (sack_changed);
626
627 /*
628 * Sort the SACK blocks so we can update the scoreboard with just one
629 * pass. The overhead of sorting up to 4+1 elements is less than
630 * making up to 4+1 passes over the scoreboard.
631 */
632 for (i = 0; i < num_sack_blks; i++) {
633 for (j = i + 1; j < num_sack_blks; j++) {
634 if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) {
635 sack = sack_blocks[i];
636 sack_blocks[i] = sack_blocks[j];
637 sack_blocks[j] = sack;
638 }
639 }
640 }
641 if (TAILQ_EMPTY(&tp->snd_holes)) {
642 /*
643 * Empty scoreboard. Need to initialize snd_fack (it may be
644 * uninitialized or have a bogus value). Scoreboard holes
645 * (from the sack blocks received) are created later below
646 * (in the logic that adds holes to the tail of the
647 * scoreboard).
648 */
649 tp->snd_fack = SEQ_MAX(tp->snd_una, th_ack);
650 }
651 /*
652 * In the while-loop below, incoming SACK blocks (sack_blocks[]) and
653 * SACK holes (snd_holes) are traversed from their tails with just
654 * one pass in order to reduce the number of compares especially when
655 * the bandwidth-delay product is large.
656 *
657 * Note: Typically, in the first RTT of SACK recovery, the highest
658 * three or four SACK blocks with the same ack number are received.
659 * In the second RTT, if retransmitted data segments are not lost,
660 * the highest three or four SACK blocks with ack number advancing
661 * are received.
662 */
663 sblkp = &sack_blocks[num_sack_blks - 1]; /* Last SACK block */
664 tp->sackhint.last_sack_ack = sblkp->end;
665 if (SEQ_LT(tp->snd_fack, sblkp->start)) {
666 /*
667 * The highest SACK block is beyond fack. First,
668 * check if there was a successful Rescue Retransmission,
669 * and move this hole left. With normal holes, snd_fack
670 * is always to the right of the end.
671 */
672 if (((temp = TAILQ_LAST(&tp->snd_holes, sackhole_head)) != NULL) &&
673 SEQ_LEQ(tp->snd_fack,temp->end)) {
674 tp->sackhint.hole_bytes -= temp->end - temp->start;
675 temp->start = SEQ_MAX(tp->snd_fack, SEQ_MAX(tp->snd_una, th_ack));
676 temp->end = sblkp->start;
677 temp->rxmit = temp->start;
678 delivered_data += sblkp->end - sblkp->start;
679 tp->sackhint.hole_bytes += temp->end - temp->start;
680 KASSERT(tp->sackhint.hole_bytes >= 0,
681 ("sackhint hole bytes < 0"));
682 tp->snd_fack = sblkp->end;
683 sblkp--;
684 sack_changed = SACK_NEWLOSS;
685 } else {
686 /*
687 * Append a new SACK hole at the tail. If the
688 * second or later highest SACK blocks are also
689 * beyond the current fack, they will be inserted
690 * by way of hole splitting in the while-loop below.
691 */
692 temp = tcp_sackhole_insert(tp, tp->snd_fack,sblkp->start,NULL);
693 if (temp != NULL) {
694 delivered_data += sblkp->end - sblkp->start;
695 tp->sackhint.hole_bytes += temp->end - temp->start;
696 tp->snd_fack = sblkp->end;
697 /* Go to the previous sack block. */
698 sblkp--;
699 sack_changed = SACK_CHANGE;
700 } else {
701 /*
702 * We failed to add a new hole based on the current
703 * sack block. Skip over all the sack blocks that
704 * fall completely to the right of snd_fack and
705 * proceed to trim the scoreboard based on the
706 * remaining sack blocks. This also trims the
707 * scoreboard for th_ack (which is sack_blocks[0]).
708 */
709 while (sblkp >= sack_blocks &&
710 SEQ_LT(tp->snd_fack, sblkp->start))
711 sblkp--;
712 if (sblkp >= sack_blocks &&
713 SEQ_LT(tp->snd_fack, sblkp->end)) {
714 delivered_data += sblkp->end - tp->snd_fack;
715 tp->snd_fack = sblkp->end;
716 /*
717 * While the Scoreboard didn't change in
718 * size, we only ended up here because
719 * some SACK data had to be dismissed.
720 */
721 sack_changed = SACK_NEWLOSS;
722 }
723 }
724 }
725 } else if (SEQ_LT(tp->snd_fack, sblkp->end)) {
726 /* fack is advanced. */
727 delivered_data += sblkp->end - tp->snd_fack;
728 tp->snd_fack = sblkp->end;
729 sack_changed = SACK_CHANGE;
730 }
731 cur = TAILQ_LAST(&tp->snd_holes, sackhole_head); /* Last SACK hole. */
732 loss_hiack = tp->snd_fack;
733
734 /*
735 * Since the incoming sack blocks are sorted, we can process them
736 * making one sweep of the scoreboard.
737 */
738 while (cur != NULL) {
739 if (!(sblkp >= sack_blocks)) {
740 if (((loss_sblks >= tcprexmtthresh) ||
741 (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))
742 break;
743 loss_thresh += loss_hiack - cur->end;
744 loss_hiack = cur->start;
745 loss_sblks++;
746 if (!((loss_sblks >= tcprexmtthresh) ||
747 (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg))) {
748 notlost_bytes += cur->end - cur->start;
749 } else {
750 break;
751 }
752 cur = TAILQ_PREV(cur, sackhole_head, scblink);
753 continue;
754 }
755 if (SEQ_GEQ(sblkp->start, cur->end)) {
756 /*
757 * SACKs data beyond the current hole. Go to the
758 * previous sack block.
759 */
760 sblkp--;
761 continue;
762 }
763 if (SEQ_LEQ(sblkp->end, cur->start)) {
764 /*
765 * SACKs data before the current hole. Go to the
766 * previous hole.
767 */
768 loss_thresh += loss_hiack - cur->end;
769 loss_hiack = cur->start;
770 loss_sblks++;
771 if (!((loss_sblks >= tcprexmtthresh) ||
772 (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))
773 notlost_bytes += cur->end - cur->start;
774 cur = TAILQ_PREV(cur, sackhole_head, scblink);
775 continue;
776 }
777 tp->sackhint.sack_bytes_rexmit -=
778 (SEQ_MIN(cur->rxmit, cur->end) - cur->start);
779 KASSERT(tp->sackhint.sack_bytes_rexmit >= 0,
780 ("sackhint bytes rtx < 0"));
781 sack_changed = SACK_CHANGE;
782 if (SEQ_LEQ(sblkp->start, cur->start)) {
783 /* Data acks at least the beginning of hole. */
784 if (SEQ_GEQ(sblkp->end, cur->end)) {
785 /* Acks entire hole, so delete hole. */
786 delivered_data += (cur->end - cur->start);
787 temp = cur;
788 cur = TAILQ_PREV(cur, sackhole_head, scblink);
789 tp->sackhint.hole_bytes -= temp->end - temp->start;
790 tcp_sackhole_remove(tp, temp);
791 /*
792 * The sack block may ack all or part of the
793 * next hole too, so continue onto the next
794 * hole.
795 */
796 continue;
797 } else {
798 /* Move start of hole forward. */
799 delivered_data += (sblkp->end - cur->start);
800 tp->sackhint.hole_bytes -= sblkp->end - cur->start;
801 cur->start = sblkp->end;
802 cur->rxmit = SEQ_MAX(cur->rxmit, cur->start);
803 }
804 } else {
805 /* Data acks at least the end of hole. */
806 if (SEQ_GEQ(sblkp->end, cur->end)) {
807 /* Move end of hole backward. */
808 delivered_data += (cur->end - sblkp->start);
809 tp->sackhint.hole_bytes -= cur->end - sblkp->start;
810 cur->end = sblkp->start;
811 cur->rxmit = SEQ_MIN(cur->rxmit, cur->end);
812 if ((tp->t_flags & TF_LRD) && SEQ_GEQ(cur->rxmit, cur->end))
813 cur->rxmit = SEQ_MAX(cur->rxmit, tp->snd_recover);
814 } else {
815 /*
816 * ACKs some data in middle of a hole; need
817 * to split current hole
818 */
819 temp = tcp_sackhole_insert(tp, sblkp->end,
820 cur->end, cur);
821 sack_changed = SACK_NEWLOSS;
822 if (temp != NULL) {
823 if (SEQ_GT(cur->rxmit, temp->rxmit)) {
824 temp->rxmit = cur->rxmit;
825 tp->sackhint.sack_bytes_rexmit +=
826 (SEQ_MIN(temp->rxmit,
827 temp->end) - temp->start);
828 }
829 tp->sackhint.hole_bytes -= sblkp->end - sblkp->start;
830 loss_thresh += loss_hiack - temp->end;
831 loss_hiack = temp->start;
832 loss_sblks++;
833 if (!((loss_sblks >= tcprexmtthresh) ||
834 (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))
835 notlost_bytes += temp->end - temp->start;
836 cur->end = sblkp->start;
837 cur->rxmit = SEQ_MIN(cur->rxmit,
838 cur->end);
839 if ((tp->t_flags & TF_LRD) && SEQ_GEQ(cur->rxmit, cur->end))
840 cur->rxmit = SEQ_MAX(cur->rxmit, tp->snd_recover);
841 delivered_data += (sblkp->end - sblkp->start);
842 }
843 }
844 }
845 tp->sackhint.sack_bytes_rexmit +=
846 (SEQ_MIN(cur->rxmit, cur->end) - cur->start);
847 /*
848 * Testing sblkp->start against cur->start tells us whether
849 * we're done with the sack block or the sack hole.
850 * Accordingly, we advance one or the other.
851 */
852 if (SEQ_LEQ(sblkp->start, cur->start)) {
853 loss_thresh += loss_hiack - cur->end;
854 loss_hiack = cur->start;
855 loss_sblks++;
856 if (!((loss_sblks >= tcprexmtthresh) ||
857 (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))
858 notlost_bytes += cur->end - cur->start;
859 cur = TAILQ_PREV(cur, sackhole_head, scblink);
860 } else {
861 sblkp--;
862 }
863 }
864
865 KASSERT(delivered_data >= 0, ("delivered_data < 0"));
866 KASSERT(notlost_bytes <= tp->sackhint.hole_bytes,
867 ("SACK: more bytes marked notlost than in scoreboard holes"));
868
869 if (TAILQ_EMPTY(&tp->snd_holes)) {
870 KASSERT(tp->sackhint.hole_bytes == 0,
871 ("SACK scoreboard empty, but sackhint hole bytes != 0"));
872 tp->sackhint.sack_bytes_rexmit = 0;
873 tp->sackhint.sacked_bytes = 0;
874 tp->sackhint.lost_bytes = 0;
875 } else {
876 KASSERT(tp->sackhint.hole_bytes > 0,
877 ("SACK scoreboard not empty, but sackhint hole bytes <= 0"));
878 tp->sackhint.delivered_data = delivered_data;
879 tp->sackhint.sacked_bytes += delivered_data - left_edge_delta;
880 KASSERT((tp->sackhint.sacked_bytes >= 0), ("sacked_bytes < 0"));
881 tp->sackhint.lost_bytes = tp->sackhint.hole_bytes -
882 notlost_bytes;
883 }
884
885 if (!(to->to_flags & TOF_SACK))
886 /*
887 * If this ACK did not contain any
888 * SACK blocks, any only moved the
889 * left edge right, it is a pure
890 * cumulative ACK. Do not count
891 * DupAck for this. Also required
892 * for RFC6675 rescue retransmission.
893 */
894 sack_changed = SACK_NOCHANGE;
895 return (sack_changed);
896 }
897
898 /*
899 * Free all SACK holes to clear the scoreboard.
900 */
901 void
tcp_free_sackholes(struct tcpcb * tp)902 tcp_free_sackholes(struct tcpcb *tp)
903 {
904 struct sackhole *q;
905
906 INP_WLOCK_ASSERT(tptoinpcb(tp));
907 while ((q = TAILQ_FIRST(&tp->snd_holes)) != NULL)
908 tcp_sackhole_remove(tp, q);
909 tp->sackhint.sack_bytes_rexmit = 0;
910 tp->sackhint.delivered_data = 0;
911 tp->sackhint.sacked_bytes = 0;
912 tp->sackhint.hole_bytes = 0;
913 tp->sackhint.lost_bytes = 0;
914
915 KASSERT(tp->snd_numholes == 0, ("tp->snd_numholes != 0"));
916 KASSERT(tp->sackhint.nexthole == NULL,
917 ("tp->sackhint.nexthole != NULL"));
918 }
919
920 /*
921 * Resend all the currently existing SACK holes of
922 * the scoreboard. This is in line with the Errata to
923 * RFC 2018, which allows the use of SACK data past
924 * an RTO to good effect typically.
925 */
926 void
tcp_resend_sackholes(struct tcpcb * tp)927 tcp_resend_sackholes(struct tcpcb *tp)
928 {
929 struct sackhole *p;
930
931 INP_WLOCK_ASSERT(tptoinpcb(tp));
932 TAILQ_FOREACH(p, &tp->snd_holes, scblink) {
933 p->rxmit = p->start;
934 }
935 tp->sackhint.nexthole = TAILQ_FIRST(&tp->snd_holes);
936 tp->sackhint.sack_bytes_rexmit = 0;
937 }
938
939 /*
940 * Partial ack handling within a sack recovery episode. Keeping this very
941 * simple for now. When a partial ack is received, force snd_cwnd to a value
942 * that will allow the sender to transmit no more than 2 segments. If
943 * necessary, a better scheme can be adopted at a later point, but for now,
944 * the goal is to prevent the sender from bursting a large amount of data in
945 * the midst of sack recovery.
946 */
947 void
tcp_sack_partialack(struct tcpcb * tp,struct tcphdr * th,u_int * maxsegp)948 tcp_sack_partialack(struct tcpcb *tp, struct tcphdr *th, u_int *maxsegp)
949 {
950 struct sackhole *temp;
951 int num_segs = 1;
952 u_int maxseg;
953
954 INP_WLOCK_ASSERT(tptoinpcb(tp));
955
956 if (*maxsegp == 0) {
957 *maxsegp = tcp_maxseg(tp);
958 }
959 maxseg = *maxsegp;
960 tcp_timer_activate(tp, TT_REXMT, 0);
961 tp->t_rtttime = 0;
962 /* Send one or 2 segments based on how much new data was acked. */
963 if ((BYTES_THIS_ACK(tp, th) / maxseg) >= 2)
964 num_segs = 2;
965 if (tp->snd_nxt == tp->snd_max) {
966 tp->snd_cwnd = (tp->sackhint.sack_bytes_rexmit +
967 (tp->snd_nxt - tp->snd_recover) + num_segs * maxseg);
968 } else {
969 /*
970 * Since cwnd is not the expected flightsize during
971 * SACK LR, not deflating cwnd allows the partial
972 * ACKed amount to be sent.
973 */
974 }
975 if (tp->snd_cwnd > tp->snd_ssthresh)
976 tp->snd_cwnd = tp->snd_ssthresh;
977 tp->t_flags |= TF_ACKNOW;
978 /*
979 * RFC6675 rescue retransmission
980 * Add a hole between th_ack (snd_una is not yet set) and snd_max,
981 * if this was a pure cumulative ACK and no data was send beyond
982 * recovery point. Since the data in the socket has not been freed
983 * at this point, we check if the scoreboard is empty, and the ACK
984 * delivered some new data, indicating a full ACK. Also, if the
985 * recovery point is still at snd_max, we are probably application
986 * limited. However, this inference might not always be true. The
987 * rescue retransmission may rarely be slightly premature
988 * compared to RFC6675.
989 * The corresponding ACK+SACK will cause any further outstanding
990 * segments to be retransmitted. This addresses a corner case, when
991 * the trailing packets of a window are lost and no further data
992 * is available for sending.
993 */
994 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
995 TAILQ_EMPTY(&tp->snd_holes) &&
996 (tp->sackhint.delivered_data > 0)) {
997 /*
998 * Exclude FIN sequence space in
999 * the hole for the rescue retransmission,
1000 * and also don't create a hole, if only
1001 * the ACK for a FIN is outstanding.
1002 */
1003 tcp_seq highdata = tp->snd_max;
1004 if (tp->t_flags & TF_SENTFIN)
1005 highdata--;
1006 highdata = SEQ_MIN(highdata, tp->snd_recover);
1007 if (SEQ_LT(th->th_ack, highdata)) {
1008 tp->snd_fack = SEQ_MAX(th->th_ack, tp->snd_fack);
1009 if ((temp = tcp_sackhole_insert(tp, SEQ_MAX(th->th_ack,
1010 highdata - maxseg), highdata, NULL)) != NULL) {
1011 tp->sackhint.hole_bytes +=
1012 temp->end - temp->start;
1013 }
1014 }
1015 }
1016 (void) tcp_output(tp);
1017 }
1018
1019 /*
1020 * Returns the next hole to retransmit and the number of retransmitted bytes
1021 * from the scoreboard. We store both the next hole and the number of
1022 * retransmitted bytes as hints (and recompute these on the fly upon SACK/ACK
1023 * reception). This avoids scoreboard traversals completely.
1024 *
1025 * The loop here will traverse *at most* one link. Here's the argument. For
1026 * the loop to traverse more than 1 link before finding the next hole to
1027 * retransmit, we would need to have at least 1 node following the current
1028 * hint with (rxmit == end). But, for all holes following the current hint,
1029 * (start == rxmit), since we have not yet retransmitted from them.
1030 * Therefore, in order to traverse more 1 link in the loop below, we need to
1031 * have at least one node following the current hint with (start == rxmit ==
1032 * end). But that can't happen, (start == end) means that all the data in
1033 * that hole has been sacked, in which case, the hole would have been removed
1034 * from the scoreboard.
1035 */
1036 struct sackhole *
tcp_sack_output(struct tcpcb * tp,int * sack_bytes_rexmt)1037 tcp_sack_output(struct tcpcb *tp, int *sack_bytes_rexmt)
1038 {
1039 struct sackhole *hole = NULL;
1040
1041 INP_WLOCK_ASSERT(tptoinpcb(tp));
1042 *sack_bytes_rexmt = tp->sackhint.sack_bytes_rexmit;
1043 hole = tp->sackhint.nexthole;
1044 if (hole == NULL)
1045 return (hole);
1046 if (SEQ_GEQ(hole->rxmit, hole->end)) {
1047 for (;;) {
1048 hole = TAILQ_NEXT(hole, scblink);
1049 if (hole == NULL)
1050 return (hole);
1051 if (SEQ_LT(hole->rxmit, hole->end)) {
1052 tp->sackhint.nexthole = hole;
1053 break;
1054 }
1055 }
1056 }
1057 KASSERT(SEQ_LT(hole->start, hole->end),
1058 ("%s: SEQ_GEQ(hole.start, hole.end)", __func__));
1059 return (hole);
1060 }
1061
1062 /*
1063 * After a timeout, the SACK list may be rebuilt. This SACK information
1064 * should be used to avoid retransmitting SACKed data. This function
1065 * traverses the SACK list to see if snd_nxt should be moved forward.
1066 * In addition, cwnd will be inflated by the sacked bytes traversed when
1067 * moving snd_nxt forward. This prevents a traffic burst after the final
1068 * full ACK, and also keeps ACKs coming back.
1069 */
1070 int
tcp_sack_adjust(struct tcpcb * tp)1071 tcp_sack_adjust(struct tcpcb *tp)
1072 {
1073 int sacked = 0;
1074 struct sackhole *p, *cur = TAILQ_FIRST(&tp->snd_holes);
1075
1076 INP_WLOCK_ASSERT(tptoinpcb(tp));
1077 if (cur == NULL) {
1078 /* No holes */
1079 return (0);
1080 }
1081 if (SEQ_GEQ(tp->snd_nxt, tp->snd_fack)) {
1082 /* We're already beyond any SACKed blocks */
1083 return (tp->sackhint.sacked_bytes);
1084 }
1085 /*
1086 * Two cases for which we want to advance snd_nxt:
1087 * i) snd_nxt lies between end of one hole and beginning of another
1088 * ii) snd_nxt lies between end of last hole and snd_fack
1089 */
1090 while ((p = TAILQ_NEXT(cur, scblink)) != NULL) {
1091 if (SEQ_LT(tp->snd_nxt, cur->end)) {
1092 return (sacked);
1093 }
1094 sacked += p->start - cur->end;
1095 if (SEQ_GEQ(tp->snd_nxt, p->start)) {
1096 cur = p;
1097 } else {
1098 tp->snd_nxt = p->start;
1099 return (sacked);
1100 }
1101 }
1102 if (SEQ_LT(tp->snd_nxt, cur->end)) {
1103 return (sacked);
1104 }
1105 tp->snd_nxt = tp->snd_fack;
1106 return (tp->sackhint.sacked_bytes);
1107 }
1108
1109 /*
1110 * Lost Retransmission Detection
1111 * Check is FACK is beyond the rexmit of the leftmost hole.
1112 * If yes, we restart sending from still existing holes,
1113 * and adjust cwnd via the congestion control module.
1114 */
1115 void
tcp_sack_lost_retransmission(struct tcpcb * tp,struct tcphdr * th)1116 tcp_sack_lost_retransmission(struct tcpcb *tp, struct tcphdr *th)
1117 {
1118 struct sackhole *temp;
1119
1120 if (IN_RECOVERY(tp->t_flags) &&
1121 SEQ_GT(tp->snd_fack, tp->snd_recover) &&
1122 ((temp = TAILQ_FIRST(&tp->snd_holes)) != NULL) &&
1123 SEQ_GEQ(temp->rxmit, temp->end) &&
1124 SEQ_GEQ(tp->snd_fack, temp->rxmit)) {
1125 TCPSTAT_INC(tcps_sack_lostrexmt);
1126 /*
1127 * Start retransmissions from the first hole, and
1128 * subsequently all other remaining holes, including
1129 * those, which had been sent completely before.
1130 */
1131 tp->sackhint.nexthole = temp;
1132 TAILQ_FOREACH(temp, &tp->snd_holes, scblink) {
1133 if (SEQ_GEQ(tp->snd_fack, temp->rxmit) &&
1134 SEQ_GEQ(temp->rxmit, temp->end))
1135 temp->rxmit = temp->start;
1136 }
1137 /*
1138 * Remember the old ssthresh, to deduct the beta factor used
1139 * by the CC module. Finally, set cwnd to ssthresh just
1140 * prior to invoking another cwnd reduction by the CC
1141 * module, to not shrink it excessively.
1142 */
1143 tp->snd_cwnd = tp->snd_ssthresh;
1144 /*
1145 * Formally exit recovery, and let the CC module adjust
1146 * ssthresh as intended.
1147 */
1148 EXIT_RECOVERY(tp->t_flags);
1149 cc_cong_signal(tp, th, CC_NDUPACK);
1150 /*
1151 * For PRR, adjust recover_fs as if this new reduction
1152 * initialized this variable.
1153 * cwnd will be adjusted by SACK or PRR processing
1154 * subsequently, only set it to a safe value here.
1155 */
1156 tp->snd_cwnd = tcp_maxseg(tp);
1157 tp->sackhint.recover_fs = (tp->snd_max - tp->snd_una) -
1158 tp->sackhint.recover_fs;
1159 }
1160 }
1161