161 tcp_dsack_block_exists(struct tcpcb *tp)  in tcp_dsack_block_exists()  argument
164 	if (tp->rcv_numsacks == 0)  in tcp_dsack_block_exists()
166 	if (SEQ_LEQ(tp->sackblks[0].end, tp->rcv_nxt))  in tcp_dsack_block_exists()
176 tcp_update_dsack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)  in tcp_update_dsack_list()  argument
182 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_update_dsack_list()
186 	if (SEQ_LT(rcv_end, tp->rcv_nxt) ||  in tcp_update_dsack_list()
187 	    ((rcv_end == tp->rcv_nxt) &&  in tcp_update_dsack_list()
188 	     (tp->rcv_numsacks > 0 ) &&  in tcp_update_dsack_list()
189 	     (tp->sackblks[0].end == tp->rcv_nxt))) {  in tcp_update_dsack_list()
201 	for (i = 0; i < tp->rcv_numsacks; i++) {  in tcp_update_dsack_list()
202 		start = tp->sackblks[i].start;  in tcp_update_dsack_list()
203 		end = tp->sackblks[i].end;  in tcp_update_dsack_list()
212 		if (SEQ_GT(tp->rcv_nxt, end)) {  in tcp_update_dsack_list()
244 	if ((SEQ_LT(tp->rcv_nxt, mid_blk.end) &&  in tcp_update_dsack_list()
251 	for (j = 0; (j < tp->rcv_numsacks) && (n < MAX_SACK_BLKS); j++) {  in tcp_update_dsack_list()
252 		if (((SEQ_LT(tp->sackblks[j].end, mid_blk.start) ||  in tcp_update_dsack_list()
253 		      SEQ_GT(tp->sackblks[j].start, mid_blk.end)) &&  in tcp_update_dsack_list()
254 		    (SEQ_GT(tp->sackblks[j].start, tp->rcv_nxt))))  in tcp_update_dsack_list()
255 		saved_blks[n++] = tp->sackblks[j];  in tcp_update_dsack_list()
261 			tp->sackblks[j++] = saved_blks[i];  in tcp_update_dsack_list()
264 	tp->rcv_numsacks = j;  in tcp_update_dsack_list()
272 tcp_update_sack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)  in tcp_update_sack_list()  argument
283 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_update_sack_list()
289 	    (tp->rcv_numsacks >= 1) &&  in tcp_update_sack_list()
290 	    (rcv_end == tp->sackblks[0].end)) {  in tcp_update_sack_list()
292 		head_blk = tp->sackblks[0];  in tcp_update_sack_list()
305 	for (i = 0; i < tp->rcv_numsacks; i++) {  in tcp_update_sack_list()
306 		tcp_seq start = tp->sackblks[i].start;  in tcp_update_sack_list()
307 		tcp_seq end = tp->sackblks[i].end;  in tcp_update_sack_list()
308 		if (SEQ_GEQ(start, end) || SEQ_LEQ(start, tp->rcv_nxt)) {  in tcp_update_sack_list()
383 		tp->sackblks[0] = head_blk;  in tcp_update_sack_list()
393 	    (rcv_start == tp->sackblks[0].end)) {  in tcp_update_sack_list()
400 		bcopy(saved_blks, &tp->sackblks[num_head],  in tcp_update_sack_list()
405 	tp->rcv_numsacks = num_head + num_saved;  in tcp_update_sack_list()
409 tcp_clean_dsack_blocks(struct tcpcb *tp)  in tcp_clean_dsack_blocks()  argument
414 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_clean_dsack_blocks()
421 	for (i = 0; i < tp->rcv_numsacks; i++) {  in tcp_clean_dsack_blocks()
422 		tcp_seq start = tp->sackblks[i].start;  in tcp_clean_dsack_blocks()
423 		tcp_seq end = tp->sackblks[i].end;  in tcp_clean_dsack_blocks()
424 		if (SEQ_GEQ(start, end) || SEQ_LEQ(start, tp->rcv_nxt)) {  in tcp_clean_dsack_blocks()
441 		bcopy(saved_blks, &tp->sackblks[0],  in tcp_clean_dsack_blocks()
444 	tp->rcv_numsacks = num_saved;  in tcp_clean_dsack_blocks()
451 tcp_clean_sackreport(struct tcpcb *tp)  in tcp_clean_sackreport()  argument
455 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_clean_sackreport()
456 	tp->rcv_numsacks = 0;  in tcp_clean_sackreport()
458 		tp->sackblks[i].start = tp->sackblks[i].end=0;  in tcp_clean_sackreport()
465 tcp_sackhole_alloc(struct tcpcb *tp, tcp_seq start, tcp_seq end)  in tcp_sackhole_alloc()  argument
469 	if (tp->snd_numholes >= V_tcp_sack_maxholes ||  in tcp_sackhole_alloc()
483 	tp->snd_numholes++;  in tcp_sackhole_alloc()
493 tcp_sackhole_free(struct tcpcb *tp, struct sackhole *hole)  in tcp_sackhole_free()  argument
498 	tp->snd_numholes--;  in tcp_sackhole_free()
501 	KASSERT(tp->snd_numholes >= 0, ("tp->snd_numholes >= 0"));  in tcp_sackhole_free()
509 tcp_sackhole_insert(struct tcpcb *tp, tcp_seq start, tcp_seq end,  in tcp_sackhole_insert()  argument
515 	hole = tcp_sackhole_alloc(tp, start, end);  in tcp_sackhole_insert()
521 		TAILQ_INSERT_AFTER(&tp->snd_holes, after, hole, scblink);  in tcp_sackhole_insert()
523 		TAILQ_INSERT_TAIL(&tp->snd_holes, hole, scblink);  in tcp_sackhole_insert()
526 	if (tp->sackhint.nexthole == NULL)  in tcp_sackhole_insert()
527 		tp->sackhint.nexthole = hole;  in tcp_sackhole_insert()
536 tcp_sackhole_remove(struct tcpcb *tp, struct sackhole *hole)  in tcp_sackhole_remove()  argument
540 	if (tp->sackhint.nexthole == hole)  in tcp_sackhole_remove()
541 		tp->sackhint.nexthole = TAILQ_NEXT(hole, scblink);  in tcp_sackhole_remove()
544 	TAILQ_REMOVE(&tp->snd_holes, hole, scblink);  in tcp_sackhole_remove()
547 	tcp_sackhole_free(tp, hole);  in tcp_sackhole_remove()
559 tcp_sack_doack(struct tcpcb *tp, struct tcpopt *to, tcp_seq th_ack)  in tcp_sack_doack()  argument
566 	int maxseg = tp->t_maxseg - MAX_TCPOPTLEN;  in tcp_sack_doack()
573 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_doack()
584 	if (SEQ_LT(tp->snd_una, th_ack) && !TAILQ_EMPTY(&tp->snd_holes)) {  in tcp_sack_doack()
585 		left_edge_delta = th_ack - tp->snd_una;  in tcp_sack_doack()
586 		sack_blocks[num_sack_blks].start = tp->snd_una;  in tcp_sack_doack()
592 		if (SEQ_LT(tp->snd_fack, th_ack)) {  in tcp_sack_doack()
593 			delivered_data += th_ack - tp->snd_una;  in tcp_sack_doack()
594 			tp->snd_fack = th_ack;  in tcp_sack_doack()
609 			    SEQ_GT(sack.start, tp->snd_una) &&  in tcp_sack_doack()
611 			    SEQ_LT(sack.start, tp->snd_max) &&  in tcp_sack_doack()
612 			    SEQ_GT(sack.end, tp->snd_una) &&  in tcp_sack_doack()
613 			    SEQ_LEQ(sack.end, tp->snd_max) &&  in tcp_sack_doack()
615 			     SEQ_GEQ(sack.end, tp->snd_max))) {  in tcp_sack_doack()
622 				tcp_record_dsack(tp, sack.start, sack.end, 0);  in tcp_sack_doack()
647 	if (TAILQ_EMPTY(&tp->snd_holes)) {  in tcp_sack_doack()
655 		tp->snd_fack = SEQ_MAX(tp->snd_una, th_ack);  in tcp_sack_doack()
670 	tp->sackhint.last_sack_ack = sblkp->end;  in tcp_sack_doack()
671 	if (SEQ_LT(tp->snd_fack, sblkp->start)) {  in tcp_sack_doack()
678 		if (((temp = TAILQ_LAST(&tp->snd_holes, sackhole_head)) != NULL) &&  in tcp_sack_doack()
679 		    SEQ_LEQ(tp->snd_fack,temp->end)) {  in tcp_sack_doack()
680 			tp->sackhint.hole_bytes -= temp->end - temp->start;  in tcp_sack_doack()
681 			temp->start = SEQ_MAX(tp->snd_fack, SEQ_MAX(tp->snd_una, th_ack));  in tcp_sack_doack()
685 			tp->sackhint.hole_bytes += temp->end - temp->start;  in tcp_sack_doack()
686 			KASSERT(tp->sackhint.hole_bytes >= 0,  in tcp_sack_doack()
688 			tp->snd_fack = sblkp->end;  in tcp_sack_doack()
698 			temp = tcp_sackhole_insert(tp, tp->snd_fack,sblkp->start,NULL);  in tcp_sack_doack()
701 				tp->sackhint.hole_bytes += temp->end - temp->start;  in tcp_sack_doack()
702 				tp->snd_fack = sblkp->end;  in tcp_sack_doack()
716 				       SEQ_LT(tp->snd_fack, sblkp->start))  in tcp_sack_doack()
719 				    SEQ_LT(tp->snd_fack, sblkp->end)) {  in tcp_sack_doack()
720 					delivered_data += sblkp->end - tp->snd_fack;  in tcp_sack_doack()
721 					tp->snd_fack = sblkp->end;  in tcp_sack_doack()
731 	} else if (SEQ_LT(tp->snd_fack, sblkp->end)) {  in tcp_sack_doack()
733 		delivered_data += sblkp->end - tp->snd_fack;  in tcp_sack_doack()
734 		tp->snd_fack = sblkp->end;  in tcp_sack_doack()
737 	cur = TAILQ_LAST(&tp->snd_holes, sackhole_head); /* Last SACK hole. */  in tcp_sack_doack()
738 	loss_hiack = tp->snd_fack;  in tcp_sack_doack()
747 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))   in tcp_sack_doack()
753 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg))) {  in tcp_sack_doack()
778 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))  in tcp_sack_doack()
783 		tp->sackhint.sack_bytes_rexmit -=  in tcp_sack_doack()
785 		KASSERT(tp->sackhint.sack_bytes_rexmit >= 0,  in tcp_sack_doack()
795 				tp->sackhint.hole_bytes -= temp->end - temp->start;  in tcp_sack_doack()
796 				tcp_sackhole_remove(tp, temp);  in tcp_sack_doack()
806 				tp->sackhint.hole_bytes -= sblkp->end - cur->start;  in tcp_sack_doack()
815 				tp->sackhint.hole_bytes -= cur->end - sblkp->start;  in tcp_sack_doack()
818 				if ((tp->t_flags & TF_LRD) && SEQ_GEQ(cur->rxmit, cur->end))  in tcp_sack_doack()
819 					cur->rxmit = tp->snd_recover;  in tcp_sack_doack()
825 				temp = tcp_sackhole_insert(tp, sblkp->end,  in tcp_sack_doack()
831 						tp->sackhint.sack_bytes_rexmit +=  in tcp_sack_doack()
835 					tp->sackhint.hole_bytes -= sblkp->end - sblkp->start;  in tcp_sack_doack()
840 					    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))  in tcp_sack_doack()
845 					if ((tp->t_flags & TF_LRD) && SEQ_GEQ(cur->rxmit, cur->end))  in tcp_sack_doack()
846 						cur->rxmit = tp->snd_recover;  in tcp_sack_doack()
851 		tp->sackhint.sack_bytes_rexmit +=  in tcp_sack_doack()
863 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))  in tcp_sack_doack()
872 	KASSERT(notlost_bytes <= tp->sackhint.hole_bytes,  in tcp_sack_doack()
875 	if (TAILQ_EMPTY(&tp->snd_holes)) {  in tcp_sack_doack()
876 		KASSERT(tp->sackhint.hole_bytes == 0,  in tcp_sack_doack()
878 		tp->sackhint.sack_bytes_rexmit = 0;  in tcp_sack_doack()
879 		tp->sackhint.sacked_bytes = 0;  in tcp_sack_doack()
880 		tp->sackhint.lost_bytes = 0;  in tcp_sack_doack()
882 		KASSERT(tp->sackhint.hole_bytes > 0,  in tcp_sack_doack()
884 		tp->sackhint.delivered_data = delivered_data;  in tcp_sack_doack()
885 		tp->sackhint.sacked_bytes += delivered_data - left_edge_delta;  in tcp_sack_doack()
886 		KASSERT((tp->sackhint.sacked_bytes >= 0), ("sacked_bytes < 0"));  in tcp_sack_doack()
887 		tp->sackhint.lost_bytes = tp->sackhint.hole_bytes -  in tcp_sack_doack()
908 tcp_free_sackholes(struct tcpcb *tp)  in tcp_free_sackholes()  argument
912 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_free_sackholes()
913 	while ((q = TAILQ_FIRST(&tp->snd_holes)) != NULL)  in tcp_free_sackholes()
914 		tcp_sackhole_remove(tp, q);  in tcp_free_sackholes()
915 	tp->sackhint.sack_bytes_rexmit = 0;  in tcp_free_sackholes()
916 	tp->sackhint.delivered_data = 0;  in tcp_free_sackholes()
917 	tp->sackhint.sacked_bytes = 0;  in tcp_free_sackholes()
918 	tp->sackhint.hole_bytes = 0;  in tcp_free_sackholes()
919 	tp->sackhint.lost_bytes = 0;  in tcp_free_sackholes()
921 	KASSERT(tp->snd_numholes == 0, ("tp->snd_numholes == 0"));  in tcp_free_sackholes()
922 	KASSERT(tp->sackhint.nexthole == NULL,  in tcp_free_sackholes()
933 tcp_resend_sackholes(struct tcpcb *tp)  in tcp_resend_sackholes()  argument
937 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_resend_sackholes()
938 	TAILQ_FOREACH(p, &tp->snd_holes, scblink) {  in tcp_resend_sackholes()
941 	tp->sackhint.nexthole = TAILQ_FIRST(&tp->snd_holes);  in tcp_resend_sackholes()
942 	tp->sackhint.sack_bytes_rexmit = 0;  in tcp_resend_sackholes()
954 tcp_sack_partialack(struct tcpcb *tp, struct tcphdr *th, u_int *maxsegp)  in tcp_sack_partialack()  argument
960 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_partialack()
963 		*maxsegp = tcp_maxseg(tp);  in tcp_sack_partialack()
966 	tcp_timer_activate(tp, TT_REXMT, 0);  in tcp_sack_partialack()
967 	tp->t_rtttime = 0;  in tcp_sack_partialack()
969 	if ((BYTES_THIS_ACK(tp, th) / maxseg) >= 2)  in tcp_sack_partialack()
971 	if (tp->snd_nxt == tp->snd_max) {  in tcp_sack_partialack()
972 		tp->snd_cwnd = (tp->sackhint.sack_bytes_rexmit +  in tcp_sack_partialack()
973 		    (tp->snd_nxt - tp->snd_recover) + num_segs * maxseg);  in tcp_sack_partialack()
981 	if (tp->snd_cwnd > tp->snd_ssthresh)  in tcp_sack_partialack()
982 		tp->snd_cwnd = tp->snd_ssthresh;  in tcp_sack_partialack()
983 	tp->t_flags |= TF_ACKNOW;  in tcp_sack_partialack()
1001 	    SEQ_LT(th->th_ack, tp->snd_recover) &&  in tcp_sack_partialack()
1002 	    TAILQ_EMPTY(&tp->snd_holes) &&  in tcp_sack_partialack()
1003 	    (tp->sackhint.delivered_data > 0)) {  in tcp_sack_partialack()
1010 		tcp_seq highdata = tp->snd_max;  in tcp_sack_partialack()
1011 		if (tp->t_flags & TF_SENTFIN)  in tcp_sack_partialack()
1013 		highdata = SEQ_MIN(highdata, tp->snd_recover);  in tcp_sack_partialack()
1015 			tp->snd_fack = SEQ_MAX(th->th_ack, tp->snd_fack);  in tcp_sack_partialack()
1016 			if ((temp = tcp_sackhole_insert(tp, SEQ_MAX(th->th_ack,  in tcp_sack_partialack()
1018 				tp->sackhint.hole_bytes +=  in tcp_sack_partialack()
1023 	(void) tcp_output(tp);  in tcp_sack_partialack()
1044 tcp_sack_output(struct tcpcb *tp, int *sack_bytes_rexmt)  in tcp_sack_output()  argument
1048 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_output()
1049 	*sack_bytes_rexmt = tp->sackhint.sack_bytes_rexmit;  in tcp_sack_output()
1050 	hole = tp->sackhint.nexthole;  in tcp_sack_output()
1059 				tp->sackhint.nexthole = hole;  in tcp_sack_output()
1066 		KASSERT(SEQ_LT(hole->start, tp->snd_fack), ("%s: hole.start >= snd.fack", __func__));  in tcp_sack_output()
1067 		KASSERT(SEQ_LT(hole->end, tp->snd_fack), ("%s: hole.end >= snd.fack", __func__));  in tcp_sack_output()
1068 		KASSERT(SEQ_LT(hole->rxmit, tp->snd_fack), ("%s: hole.rxmit >= snd.fack", __func__));  in tcp_sack_output()
1070 		    SEQ_GEQ(hole->start, tp->snd_fack) ||  in tcp_sack_output()
1071 		    SEQ_GEQ(hole->end, tp->snd_fack) ||  in tcp_sack_output()
1072 		    SEQ_GEQ(hole->rxmit, tp->snd_fack)) {  in tcp_sack_output()
1074 					hole->start, hole->end, hole->rxmit, tp->snd_fack);  in tcp_sack_output()
1090 tcp_sack_adjust(struct tcpcb *tp)  in tcp_sack_adjust()  argument
1093 	struct sackhole *p, *cur = TAILQ_FIRST(&tp->snd_holes);  in tcp_sack_adjust()
1095 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_adjust()
1100 	if (SEQ_GEQ(tp->snd_nxt, tp->snd_fack)) {  in tcp_sack_adjust()
1102 		return (tp->sackhint.sacked_bytes);  in tcp_sack_adjust()
1110 		if (SEQ_LT(tp->snd_nxt, cur->end)) {  in tcp_sack_adjust()
1114 		if (SEQ_GEQ(tp->snd_nxt, p->start)) {  in tcp_sack_adjust()
1117 			tp->snd_nxt = p->start;  in tcp_sack_adjust()
1121 	if (SEQ_LT(tp->snd_nxt, cur->end)) {  in tcp_sack_adjust()
1124 	tp->snd_nxt = tp->snd_fack;  in tcp_sack_adjust()
1125 	return (tp->sackhint.sacked_bytes);  in tcp_sack_adjust()
1135 tcp_sack_lost_retransmission(struct tcpcb *tp, struct tcphdr *th)  in tcp_sack_lost_retransmission()  argument
1139 	if (IN_RECOVERY(tp->t_flags) &&  in tcp_sack_lost_retransmission()
1140 	    SEQ_GT(tp->snd_fack, tp->snd_recover) &&  in tcp_sack_lost_retransmission()
1141 	    ((temp = TAILQ_FIRST(&tp->snd_holes)) != NULL) &&  in tcp_sack_lost_retransmission()
1143 	    SEQ_GEQ(tp->snd_fack, temp->rxmit)) {  in tcp_sack_lost_retransmission()
1150 		tp->sackhint.nexthole = temp;  in tcp_sack_lost_retransmission()
1151 		TAILQ_FOREACH(temp, &tp->snd_holes, scblink) {  in tcp_sack_lost_retransmission()
1152 			if (SEQ_GEQ(tp->snd_fack, temp->rxmit) &&  in tcp_sack_lost_retransmission()
1162 		tp->snd_cwnd = tp->snd_ssthresh;  in tcp_sack_lost_retransmission()
1167 		EXIT_RECOVERY(tp->t_flags);  in tcp_sack_lost_retransmission()
1168 		cc_cong_signal(tp, th, CC_NDUPACK);  in tcp_sack_lost_retransmission()
1175 		tp->snd_cwnd = tcp_maxseg(tp);  in tcp_sack_lost_retransmission()
1176 		tp->sackhint.recover_fs = (tp->snd_max - tp->snd_una) -  in tcp_sack_lost_retransmission()
1177 					    tp->sackhint.recover_fs;  in tcp_sack_lost_retransmission()