86 static void	tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le);
92 int	(*tcp_lro_flush_tcphpts)(struct lro_ctrl *lc, struct lro_entry *le);
152     struct lro_entry *le)  in tcp_lro_active_insert()  argument
155 	LIST_INSERT_HEAD(&lc->lro_active, le, next);  in tcp_lro_active_insert()
156 	LIST_INSERT_HEAD(bucket, le, hash_next);  in tcp_lro_active_insert()
160 tcp_lro_active_remove(struct lro_entry *le)  in tcp_lro_active_remove()  argument
163 	LIST_REMOVE(le, next);		/* active list */  in tcp_lro_active_remove()
164 	LIST_REMOVE(le, hash_next);	/* hash bucket */  in tcp_lro_active_remove()
177 	struct lro_entry *le;  in tcp_lro_init_args()  local
203 	    (lro_entries * sizeof(*le));  in tcp_lro_init_args()
214 	le = (struct lro_entry *)  in tcp_lro_init_args()
219 		LIST_INSERT_HEAD(&lc->lro_free, le + i, next);  in tcp_lro_init_args()
494 	struct lro_entry *le;  in tcp_lro_free()  local
501 	while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {  in tcp_lro_free()
502 		tcp_lro_active_remove(le);  in tcp_lro_free()
503 		lro_free_mbuf_chain(le->m_head);  in tcp_lro_free()
588 	struct lro_entry *le;  in tcp_lro_rx_done()  local
590 	while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {  in tcp_lro_rx_done()
591 		tcp_lro_active_remove(le);  in tcp_lro_rx_done()
592 		tcp_lro_flush(lc, le);  in tcp_lro_rx_done()
599 	struct lro_entry *le, *le_tmp;  in tcp_lro_flush_active()  local
602 	 * Walk through the list of le entries, and  in tcp_lro_flush_active()
611 	LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {  in tcp_lro_flush_active()
612 		if (le->m_head != NULL) {  in tcp_lro_flush_active()
613 			tcp_lro_active_remove(le);  in tcp_lro_flush_active()
614 			tcp_lro_flush(lc, le);  in tcp_lro_flush_active()
622 	struct lro_entry *le, *le_tmp;  in tcp_lro_flush_inactive()  local
634 	LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {  in tcp_lro_flush_inactive()
635 		if (now >= (bintime2ns(&le->alloc_time) + tov)) {  in tcp_lro_flush_inactive()
636 			tcp_lro_active_remove(le);  in tcp_lro_flush_inactive()
637 			tcp_lro_flush(lc, le);  in tcp_lro_flush_inactive()
678 tcp_lro_update_checksum(const struct lro_parser *pa, const struct lro_entry *le,  in tcp_lro_update_checksum()  argument
731 	pa->tcp->th_ack = le->ack_seq;  in tcp_lro_update_checksum()
732 	pa->tcp->th_win = le->window;  in tcp_lro_update_checksum()
735 	if (le->timestamp != 0) {  in tcp_lro_update_checksum()
739 		ts_ptr[1] = htonl(le->tsval);  in tcp_lro_update_checksum()
740 		ts_ptr[2] = le->tsecr;  in tcp_lro_update_checksum()
790 tcp_flush_out_entry(struct lro_ctrl *lc, struct lro_entry *le)  in tcp_flush_out_entry()  argument
793 	if (le->needs_merge) {  in tcp_flush_out_entry()
796 		switch (le->inner.data.lro_type) {  in tcp_flush_out_entry()
798 			csum = tcp_lro_update_checksum(&le->inner, le,  in tcp_flush_out_entry()
799 			    le->m_head->m_pkthdr.lro_tcp_d_len,  in tcp_flush_out_entry()
800 			    le->m_head->m_pkthdr.lro_tcp_d_csum);  in tcp_flush_out_entry()
801 			csum = tcp_lro_update_checksum(&le->outer, NULL,  in tcp_flush_out_entry()
802 			    le->m_head->m_pkthdr.lro_tcp_d_len +  in tcp_flush_out_entry()
803 			    le->inner.total_hdr_len, csum);  in tcp_flush_out_entry()
804 			le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |  in tcp_flush_out_entry()
806 			le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
807 			if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED))  in tcp_flush_out_entry()
808 				le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;  in tcp_flush_out_entry()
811 			csum = tcp_lro_update_checksum(&le->inner, le,  in tcp_flush_out_entry()
812 			    le->m_head->m_pkthdr.lro_tcp_d_len,  in tcp_flush_out_entry()
813 			    le->m_head->m_pkthdr.lro_tcp_d_csum);  in tcp_flush_out_entry()
814 			csum = tcp_lro_update_checksum(&le->outer, NULL,  in tcp_flush_out_entry()
815 			    le->m_head->m_pkthdr.lro_tcp_d_len +  in tcp_flush_out_entry()
816 			    le->inner.total_hdr_len, csum);  in tcp_flush_out_entry()
817 			le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |  in tcp_flush_out_entry()
819 			le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
820 			if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED))  in tcp_flush_out_entry()
821 				le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;  in tcp_flush_out_entry()
824 			switch (le->outer.data.lro_type) {  in tcp_flush_out_entry()
826 				csum = tcp_lro_update_checksum(&le->outer, le,  in tcp_flush_out_entry()
827 				    le->m_head->m_pkthdr.lro_tcp_d_len,  in tcp_flush_out_entry()
828 				    le->m_head->m_pkthdr.lro_tcp_d_csum);  in tcp_flush_out_entry()
829 				le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |  in tcp_flush_out_entry()
831 				le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
832 				if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED))  in tcp_flush_out_entry()
833 					le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;  in tcp_flush_out_entry()
836 				csum = tcp_lro_update_checksum(&le->outer, le,  in tcp_flush_out_entry()
837 				    le->m_head->m_pkthdr.lro_tcp_d_len,  in tcp_flush_out_entry()
838 				    le->m_head->m_pkthdr.lro_tcp_d_csum);  in tcp_flush_out_entry()
839 				le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |  in tcp_flush_out_entry()
841 				le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
842 				if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED))  in tcp_flush_out_entry()
843 					le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;  in tcp_flush_out_entry()
859 	le->m_head->m_nextpkt = NULL;  in tcp_flush_out_entry()
860 	lc->lro_queued += le->m_head->m_pkthdr.lro_nsegs;  in tcp_flush_out_entry()
861 	(*lc->ifp->if_input)(lc->ifp, le->m_head);  in tcp_flush_out_entry()
865 tcp_set_entry_to_mbuf(struct lro_ctrl *lc, struct lro_entry *le,  in tcp_set_entry_to_mbuf()  argument
881 		le->timestamp = 0;  in tcp_set_entry_to_mbuf()
883 		le->timestamp = 1;  in tcp_set_entry_to_mbuf()
884 		le->tsval = ntohl(*(ts_ptr + 1));  in tcp_set_entry_to_mbuf()
885 		le->tsecr = *(ts_ptr + 2);  in tcp_set_entry_to_mbuf()
891 	le->next_seq = ntohl(th->th_seq) + tcp_data_len;  in tcp_set_entry_to_mbuf()
892 	le->ack_seq = th->th_ack;  in tcp_set_entry_to_mbuf()
893 	le->window = th->th_win;  in tcp_set_entry_to_mbuf()
894 	le->flags = tcp_get_flags(th);  in tcp_set_entry_to_mbuf()
895 	le->needs_merge = 0;  in tcp_set_entry_to_mbuf()
898 	le->m_head = m;  in tcp_set_entry_to_mbuf()
899 	le->m_tail = m_last(m);  in tcp_set_entry_to_mbuf()
903 tcp_push_and_replace(struct lro_ctrl *lc, struct lro_entry *le, struct mbuf *m)  in tcp_push_and_replace()  argument
914 	msave = le->m_head->m_nextpkt;  in tcp_push_and_replace()
915 	le->m_head->m_nextpkt = NULL;  in tcp_push_and_replace()
918 	tcp_flush_out_entry(lc, le);  in tcp_push_and_replace()
921 	pa = tcp_lro_parser(m, &le->outer, &le->inner, false);  in tcp_push_and_replace()
930 	tcp_set_entry_to_mbuf(lc, le, m, pa->tcp);  in tcp_push_and_replace()
937 tcp_lro_mbuf_append_pkthdr(struct lro_entry *le, const struct mbuf *p)  in tcp_lro_mbuf_append_pkthdr()  argument
942 	m = le->m_head;  in tcp_lro_mbuf_append_pkthdr()
959 	le->needs_merge = 1;  in tcp_lro_mbuf_append_pkthdr()
963 tcp_lro_condense(struct lro_ctrl *lc, struct lro_entry *le)  in tcp_lro_condense()  argument
985 	m = le->m_head->m_nextpkt;  in tcp_lro_condense()
1002 		le->m_head->m_nextpkt = m->m_nextpkt;  in tcp_lro_condense()
1003 		tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1011 		le->m_head->m_nextpkt = m->m_nextpkt;  in tcp_lro_condense()
1012 		tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1015 	while((m = le->m_head->m_nextpkt) != NULL) {  in tcp_lro_condense()
1017 		 * condense m into le, first  in tcp_lro_condense()
1020 		le->m_head->m_nextpkt = m->m_nextpkt;  in tcp_lro_condense()
1028 		tcp_data_len_total = le->m_head->m_pkthdr.lro_tcp_d_len + tcp_data_len;  in tcp_lro_condense()
1029 		tcp_data_seg_total = le->m_head->m_pkthdr.lro_nsegs + m->m_pkthdr.lro_nsegs;  in tcp_lro_condense()
1034 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1043 			 * current le. We will go up to the beginning  in tcp_lro_condense()
1047 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1051 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1057 			if (TSTMP_GT(le->tsval, tsval))  {  in tcp_lro_condense()
1058 				tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1061 			le->tsval = tsval;  in tcp_lro_condense()
1062 			le->tsecr = *(ts_ptr + 2);  in tcp_lro_condense()
1065 		if (__predict_false(ntohl(th->th_seq) != le->next_seq ||  in tcp_lro_condense()
1067 				      (le->flags & TH_ACK)) ||  in tcp_lro_condense()
1069 				     le->ack_seq == th->th_ack &&  in tcp_lro_condense()
1070 				     le->window == th->th_win))) {  in tcp_lro_condense()
1072 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1076 		    SEQ_GT(ntohl(th->th_ack), ntohl(le->ack_seq))) {  in tcp_lro_condense()
1077 			le->next_seq += tcp_data_len;  in tcp_lro_condense()
1078 			le->ack_seq = th->th_ack;  in tcp_lro_condense()
1079 			le->window = th->th_win;  in tcp_lro_condense()
1080 			le->needs_merge = 1;  in tcp_lro_condense()
1081 		} else if (th->th_ack == le->ack_seq) {  in tcp_lro_condense()
1082 			if (WIN_GT(th->th_win, le->window)) {  in tcp_lro_condense()
1083 				le->window = th->th_win;  in tcp_lro_condense()
1084 				le->needs_merge = 1;  in tcp_lro_condense()
1094 		tcp_lro_mbuf_append_pkthdr(le, m);  in tcp_lro_condense()
1103 		le->m_tail->m_next = m;  in tcp_lro_condense()
1104 		le->m_tail = m_last(m);  in tcp_lro_condense()
1109 tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le)  in tcp_lro_flush()  argument
1115 	    tcp_lro_flush_tcphpts(lc, le) != 0) {  in tcp_lro_flush()
1116 		tcp_lro_condense(lc, le);  in tcp_lro_flush()
1117 		tcp_flush_out_entry(lc, le);  in tcp_lro_flush()
1120 	bzero(le, sizeof(*le));  in tcp_lro_flush()
1121 	LIST_INSERT_HEAD(&lc->lro_free, le, next);  in tcp_lro_flush()
1285 	struct lro_entry *le;  in tcp_lro_rx_common()  local
1382 	LIST_FOREACH(le, bucket, hash_next) {  in tcp_lro_rx_common()
1384 		if (lro_address_compare(&po.data, &le->outer.data) == false ||  in tcp_lro_rx_common()
1385 		    lro_address_compare(&pi.data, &le->inner.data) == false)  in tcp_lro_rx_common()
1390 		    SEQ_LT(ntohl(th->th_ack), ntohl(le->ack_seq))) {  in tcp_lro_rx_common()
1396 		le->m_last_mbuf->m_nextpkt = m;  in tcp_lro_rx_common()
1398 		le->m_last_mbuf = m;  in tcp_lro_rx_common()
1407 	le = LIST_FIRST(&lc->lro_free);  in tcp_lro_rx_common()
1408 	LIST_REMOVE(le, next);  in tcp_lro_rx_common()
1409 	tcp_lro_active_insert(lc, bucket, le);  in tcp_lro_rx_common()
1412 	le->inner = pi;  in tcp_lro_rx_common()
1413 	le->outer = po;  in tcp_lro_rx_common()
1416 	le->alloc_time = lc->lro_last_queue_time;  in tcp_lro_rx_common()
1418 	tcp_set_entry_to_mbuf(lc, le, m, th);  in tcp_lro_rx_common()
1421 	le->m_last_mbuf = m;  in tcp_lro_rx_common()