83 static void	tcp_lro_rx_done(struct lro_ctrl *lc);
84 static int	tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m,
86 static void	tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le);
92 int	(*tcp_lro_flush_tcphpts)(struct lro_ctrl *lc, struct lro_entry *le);
151 tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_head *bucket,  in tcp_lro_active_insert()  argument
155 	LIST_INSERT_HEAD(&lc->lro_active, le, next);  in tcp_lro_active_insert()
168 tcp_lro_init(struct lro_ctrl *lc)  in tcp_lro_init()  argument
170 	return (tcp_lro_init_args(lc, NULL, tcp_lro_entries, 0));  in tcp_lro_init()
174 tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp,  in tcp_lro_init_args()  argument
181 	lc->lro_bad_csum = 0;  in tcp_lro_init_args()
182 	lc->lro_queued = 0;  in tcp_lro_init_args()
183 	lc->lro_flushed = 0;  in tcp_lro_init_args()
184 	lc->lro_mbuf_count = 0;  in tcp_lro_init_args()
185 	lc->lro_mbuf_max = lro_mbufs;  in tcp_lro_init_args()
186 	lc->lro_cnt = lro_entries;  in tcp_lro_init_args()
187 	lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX;  in tcp_lro_init_args()
188 	lc->lro_length_lim = TCP_LRO_LENGTH_MAX;  in tcp_lro_init_args()
189 	lc->ifp = ifp;  in tcp_lro_init_args()
190 	LIST_INIT(&lc->lro_free);  in tcp_lro_init_args()
191 	LIST_INIT(&lc->lro_active);  in tcp_lro_init_args()
194 	lc->lro_hash = phashinit_flags(lro_entries, M_LRO, &lc->lro_hashsz,  in tcp_lro_init_args()
196 	if (lc->lro_hash == NULL) {  in tcp_lro_init_args()
197 		memset(lc, 0, sizeof(*lc));  in tcp_lro_init_args()
204 	lc->lro_mbuf_data = (struct lro_mbuf_sort *)  in tcp_lro_init_args()
208 	if (lc->lro_mbuf_data == NULL) {  in tcp_lro_init_args()
209 		free(lc->lro_hash, M_LRO);  in tcp_lro_init_args()
210 		memset(lc, 0, sizeof(*lc));  in tcp_lro_init_args()
215 	    (lc->lro_mbuf_data + lro_mbufs);  in tcp_lro_init_args()
219 		LIST_INSERT_HEAD(&lc->lro_free, le + i, next);  in tcp_lro_init_args()
492 tcp_lro_free(struct lro_ctrl *lc)  in tcp_lro_free()  argument
498 	LIST_INIT(&lc->lro_free);  in tcp_lro_free()
501 	while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {  in tcp_lro_free()
507 	free(lc->lro_hash, M_LRO);  in tcp_lro_free()
508 	lc->lro_hash = NULL;  in tcp_lro_free()
509 	lc->lro_hashsz = 0;  in tcp_lro_free()
512 	for (x = 0; x != lc->lro_mbuf_count; x++)  in tcp_lro_free()
513 		m_freem(lc->lro_mbuf_data[x].mb);  in tcp_lro_free()
514 	lc->lro_mbuf_count = 0;  in tcp_lro_free()
517 	free(lc->lro_mbuf_data, M_LRO);  in tcp_lro_free()
518 	lc->lro_mbuf_data = NULL;  in tcp_lro_free()
586 tcp_lro_rx_done(struct lro_ctrl *lc)  in tcp_lro_rx_done()  argument
590 	while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {  in tcp_lro_rx_done()
592 		tcp_lro_flush(lc, le);  in tcp_lro_rx_done()
597 tcp_lro_flush_active(struct lro_ctrl *lc)  in tcp_lro_flush_active()  argument
611 	LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {  in tcp_lro_flush_active()
614 			tcp_lro_flush(lc, le);  in tcp_lro_flush_active()
620 tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout)  in tcp_lro_flush_inactive()  argument
627 	if (LIST_EMPTY(&lc->lro_active))  in tcp_lro_flush_inactive()
634 	LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {  in tcp_lro_flush_inactive()
637 			tcp_lro_flush(lc, le);  in tcp_lro_flush_inactive()
644 tcp_lro_rx_ipv4(struct lro_ctrl *lc, struct mbuf *m, struct ip *ip4)  in tcp_lro_rx_ipv4()  argument
651 			lc->lro_bad_csum++;  in tcp_lro_rx_ipv4()
657 			lc->lro_bad_csum++;  in tcp_lro_rx_ipv4()
790 tcp_flush_out_entry(struct lro_ctrl *lc, struct lro_entry *le)  in tcp_flush_out_entry()  argument
860 	lc->lro_queued += le->m_head->m_pkthdr.lro_nsegs;  in tcp_flush_out_entry()
861 	(*lc->ifp->if_input)(lc->ifp, le->m_head);  in tcp_flush_out_entry()
865 tcp_set_entry_to_mbuf(struct lro_ctrl *lc, struct lro_entry *le,  in tcp_set_entry_to_mbuf()  argument
903 tcp_push_and_replace(struct lro_ctrl *lc, struct lro_entry *le, struct mbuf *m)  in tcp_push_and_replace()  argument
918 	tcp_flush_out_entry(lc, le);  in tcp_push_and_replace()
930 	tcp_set_entry_to_mbuf(lc, le, m, pa->tcp);  in tcp_push_and_replace()
963 tcp_lro_condense(struct lro_ctrl *lc, struct lro_entry *le)  in tcp_lro_condense()  argument
1003 		tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1012 		tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1031 		if (tcp_data_seg_total >= lc->lro_ackcnt_lim ||  in tcp_lro_condense()
1032 		    tcp_data_len_total >= lc->lro_length_lim) {  in tcp_lro_condense()
1034 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1047 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1051 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1058 				tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1072 			tcp_push_and_replace(lc, le, m);  in tcp_lro_condense()
1109 tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le)  in tcp_lro_flush()  argument
1115 	    tcp_lro_flush_tcphpts(lc, le) != 0) {  in tcp_lro_flush()
1116 		tcp_lro_condense(lc, le);  in tcp_lro_flush()
1117 		tcp_flush_out_entry(lc, le);  in tcp_lro_flush()
1119 	lc->lro_flushed++;  in tcp_lro_flush()
1121 	LIST_INSERT_HEAD(&lc->lro_free, le, next);  in tcp_lro_flush()
1199 tcp_lro_flush_all(struct lro_ctrl *lc)  in tcp_lro_flush_all()  argument
1207 	if (lc->lro_mbuf_count == 0)  in tcp_lro_flush_all()
1209 	if (lc->lro_cpu_is_set == 0) {  in tcp_lro_flush_all()
1210 		if (lc->lro_last_cpu == curcpu) {  in tcp_lro_flush_all()
1211 			lc->lro_cnt_of_same_cpu++;  in tcp_lro_flush_all()
1213 			if (lc->lro_cnt_of_same_cpu > tcp_lro_cpu_set_thresh)  in tcp_lro_flush_all()
1214 				lc->lro_cpu_is_set = 1;  in tcp_lro_flush_all()
1216 			lc->lro_last_cpu = curcpu;  in tcp_lro_flush_all()
1217 			lc->lro_cnt_of_same_cpu = 0;  in tcp_lro_flush_all()
1220 	CURVNET_SET(lc->ifp->if_vnet);  in tcp_lro_flush_all()
1223 	binuptime(&lc->lro_last_queue_time);  in tcp_lro_flush_all()
1226 	tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count);  in tcp_lro_flush_all()
1230 	for (x = 0; x != lc->lro_mbuf_count; x++) {  in tcp_lro_flush_all()
1234 		mb = lc->lro_mbuf_data[x].mb;  in tcp_lro_flush_all()
1237 		nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24);  in tcp_lro_flush_all()
1244 			tcp_lro_rx_done(lc);  in tcp_lro_flush_all()
1248 		if (tcp_lro_rx_common(lc, mb, 0, false) != 0) {  in tcp_lro_flush_all()
1250  			tcp_lro_flush_active(lc);  in tcp_lro_flush_all()
1252 			(*lc->ifp->if_input)(lc->ifp, mb);  in tcp_lro_flush_all()
1253 			lc->lro_queued++;  in tcp_lro_flush_all()
1254 			lc->lro_flushed++;  in tcp_lro_flush_all()
1260 	tcp_lro_rx_done(lc);  in tcp_lro_flush_all()
1262 	lc->lro_mbuf_count = 0;  in tcp_lro_flush_all()
1266 tcp_lro_rx_get_bucket(struct lro_ctrl *lc, struct mbuf *m, struct lro_parser *parser)  in tcp_lro_rx_get_bucket()  argument
1276 	return (&lc->lro_hash[hash % lc->lro_hashsz]);  in tcp_lro_rx_get_bucket()
1280 tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, bool use_hash)  in tcp_lro_rx_common()  argument
1327 		error = tcp_lro_rx_ipv4(lc, m, pa->ip4);  in tcp_lro_rx_common()
1337 		m->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time);   in tcp_lro_rx_common()
1368 	m->m_pkthdr.rcvif = lc->ifp;  in tcp_lro_rx_common()
1376 		bucket = &lc->lro_hash[0];  in tcp_lro_rx_common()
1378 		bucket = tcp_lro_rx_get_bucket(lc, m, pa);  in tcp_lro_rx_common()
1403 	if (LIST_EMPTY(&lc->lro_free))  in tcp_lro_rx_common()
1407 	le = LIST_FIRST(&lc->lro_free);  in tcp_lro_rx_common()
1409 	tcp_lro_active_insert(lc, bucket, le);  in tcp_lro_rx_common()
1416 	le->alloc_time = lc->lro_last_queue_time;  in tcp_lro_rx_common()
1418 	tcp_set_entry_to_mbuf(lc, le, m, th);  in tcp_lro_rx_common()
1427 tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)  in tcp_lro_rx()  argument
1443 	binuptime(&lc->lro_last_queue_time);  in tcp_lro_rx()
1444 	CURVNET_SET(lc->ifp->if_vnet);  in tcp_lro_rx()
1445 	error = tcp_lro_rx_common(lc, m, csum, true);  in tcp_lro_rx()
1452 		tcp_lro_flush_active(lc);  in tcp_lro_rx()
1460 tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)  in tcp_lro_queue_mbuf()  argument
1464 	if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL ||  in tcp_lro_queue_mbuf()
1465 	    lc->lro_mbuf_max == 0)) {  in tcp_lro_queue_mbuf()
1472 	if (__predict_false((lc->ifp->if_capenable & IFCAP_LRO) == 0)) {  in tcp_lro_queue_mbuf()
1474 		(*lc->ifp->if_input) (lc->ifp, mb);  in tcp_lro_queue_mbuf()
1483  		binuptime(&lc->lro_last_queue_time);  in tcp_lro_queue_mbuf()
1484  		mb->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time);   in tcp_lro_queue_mbuf()
1489 	lc->lro_mbuf_data[lc->lro_mbuf_count].seq =  in tcp_lro_queue_mbuf()
1492 	    ((uint64_t)lc->lro_mbuf_count);  in tcp_lro_queue_mbuf()
1495 	lc->lro_mbuf_data[lc->lro_mbuf_count].mb = mb;  in tcp_lro_queue_mbuf()
1498 	if (__predict_false(++lc->lro_mbuf_count == lc->lro_mbuf_max))  in tcp_lro_queue_mbuf()
1499 		tcp_lro_flush_all(lc);  in tcp_lro_queue_mbuf()