88 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, lro,  CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
106     CTLFLAG_RDTUN | CTLFLAG_MPSAFE, &tcp_lro_entries, 0,
111     CTLFLAG_RDTUN | CTLFLAG_MPSAFE, &tcp_lro_cpu_set_thresh, 0,
114 static uint32_t tcp_less_accurate_lro_ts = 0;
116     CTLFLAG_MPSAFE, &tcp_less_accurate_lro_ts, 0,
170 	return (tcp_lro_init_args(lc, NULL, tcp_lro_entries, 0));  in tcp_lro_init()
181 	lc->lro_bad_csum = 0;  in tcp_lro_init_args()
182 	lc->lro_queued = 0;  in tcp_lro_init_args()
183 	lc->lro_flushed = 0;  in tcp_lro_init_args()
184 	lc->lro_mbuf_count = 0;  in tcp_lro_init_args()
197 		memset(lc, 0, sizeof(*lc));  in tcp_lro_init_args()
210 		memset(lc, 0, sizeof(*lc));  in tcp_lro_init_args()
218 	for (i = 0; i != lro_entries; i++)  in tcp_lro_init_args()
221 	return (0);  in tcp_lro_init_args()
237 		memset(parser, 0, sizeof(*parser));  in tcp_lro_low_level_parser()
247 			    vxh->vxlh_vni & htonl(0xffffff00);  in tcp_lro_low_level_parser()
267 	if (__predict_false(mlen <= 0))  in tcp_lro_low_level_parser()
435 			memset(pi, 0, sizeof(*pi));  in tcp_lro_parser()
470 		return (0);  in tcp_lro_trim_mbuf_chain()
473 		return (0);  in tcp_lro_trim_mbuf_chain()
509 	lc->lro_hashsz = 0;  in tcp_lro_free()
512 	for (x = 0; x != lc->lro_mbuf_count; x++)  in tcp_lro_free()
514 	lc->lro_mbuf_count = 0;  in tcp_lro_free()
537 	while (csum > 0xffff)  in tcp_lro_rx_csum_tcphdr()
538 		csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_rx_csum_tcphdr()
555 		cs = in6_cksum_pseudo(pa->ip6, ntohs(pa->ip6->ip6_plen), pa->ip6->ip6_nxt, 0);  in tcp_lro_rx_csum_data()
566 		cs = 0;		/* Keep compiler happy. */  in tcp_lro_rx_csum_data()
579 	while (c > 0xffff)  in tcp_lro_rx_csum_data()
580 		c = (c >> 16) + (c & 0xffff);  in tcp_lro_rx_csum_data()
650 		if (__predict_false((m->m_pkthdr.csum_flags & CSUM_IP_VALID) == 0)) {  in tcp_lro_rx_ipv4()
656 		if (__predict_false(csum != 0)) {  in tcp_lro_rx_ipv4()
661 	return (0);  in tcp_lro_rx_ipv4()
670 	csum = 0xffff - *ptr + value;  in tcp_lro_assign_and_checksum_16()
671 	while (csum > 0xffff)  in tcp_lro_assign_and_checksum_16()
672 		csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_assign_and_checksum_16()
689 		tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]);  in tcp_lro_update_checksum()
692 		csum = pa->ip4->ip_sum + 0xffff - temp[0];  in tcp_lro_update_checksum()
693 		while (csum > 0xffff)  in tcp_lro_update_checksum()
694 			csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_update_checksum()
701 		tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]);  in tcp_lro_update_checksum()
707 		tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]);  in tcp_lro_update_checksum()
710 		csum = pa->ip4->ip_sum + 0xffff - temp[0];  in tcp_lro_update_checksum()
711 		while (csum > 0xffff)  in tcp_lro_update_checksum()
712 			csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_update_checksum()
719 		tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]);  in tcp_lro_update_checksum()
723 		return (0);  in tcp_lro_update_checksum()
735 	if (le->timestamp != 0) {  in tcp_lro_update_checksum()
747 	csum = pa->tcp->th_sum + 0xffff - delta_sum +  in tcp_lro_update_checksum()
748 	    0xffff - temp[0] + 0xffff - temp[3] + temp[2];  in tcp_lro_update_checksum()
749 	while (csum > 0xffff)  in tcp_lro_update_checksum()
750 		csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_update_checksum()
756 	csum = temp[0] + temp[1] + 0xffff - temp[2] +  in tcp_lro_update_checksum()
758 	while (csum > 0xffff)  in tcp_lro_update_checksum()
759 		csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_update_checksum()
770 	if (__predict_false(pa->udp->uh_sum != 0)) {  in tcp_lro_update_checksum()
772 		csum = pa->udp->uh_sum + 0xffff - delta_sum +  in tcp_lro_update_checksum()
773 		    0xffff - temp[0] + 0xffff - temp[2];  in tcp_lro_update_checksum()
774 		while (csum > 0xffff)  in tcp_lro_update_checksum()
775 			csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_update_checksum()
781 	csum = temp[0] + temp[1] + temp[2] + temp[3] + delta_sum;  in tcp_lro_update_checksum()
782 	while (csum > 0xffff)  in tcp_lro_update_checksum()
783 		csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_update_checksum()
806 			le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
819 			le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
831 				le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
841 				le->m_head->m_pkthdr.csum_data = 0xffff;  in tcp_flush_out_entry()
877 	if (tcp_opt_len == 0 ||  in tcp_set_entry_to_mbuf()
881 		le->timestamp = 0;  in tcp_set_entry_to_mbuf()
895 	le->needs_merge = 0;  in tcp_set_entry_to_mbuf()
950 		while (csum > 0xffff)  in tcp_lro_mbuf_append_pkthdr()
951 			csum = (csum >> 16) + (csum & 0xffff);  in tcp_lro_mbuf_append_pkthdr()
996 	if (tcp_opt_len != 0 && __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA ||  in tcp_lro_condense()
1006 	if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH)) != 0) {  in tcp_lro_condense()
1037 		if (tcp_opt_len != 0 &&  in tcp_lro_condense()
1050 		if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH)) != 0) {  in tcp_lro_condense()
1054 		if (tcp_opt_len != 0) {  in tcp_lro_condense()
1068 				    (tcp_data_len == 0 &&  in tcp_lro_condense()
1075 		if (tcp_data_len != 0 ||  in tcp_lro_condense()
1088 		if (tcp_data_len == 0) {  in tcp_lro_condense()
1115 	    tcp_lro_flush_tcphpts(lc, le) != 0) {  in tcp_lro_flush()
1149 			for (y = x; y > 0 && temp.seq < parray[y - 1].seq; y--)  in tcp_lro_sort()
1157 	ones = 0;  in tcp_lro_sort()
1158 	zeros = 0;  in tcp_lro_sort()
1159 	for (x = 0; x != size; x++) {  in tcp_lro_sort()
1166 	if (ones == 0)  in tcp_lro_sort()
1176 	for (x = y = 0; y != size; y++) {  in tcp_lro_sort()
1187 	KASSERT(x != 0 && x != size, ("Memory is corrupted\n"));  in tcp_lro_sort()
1207 	if (lc->lro_mbuf_count == 0)  in tcp_lro_flush_all()
1209 	if (lc->lro_cpu_is_set == 0) {  in tcp_lro_flush_all()
1217 			lc->lro_cnt_of_same_cpu = 0;  in tcp_lro_flush_all()
1229 	seq = 0;  in tcp_lro_flush_all()
1230 	for (x = 0; x != lc->lro_mbuf_count; x++) {  in tcp_lro_flush_all()
1248 		if (tcp_lro_rx_common(lc, mb, 0, false) != 0) {  in tcp_lro_flush_all()
1262 	lc->lro_mbuf_count = 0;  in tcp_lro_flush_all()
1273 		for (unsigned i = hash = 0; i != LRO_RAW_ADDRESS_MAX; i++)  in tcp_lro_rx_get_bucket()
1295 	if (__predict_false(V_ipforwarding != 0))  in tcp_lro_rx_common()
1300 	if (__predict_false(V_ip6_forwarding != 0))  in tcp_lro_rx_common()
1305 	    (m->m_pkthdr.csum_data != 0xffff)) {  in tcp_lro_rx_common()
1321 	if (__predict_false(error != 0))  in tcp_lro_rx_common()
1328 		if (__predict_false(error != 0))  in tcp_lro_rx_common()
1336 	if ((m->m_flags & (M_TSTMP_LRO | M_TSTMP)) == 0) {  in tcp_lro_rx_common()
1355 	if (__predict_false(tcp_opt_len < 0 || tcp_data_len < 0))  in tcp_lro_rx_common()
1359 	if (tcp_data_len == 0)  in tcp_lro_rx_common()
1360 		tcp_data_sum = 0;	/* no data, no checksum */  in tcp_lro_rx_common()
1361 	else if (__predict_false(csum != 0))  in tcp_lro_rx_common()
1376 		bucket = &lc->lro_hash[0];  in tcp_lro_rx_common()
1389 		if (tcp_data_len == 0 &&  in tcp_lro_rx_common()
1392 			return (0);  in tcp_lro_rx_common()
1399 		return (0);  in tcp_lro_rx_common()
1423 	return (0);  in tcp_lro_rx_common()
1433 	    (m->m_pkthdr.csum_data != 0xffff)) {  in tcp_lro_rx()
1446 	if (__predict_false(error != 0)) {  in tcp_lro_rx()
1465 	    lc->lro_mbuf_max == 0)) {  in tcp_lro_queue_mbuf()
1472 	if (__predict_false((lc->ifp->if_capenable & IFCAP_LRO) == 0)) {  in tcp_lro_queue_mbuf()
1479  	if ((tcplro_stacks_wanting_mbufq > 0) &&  in tcp_lro_queue_mbuf()
1480  	    (tcp_less_accurate_lro_ts == 0) &&  in tcp_lro_queue_mbuf()
1481  	    ((mb->m_flags & M_TSTMP) == 0)) {  in tcp_lro_queue_mbuf()