Lines Matching +full:delta +full:- +full:y +full:- +full:threshold
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
7 * Copyright (c) 2016-2021 Mellanox Technologies.
147 atomic_fetchadd_long(&tcplro_stacks_wanting_mbufq, -1); in tcp_lro_dereg_mbufq()
155 LIST_INSERT_HEAD(&lc->lro_active, le, next); in tcp_lro_active_insert()
181 lc->lro_bad_csum = 0; in tcp_lro_init_args()
182 lc->lro_queued = 0; in tcp_lro_init_args()
183 lc->lro_flushed = 0; in tcp_lro_init_args()
184 lc->lro_mbuf_count = 0; in tcp_lro_init_args()
185 lc->lro_mbuf_max = lro_mbufs; in tcp_lro_init_args()
186 lc->lro_cnt = lro_entries; in tcp_lro_init_args()
187 lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX; in tcp_lro_init_args()
188 lc->lro_length_lim = TCP_LRO_LENGTH_MAX; in tcp_lro_init_args()
189 lc->ifp = ifp; in tcp_lro_init_args()
190 LIST_INIT(&lc->lro_free); in tcp_lro_init_args()
191 LIST_INIT(&lc->lro_active); in tcp_lro_init_args()
194 lc->lro_hash = phashinit_flags(lro_entries, M_LRO, &lc->lro_hashsz, in tcp_lro_init_args()
196 if (lc->lro_hash == NULL) { in tcp_lro_init_args()
204 lc->lro_mbuf_data = (struct lro_mbuf_sort *) in tcp_lro_init_args()
208 if (lc->lro_mbuf_data == NULL) { in tcp_lro_init_args()
209 free(lc->lro_hash, M_LRO); in tcp_lro_init_args()
215 (lc->lro_mbuf_data + lro_mbufs); in tcp_lro_init_args()
219 LIST_INSERT_HEAD(&lc->lro_free, le + i, next); in tcp_lro_init_args()
246 parser->data.vxlan_vni = in tcp_lro_low_level_parser()
247 vxh->vxlh_vni & htonl(0xffffff00); in tcp_lro_low_level_parser()
252 if (__predict_false(eh->evl_encap_proto == htons(ETHERTYPE_VLAN))) { in tcp_lro_low_level_parser()
253 eth_type = eh->evl_proto; in tcp_lro_low_level_parser()
256 parser->data.vlan_id = eh->evl_tag & htons(EVL_VLID_MASK); in tcp_lro_low_level_parser()
260 mlen -= (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); in tcp_lro_low_level_parser()
262 eth_type = eh->evl_encap_proto; in tcp_lro_low_level_parser()
264 mlen -= ETHER_HDR_LEN; in tcp_lro_low_level_parser()
272 parser->ip4 = ptr; in tcp_lro_low_level_parser()
276 if ((parser->ip4->ip_hl << 2) != sizeof (*parser->ip4)) in tcp_lro_low_level_parser()
279 if (parser->ip4->ip_off & htons(IP_MF|IP_OFFMASK)) in tcp_lro_low_level_parser()
282 if (__predict_false(parser->ip4->ip_src.s_addr == INADDR_ANY || in tcp_lro_low_level_parser()
283 parser->ip4->ip_dst.s_addr == INADDR_ANY)) in tcp_lro_low_level_parser()
285 ptr = (uint8_t *)ptr + (parser->ip4->ip_hl << 2); in tcp_lro_low_level_parser()
286 mlen -= sizeof(struct ip); in tcp_lro_low_level_parser()
288 parser->data.s_addr.v4 = parser->ip4->ip_src; in tcp_lro_low_level_parser()
289 parser->data.d_addr.v4 = parser->ip4->ip_dst; in tcp_lro_low_level_parser()
291 switch (parser->ip4->ip_p) { in tcp_lro_low_level_parser()
295 parser->udp = ptr; in tcp_lro_low_level_parser()
297 parser->data.lro_type = LRO_TYPE_IPV4_UDP; in tcp_lro_low_level_parser()
298 parser->data.s_port = parser->udp->uh_sport; in tcp_lro_low_level_parser()
299 parser->data.d_port = parser->udp->uh_dport; in tcp_lro_low_level_parser()
301 MPASS(parser->data.lro_type == LRO_TYPE_IPV4_UDP); in tcp_lro_low_level_parser()
303 ptr = ((uint8_t *)ptr + sizeof(*parser->udp)); in tcp_lro_low_level_parser()
304 parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; in tcp_lro_low_level_parser()
307 parser->tcp = ptr; in tcp_lro_low_level_parser()
311 parser->data.lro_type = LRO_TYPE_IPV4_TCP; in tcp_lro_low_level_parser()
312 parser->data.s_port = parser->tcp->th_sport; in tcp_lro_low_level_parser()
313 parser->data.d_port = parser->tcp->th_dport; in tcp_lro_low_level_parser()
315 MPASS(parser->data.lro_type == LRO_TYPE_IPV4_TCP); in tcp_lro_low_level_parser()
317 if (__predict_false(mlen < (parser->tcp->th_off << 2))) in tcp_lro_low_level_parser()
319 ptr = (uint8_t *)ptr + (parser->tcp->th_off << 2); in tcp_lro_low_level_parser()
320 parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; in tcp_lro_low_level_parser()
329 parser->ip6 = ptr; in tcp_lro_low_level_parser()
333 if (__predict_false(IN6_IS_ADDR_UNSPECIFIED(&parser->ip6->ip6_src) || in tcp_lro_low_level_parser()
334 IN6_IS_ADDR_UNSPECIFIED(&parser->ip6->ip6_dst))) in tcp_lro_low_level_parser()
336 ptr = (uint8_t *)ptr + sizeof(*parser->ip6); in tcp_lro_low_level_parser()
338 parser->data.s_addr.v6 = parser->ip6->ip6_src; in tcp_lro_low_level_parser()
339 parser->data.d_addr.v6 = parser->ip6->ip6_dst; in tcp_lro_low_level_parser()
341 mlen -= sizeof(struct ip6_hdr); in tcp_lro_low_level_parser()
342 switch (parser->ip6->ip6_nxt) { in tcp_lro_low_level_parser()
346 parser->udp = ptr; in tcp_lro_low_level_parser()
348 parser->data.lro_type = LRO_TYPE_IPV6_UDP; in tcp_lro_low_level_parser()
349 parser->data.s_port = parser->udp->uh_sport; in tcp_lro_low_level_parser()
350 parser->data.d_port = parser->udp->uh_dport; in tcp_lro_low_level_parser()
352 MPASS(parser->data.lro_type == LRO_TYPE_IPV6_UDP); in tcp_lro_low_level_parser()
354 ptr = (uint8_t *)ptr + sizeof(*parser->udp); in tcp_lro_low_level_parser()
355 parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; in tcp_lro_low_level_parser()
360 parser->tcp = ptr; in tcp_lro_low_level_parser()
362 parser->data.lro_type = LRO_TYPE_IPV6_TCP; in tcp_lro_low_level_parser()
363 parser->data.s_port = parser->tcp->th_sport; in tcp_lro_low_level_parser()
364 parser->data.d_port = parser->tcp->th_dport; in tcp_lro_low_level_parser()
366 MPASS(parser->data.lro_type == LRO_TYPE_IPV6_TCP); in tcp_lro_low_level_parser()
368 if (__predict_false(mlen < (parser->tcp->th_off << 2))) in tcp_lro_low_level_parser()
370 ptr = (uint8_t *)ptr + (parser->tcp->th_off << 2); in tcp_lro_low_level_parser()
371 parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; in tcp_lro_low_level_parser()
381 /* Invalid packet - cannot parse */ in tcp_lro_low_level_parser()
394 data_ptr = tcp_lro_low_level_parser(m->m_data, po, update_data, false, m->m_len); in tcp_lro_parser()
395 if (data_ptr == NULL || po->total_hdr_len > m->m_len) in tcp_lro_parser()
400 if (__predict_false(m->m_flags & M_VLANTAG)) { in tcp_lro_parser()
401 po->data.vlan_id = in tcp_lro_parser()
402 htons(m->m_pkthdr.ether_vtag) & htons(EVL_VLID_MASK); in tcp_lro_parser()
405 if (__predict_false((m->m_pkthdr.csum_flags & in tcp_lro_parser()
407 po->data.lro_flags |= LRO_FLAG_DECRYPTED; in tcp_lro_parser()
410 switch (po->data.lro_type) { in tcp_lro_parser()
414 if ((m->m_pkthdr.csum_flags & vxlan_csum) != vxlan_csum) in tcp_lro_parser()
419 (m->m_len - ((caddr_t)data_ptr - m->m_data))); in tcp_lro_parser()
420 if (data_ptr == NULL || (pi->total_hdr_len + po->total_hdr_len) > m->m_len) in tcp_lro_parser()
424 switch (pi->data.lro_type) { in tcp_lro_parser()
448 switch (po->data.lro_type) { in tcp_lro_trim_mbuf_chain()
451 len = ((uint8_t *)po->ip4 - (uint8_t *)m->m_data) + in tcp_lro_trim_mbuf_chain()
452 ntohs(po->ip4->ip_len); in tcp_lro_trim_mbuf_chain()
457 len = ((uint8_t *)po->ip6 - (uint8_t *)m->m_data) + in tcp_lro_trim_mbuf_chain()
458 ntohs(po->ip6->ip6_plen) + sizeof(*po->ip6); in tcp_lro_trim_mbuf_chain()
469 if (__predict_true(m->m_pkthdr.len == len)) { in tcp_lro_trim_mbuf_chain()
471 } else if (m->m_pkthdr.len > len) { in tcp_lro_trim_mbuf_chain()
472 m_adj(m, len - m->m_pkthdr.len); in tcp_lro_trim_mbuf_chain()
484 save = m->m_nextpkt; in lro_free_mbuf_chain()
485 m->m_nextpkt = NULL; in lro_free_mbuf_chain()
498 LIST_INIT(&lc->lro_free); in tcp_lro_free()
501 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { in tcp_lro_free()
503 lro_free_mbuf_chain(le->m_head); in tcp_lro_free()
507 free(lc->lro_hash, M_LRO); in tcp_lro_free()
508 lc->lro_hash = NULL; in tcp_lro_free()
509 lc->lro_hashsz = 0; in tcp_lro_free()
512 for (x = 0; x != lc->lro_mbuf_count; x++) in tcp_lro_free()
513 m_freem(lc->lro_mbuf_data[x].mb); in tcp_lro_free()
514 lc->lro_mbuf_count = 0; in tcp_lro_free()
517 free(lc->lro_mbuf_data, M_LRO); in tcp_lro_free()
518 lc->lro_mbuf_data = NULL; in tcp_lro_free()
528 csum = -th->th_sum; /* exclude checksum field */ in tcp_lro_rx_csum_tcphdr()
529 len = th->th_off; in tcp_lro_rx_csum_tcphdr()
531 while (len--) { in tcp_lro_rx_csum_tcphdr()
551 switch (pa->data.lro_type) { in tcp_lro_rx_csum_data()
555 cs = in6_cksum_pseudo(pa->ip6, ntohs(pa->ip6->ip6_plen), pa->ip6->ip6_nxt, 0); in tcp_lro_rx_csum_data()
561 cs = in_addword(ntohs(pa->ip4->ip_len) - sizeof(*pa->ip4), IPPROTO_TCP); in tcp_lro_rx_csum_data()
562 cs = in_pseudo(pa->ip4->ip_src.s_addr, pa->ip4->ip_dst.s_addr, htons(cs)); in tcp_lro_rx_csum_data()
575 cs = ~tcp_lro_rx_csum_tcphdr(pa->tcp); in tcp_lro_rx_csum_data()
590 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { in tcp_lro_rx_done()
611 LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { in tcp_lro_flush_active()
612 if (le->m_head != NULL) { in tcp_lro_flush_active()
627 if (LIST_EMPTY(&lc->lro_active)) in tcp_lro_flush_inactive()
633 tov = ((timeout->tv_sec * 1000000000) + (timeout->tv_usec * 1000)); in tcp_lro_flush_inactive()
634 LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { in tcp_lro_flush_inactive()
635 if (now >= (bintime2ns(&le->alloc_time) + tov)) { in tcp_lro_flush_inactive()
649 if (m->m_pkthdr.csum_flags & CSUM_IP_CHECKED) { in tcp_lro_rx_ipv4()
650 if (__predict_false((m->m_pkthdr.csum_flags & CSUM_IP_VALID) == 0)) { in tcp_lro_rx_ipv4()
651 lc->lro_bad_csum++; in tcp_lro_rx_ipv4()
657 lc->lro_bad_csum++; in tcp_lro_rx_ipv4()
670 csum = 0xffff - *ptr + value; in tcp_lro_assign_and_checksum_16()
685 switch (pa->data.lro_type) { in tcp_lro_update_checksum()
688 tlen = (pa->ip4->ip_hl << 2) + (pa->tcp->th_off << 2) + payload_len; in tcp_lro_update_checksum()
689 tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]); in tcp_lro_update_checksum()
691 /* Subtract delta from current IPv4 checksum. */ in tcp_lro_update_checksum()
692 csum = pa->ip4->ip_sum + 0xffff - temp[0]; in tcp_lro_update_checksum()
695 tcp_lro_assign_and_checksum_16(&pa->ip4->ip_sum, csum, &temp[1]); in tcp_lro_update_checksum()
700 tlen = (pa->tcp->th_off << 2) + payload_len; in tcp_lro_update_checksum()
701 tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]); in tcp_lro_update_checksum()
706 tlen = (pa->ip4->ip_hl << 2) + sizeof(*pa->udp) + payload_len; in tcp_lro_update_checksum()
707 tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]); in tcp_lro_update_checksum()
709 /* Subtract delta from current IPv4 checksum. */ in tcp_lro_update_checksum()
710 csum = pa->ip4->ip_sum + 0xffff - temp[0]; in tcp_lro_update_checksum()
713 tcp_lro_assign_and_checksum_16(&pa->ip4->ip_sum, csum, &temp[1]); in tcp_lro_update_checksum()
718 tlen = sizeof(*pa->udp) + payload_len; in tcp_lro_update_checksum()
719 tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]); in tcp_lro_update_checksum()
728 temp[2] = tcp_lro_rx_csum_tcphdr(pa->tcp); in tcp_lro_update_checksum()
731 pa->tcp->th_ack = le->ack_seq; in tcp_lro_update_checksum()
732 pa->tcp->th_win = le->window; in tcp_lro_update_checksum()
735 if (le->timestamp != 0) { in tcp_lro_update_checksum()
738 ts_ptr = (uint32_t *)(pa->tcp + 1); in tcp_lro_update_checksum()
739 ts_ptr[1] = htonl(le->tsval); in tcp_lro_update_checksum()
740 ts_ptr[2] = le->tsecr; in tcp_lro_update_checksum()
744 temp[3] = tcp_lro_rx_csum_tcphdr(pa->tcp); in tcp_lro_update_checksum()
747 csum = pa->tcp->th_sum + 0xffff - delta_sum + in tcp_lro_update_checksum()
748 0xffff - temp[0] + 0xffff - temp[3] + temp[2]; in tcp_lro_update_checksum()
753 tcp_lro_assign_and_checksum_16(&pa->tcp->th_sum, csum, &temp[4]); in tcp_lro_update_checksum()
756 csum = temp[0] + temp[1] + 0xffff - temp[2] + in tcp_lro_update_checksum()
761 /* Return delta checksum to next stage, if any. */ in tcp_lro_update_checksum()
765 tlen = sizeof(*pa->udp) + payload_len; in tcp_lro_update_checksum()
766 /* Assign new UDP length and compute checksum delta. */ in tcp_lro_update_checksum()
767 tcp_lro_assign_and_checksum_16(&pa->udp->uh_ulen, htons(tlen), &temp[2]); in tcp_lro_update_checksum()
770 if (__predict_false(pa->udp->uh_sum != 0)) { in tcp_lro_update_checksum()
772 csum = pa->udp->uh_sum + 0xffff - delta_sum + in tcp_lro_update_checksum()
773 0xffff - temp[0] + 0xffff - temp[2]; in tcp_lro_update_checksum()
777 tcp_lro_assign_and_checksum_16(&pa->udp->uh_sum, csum, &temp[3]); in tcp_lro_update_checksum()
785 /* Return delta checksum to next stage, if any. */ in tcp_lro_update_checksum()
793 if (le->needs_merge) { in tcp_flush_out_entry()
796 switch (le->inner.data.lro_type) { in tcp_flush_out_entry()
798 csum = tcp_lro_update_checksum(&le->inner, le, in tcp_flush_out_entry()
799 le->m_head->m_pkthdr.lro_tcp_d_len, in tcp_flush_out_entry()
800 le->m_head->m_pkthdr.lro_tcp_d_csum); in tcp_flush_out_entry()
801 csum = tcp_lro_update_checksum(&le->outer, NULL, in tcp_flush_out_entry()
802 le->m_head->m_pkthdr.lro_tcp_d_len + in tcp_flush_out_entry()
803 le->inner.total_hdr_len, csum); in tcp_flush_out_entry()
804 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | in tcp_flush_out_entry()
806 le->m_head->m_pkthdr.csum_data = 0xffff; in tcp_flush_out_entry()
807 if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) in tcp_flush_out_entry()
808 le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; in tcp_flush_out_entry()
811 csum = tcp_lro_update_checksum(&le->inner, le, in tcp_flush_out_entry()
812 le->m_head->m_pkthdr.lro_tcp_d_len, in tcp_flush_out_entry()
813 le->m_head->m_pkthdr.lro_tcp_d_csum); in tcp_flush_out_entry()
814 csum = tcp_lro_update_checksum(&le->outer, NULL, in tcp_flush_out_entry()
815 le->m_head->m_pkthdr.lro_tcp_d_len + in tcp_flush_out_entry()
816 le->inner.total_hdr_len, csum); in tcp_flush_out_entry()
817 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | in tcp_flush_out_entry()
819 le->m_head->m_pkthdr.csum_data = 0xffff; in tcp_flush_out_entry()
820 if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) in tcp_flush_out_entry()
821 le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; in tcp_flush_out_entry()
824 switch (le->outer.data.lro_type) { in tcp_flush_out_entry()
826 csum = tcp_lro_update_checksum(&le->outer, le, in tcp_flush_out_entry()
827 le->m_head->m_pkthdr.lro_tcp_d_len, in tcp_flush_out_entry()
828 le->m_head->m_pkthdr.lro_tcp_d_csum); in tcp_flush_out_entry()
829 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | in tcp_flush_out_entry()
831 le->m_head->m_pkthdr.csum_data = 0xffff; in tcp_flush_out_entry()
832 if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) in tcp_flush_out_entry()
833 le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; in tcp_flush_out_entry()
836 csum = tcp_lro_update_checksum(&le->outer, le, in tcp_flush_out_entry()
837 le->m_head->m_pkthdr.lro_tcp_d_len, in tcp_flush_out_entry()
838 le->m_head->m_pkthdr.lro_tcp_d_csum); in tcp_flush_out_entry()
839 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | in tcp_flush_out_entry()
841 le->m_head->m_pkthdr.csum_data = 0xffff; in tcp_flush_out_entry()
842 if (__predict_false(le->outer.data.lro_flags & LRO_FLAG_DECRYPTED)) in tcp_flush_out_entry()
843 le->m_head->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED; in tcp_flush_out_entry()
859 le->m_head->m_nextpkt = NULL; in tcp_flush_out_entry()
860 lc->lro_queued += le->m_head->m_pkthdr.lro_nsegs; in tcp_flush_out_entry()
861 (*lc->ifp->if_input)(lc->ifp, le->m_head); in tcp_flush_out_entry()
873 tcp_opt_len = (th->th_off << 2); in tcp_set_entry_to_mbuf()
874 tcp_opt_len -= sizeof(*th); in tcp_set_entry_to_mbuf()
881 le->timestamp = 0; in tcp_set_entry_to_mbuf()
883 le->timestamp = 1; in tcp_set_entry_to_mbuf()
884 le->tsval = ntohl(*(ts_ptr + 1)); in tcp_set_entry_to_mbuf()
885 le->tsecr = *(ts_ptr + 2); in tcp_set_entry_to_mbuf()
888 tcp_data_len = m->m_pkthdr.lro_tcp_d_len; in tcp_set_entry_to_mbuf()
891 le->next_seq = ntohl(th->th_seq) + tcp_data_len; in tcp_set_entry_to_mbuf()
892 le->ack_seq = th->th_ack; in tcp_set_entry_to_mbuf()
893 le->window = th->th_win; in tcp_set_entry_to_mbuf()
894 le->flags = tcp_get_flags(th); in tcp_set_entry_to_mbuf()
895 le->needs_merge = 0; in tcp_set_entry_to_mbuf()
898 le->m_head = m; in tcp_set_entry_to_mbuf()
899 le->m_tail = m_last(m); in tcp_set_entry_to_mbuf()
914 msave = le->m_head->m_nextpkt; in tcp_push_and_replace()
915 le->m_head->m_nextpkt = NULL; in tcp_push_and_replace()
920 /* Re-parse new header, should not fail. */ in tcp_push_and_replace()
921 pa = tcp_lro_parser(m, &le->outer, &le->inner, false); in tcp_push_and_replace()
930 tcp_set_entry_to_mbuf(lc, le, m, pa->tcp); in tcp_push_and_replace()
933 m->m_nextpkt = msave; in tcp_push_and_replace()
942 m = le->m_head; in tcp_lro_mbuf_append_pkthdr()
943 if (m->m_pkthdr.lro_nsegs == 1) { in tcp_lro_mbuf_append_pkthdr()
945 csum = p->m_pkthdr.lro_tcp_d_csum; in tcp_lro_mbuf_append_pkthdr()
948 csum = (uint32_t)m->m_pkthdr.lro_tcp_d_csum + in tcp_lro_mbuf_append_pkthdr()
949 (uint32_t)p->m_pkthdr.lro_tcp_d_csum; in tcp_lro_mbuf_append_pkthdr()
955 m->m_pkthdr.len += p->m_pkthdr.lro_tcp_d_len; in tcp_lro_mbuf_append_pkthdr()
956 m->m_pkthdr.lro_tcp_d_csum = csum; in tcp_lro_mbuf_append_pkthdr()
957 m->m_pkthdr.lro_tcp_d_len += p->m_pkthdr.lro_tcp_d_len; in tcp_lro_mbuf_append_pkthdr()
958 m->m_pkthdr.lro_nsegs += p->m_pkthdr.lro_nsegs; in tcp_lro_mbuf_append_pkthdr()
959 le->needs_merge = 1; in tcp_lro_mbuf_append_pkthdr()
985 m = le->m_head->m_nextpkt; in tcp_lro_condense()
992 tcp_opt_len = (th->th_off << 2); in tcp_lro_condense()
993 tcp_opt_len -= sizeof(*th); in tcp_lro_condense()
1002 le->m_head->m_nextpkt = m->m_nextpkt; in tcp_lro_condense()
1011 le->m_head->m_nextpkt = m->m_nextpkt; in tcp_lro_condense()
1015 while((m = le->m_head->m_nextpkt) != NULL) { in tcp_lro_condense()
1020 le->m_head->m_nextpkt = m->m_nextpkt; in tcp_lro_condense()
1021 m->m_nextpkt = NULL; in tcp_lro_condense()
1023 tcp_data_len = m->m_pkthdr.lro_tcp_d_len; in tcp_lro_condense()
1026 tcp_opt_len = (th->th_off << 2); in tcp_lro_condense()
1027 tcp_opt_len -= sizeof(*th); in tcp_lro_condense()
1028 tcp_data_len_total = le->m_head->m_pkthdr.lro_tcp_d_len + tcp_data_len; in tcp_lro_condense()
1029 tcp_data_seg_total = le->m_head->m_pkthdr.lro_nsegs + m->m_pkthdr.lro_nsegs; in tcp_lro_condense()
1031 if (tcp_data_seg_total >= lc->lro_ackcnt_lim || in tcp_lro_condense()
1032 tcp_data_len_total >= lc->lro_length_lim) { in tcp_lro_condense()
1057 if (TSTMP_GT(le->tsval, tsval)) { in tcp_lro_condense()
1061 le->tsval = tsval; in tcp_lro_condense()
1062 le->tsecr = *(ts_ptr + 2); in tcp_lro_condense()
1065 if (__predict_false(ntohl(th->th_seq) != le->next_seq || in tcp_lro_condense()
1067 (le->flags & TH_ACK)) || in tcp_lro_condense()
1069 le->ack_seq == th->th_ack && in tcp_lro_condense()
1070 le->window == th->th_win))) { in tcp_lro_condense()
1071 /* Out of order packet, non-ACK + ACK or dup ACK. */ in tcp_lro_condense()
1076 SEQ_GT(ntohl(th->th_ack), ntohl(le->ack_seq))) { in tcp_lro_condense()
1077 le->next_seq += tcp_data_len; in tcp_lro_condense()
1078 le->ack_seq = th->th_ack; in tcp_lro_condense()
1079 le->window = th->th_win; in tcp_lro_condense()
1080 le->needs_merge = 1; in tcp_lro_condense()
1081 } else if (th->th_ack == le->ack_seq) { in tcp_lro_condense()
1082 if (WIN_GT(th->th_win, le->window)) { in tcp_lro_condense()
1083 le->window = th->th_win; in tcp_lro_condense()
1084 le->needs_merge = 1; in tcp_lro_condense()
1101 m_adj(m, m->m_pkthdr.len - tcp_data_len); in tcp_lro_condense()
1103 le->m_tail->m_next = m; in tcp_lro_condense()
1104 le->m_tail = m_last(m); in tcp_lro_condense()
1119 lc->lro_flushed++; in tcp_lro_flush()
1121 LIST_INSERT_HEAD(&lc->lro_free, le, next); in tcp_lro_flush()
1124 #define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1))
1130 * available. The algorithm is bit-slicing the 64-bit sequence number,
1142 uint32_t y; in tcp_lro_sort() local
1149 for (y = x; y > 0 && temp.seq < parray[y - 1].seq; y--) in tcp_lro_sort()
1150 parray[y] = parray[y - 1]; in tcp_lro_sort()
1151 parray[y] = temp; in tcp_lro_sort()
1176 for (x = y = 0; y != size; y++) { in tcp_lro_sort()
1178 if (parray[y].seq & ones) in tcp_lro_sort()
1182 parray[x] = parray[y]; in tcp_lro_sort()
1183 parray[y] = temp; in tcp_lro_sort()
1194 size -= x; in tcp_lro_sort()
1207 if (lc->lro_mbuf_count == 0) in tcp_lro_flush_all()
1209 if (lc->lro_cpu_is_set == 0) { in tcp_lro_flush_all()
1210 if (lc->lro_last_cpu == curcpu) { in tcp_lro_flush_all()
1211 lc->lro_cnt_of_same_cpu++; in tcp_lro_flush_all()
1212 /* Have we reached the threshold to declare a cpu? */ in tcp_lro_flush_all()
1213 if (lc->lro_cnt_of_same_cpu > tcp_lro_cpu_set_thresh) in tcp_lro_flush_all()
1214 lc->lro_cpu_is_set = 1; in tcp_lro_flush_all()
1216 lc->lro_last_cpu = curcpu; in tcp_lro_flush_all()
1217 lc->lro_cnt_of_same_cpu = 0; in tcp_lro_flush_all()
1220 CURVNET_SET(lc->ifp->if_vnet); in tcp_lro_flush_all()
1223 binuptime(&lc->lro_last_queue_time); in tcp_lro_flush_all()
1226 tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count); in tcp_lro_flush_all()
1230 for (x = 0; x != lc->lro_mbuf_count; x++) { in tcp_lro_flush_all()
1234 mb = lc->lro_mbuf_data[x].mb; in tcp_lro_flush_all()
1237 nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24); in tcp_lro_flush_all()
1252 (*lc->ifp->if_input)(lc->ifp, mb); in tcp_lro_flush_all()
1253 lc->lro_queued++; in tcp_lro_flush_all()
1254 lc->lro_flushed++; in tcp_lro_flush_all()
1262 lc->lro_mbuf_count = 0; in tcp_lro_flush_all()
1271 hash = m->m_pkthdr.flowid; in tcp_lro_rx_get_bucket()
1274 hash += parser->data.raw[i]; in tcp_lro_rx_get_bucket()
1276 return (&lc->lro_hash[hash % lc->lro_hashsz]); in tcp_lro_rx_get_bucket()
1303 if (((m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) != in tcp_lro_rx_common()
1305 (m->m_pkthdr.csum_data != 0xffff)) { in tcp_lro_rx_common()
1325 switch (pa->data.lro_type) { in tcp_lro_rx_common()
1327 error = tcp_lro_rx_ipv4(lc, m, pa->ip4); in tcp_lro_rx_common()
1336 if ((m->m_flags & (M_TSTMP_LRO | M_TSTMP)) == 0) { in tcp_lro_rx_common()
1337 m->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time); in tcp_lro_rx_common()
1338 m->m_flags |= M_TSTMP_LRO; in tcp_lro_rx_common()
1342 th = pa->tcp; in tcp_lro_rx_common()
1349 tcp_opt_len = (th->th_off << 2); in tcp_lro_rx_common()
1350 tcp_data_len = m->m_pkthdr.len - ((uint8_t *)th - in tcp_lro_rx_common()
1351 (uint8_t *)m->m_data) - tcp_opt_len; in tcp_lro_rx_common()
1352 tcp_opt_len -= sizeof(*th); in tcp_lro_rx_common()
1364 tcp_data_sum = tcp_lro_rx_csum_data(pa, ~th->th_sum); in tcp_lro_rx_common()
1367 m->m_nextpkt = NULL; in tcp_lro_rx_common()
1368 m->m_pkthdr.rcvif = lc->ifp; in tcp_lro_rx_common()
1369 m->m_pkthdr.lro_tcp_d_csum = tcp_data_sum; in tcp_lro_rx_common()
1370 m->m_pkthdr.lro_tcp_d_len = tcp_data_len; in tcp_lro_rx_common()
1371 m->m_pkthdr.lro_tcp_h_off = ((uint8_t *)th - (uint8_t *)m->m_data); in tcp_lro_rx_common()
1372 m->m_pkthdr.lro_nsegs = 1; in tcp_lro_rx_common()
1376 bucket = &lc->lro_hash[0]; in tcp_lro_rx_common()
1384 if (lro_address_compare(&po.data, &le->outer.data) == false || in tcp_lro_rx_common()
1385 lro_address_compare(&pi.data, &le->inner.data) == false) in tcp_lro_rx_common()
1390 SEQ_LT(ntohl(th->th_ack), ntohl(le->ack_seq))) { in tcp_lro_rx_common()
1396 le->m_last_mbuf->m_nextpkt = m; in tcp_lro_rx_common()
1398 le->m_last_mbuf = m; in tcp_lro_rx_common()
1403 if (LIST_EMPTY(&lc->lro_free)) in tcp_lro_rx_common()
1407 le = LIST_FIRST(&lc->lro_free); in tcp_lro_rx_common()
1412 le->inner = pi; in tcp_lro_rx_common()
1413 le->outer = po; in tcp_lro_rx_common()
1416 le->alloc_time = lc->lro_last_queue_time; in tcp_lro_rx_common()
1421 le->m_last_mbuf = m; in tcp_lro_rx_common()
1432 binuptime(&lc->lro_last_queue_time); in tcp_lro_rx()
1433 CURVNET_SET(lc->ifp->if_vnet); in tcp_lro_rx()
1453 if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL || in tcp_lro_queue_mbuf()
1454 lc->lro_mbuf_max == 0)) { in tcp_lro_queue_mbuf()
1461 if (__predict_false((lc->ifp->if_capenable & IFCAP_LRO) == 0)) { in tcp_lro_queue_mbuf()
1463 (*lc->ifp->if_input) (lc->ifp, mb); in tcp_lro_queue_mbuf()
1470 ((mb->m_flags & M_TSTMP) == 0)) { in tcp_lro_queue_mbuf()
1472 binuptime(&lc->lro_last_queue_time); in tcp_lro_queue_mbuf()
1473 mb->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time); in tcp_lro_queue_mbuf()
1474 mb->m_flags |= M_TSTMP_LRO; in tcp_lro_queue_mbuf()
1478 lc->lro_mbuf_data[lc->lro_mbuf_count].seq = lc->lro_mbuf_count; in tcp_lro_queue_mbuf()
1480 lc->lro_mbuf_data[lc->lro_mbuf_count].seq |= in tcp_lro_queue_mbuf()
1482 (((uint64_t)mb->m_pkthdr.flowid) << 24); in tcp_lro_queue_mbuf()
1485 lc->lro_mbuf_data[lc->lro_mbuf_count].mb = mb; in tcp_lro_queue_mbuf()
1488 if (__predict_false(++lc->lro_mbuf_count == lc->lro_mbuf_max)) in tcp_lro_queue_mbuf()