127f190a3SBjoern A. Zeeb /*- 2fe267a55SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3fe267a55SPedro F. Giffuni * 427f190a3SBjoern A. Zeeb * Copyright (c) 2007, Myricom Inc. 527f190a3SBjoern A. Zeeb * Copyright (c) 2008, Intel Corporation. 662b5b6ecSBjoern A. Zeeb * Copyright (c) 2012 The FreeBSD Foundation 79ca874cfSHans Petter Selasky * Copyright (c) 2016-2021 Mellanox Technologies. 827f190a3SBjoern A. Zeeb * All rights reserved. 927f190a3SBjoern A. Zeeb * 1062b5b6ecSBjoern A. Zeeb * Portions of this software were developed by Bjoern Zeeb 1162b5b6ecSBjoern A. Zeeb * under sponsorship from the FreeBSD Foundation. 1262b5b6ecSBjoern A. Zeeb * 1327f190a3SBjoern A. Zeeb * Redistribution and use in source and binary forms, with or without 1427f190a3SBjoern A. Zeeb * modification, are permitted provided that the following conditions 1527f190a3SBjoern A. Zeeb * are met: 1627f190a3SBjoern A. Zeeb * 1. Redistributions of source code must retain the above copyright 1727f190a3SBjoern A. Zeeb * notice, this list of conditions and the following disclaimer. 1827f190a3SBjoern A. Zeeb * 2. Redistributions in binary form must reproduce the above copyright 1927f190a3SBjoern A. Zeeb * notice, this list of conditions and the following disclaimer in the 2027f190a3SBjoern A. Zeeb * documentation and/or other materials provided with the distribution. 2127f190a3SBjoern A. Zeeb * 2227f190a3SBjoern A. Zeeb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 2327f190a3SBjoern A. Zeeb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2427f190a3SBjoern A. Zeeb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2527f190a3SBjoern A. Zeeb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 2627f190a3SBjoern A. Zeeb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2727f190a3SBjoern A. Zeeb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2827f190a3SBjoern A. Zeeb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2927f190a3SBjoern A. Zeeb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3027f190a3SBjoern A. Zeeb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3127f190a3SBjoern A. Zeeb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3227f190a3SBjoern A. Zeeb * SUCH DAMAGE. 3327f190a3SBjoern A. Zeeb */ 346c5087a8SJack F Vogel 3562b5b6ecSBjoern A. Zeeb #include <sys/cdefs.h> 3662b5b6ecSBjoern A. Zeeb __FBSDID("$FreeBSD$"); 3762b5b6ecSBjoern A. Zeeb 3862b5b6ecSBjoern A. Zeeb #include "opt_inet.h" 3962b5b6ecSBjoern A. Zeeb #include "opt_inet6.h" 4062b5b6ecSBjoern A. Zeeb 416c5087a8SJack F Vogel #include <sys/param.h> 426c5087a8SJack F Vogel #include <sys/systm.h> 436c5087a8SJack F Vogel #include <sys/kernel.h> 448ec07310SGleb Smirnoff #include <sys/malloc.h> 458ec07310SGleb Smirnoff #include <sys/mbuf.h> 466c5087a8SJack F Vogel #include <sys/socket.h> 47e57b2d0eSRandall Stewart #include <sys/socketvar.h> 48e57b2d0eSRandall Stewart #include <sys/sockbuf.h> 498452c1b3SSepherosa Ziehau #include <sys/sysctl.h> 506c5087a8SJack F Vogel 516c5087a8SJack F Vogel #include <net/if.h> 5262b5b6ecSBjoern A. Zeeb #include <net/if_var.h> 536c5087a8SJack F Vogel #include <net/ethernet.h> 5469a34e8dSRandall Stewart #include <net/bpf.h> 555fa2656eSBjoern A. Zeeb #include <net/vnet.h> 566c5087a8SJack F Vogel 576c5087a8SJack F Vogel #include <netinet/in_systm.h> 586c5087a8SJack F Vogel #include <netinet/in.h> 5962b5b6ecSBjoern A. Zeeb #include <netinet/ip6.h> 606c5087a8SJack F Vogel #include <netinet/ip.h> 6131bfc56eSBjoern A. Zeeb #include <netinet/ip_var.h> 62e57b2d0eSRandall Stewart #include <netinet/in_pcb.h> 63e57b2d0eSRandall Stewart #include <netinet6/in6_pcb.h> 646c5087a8SJack F Vogel #include <netinet/tcp.h> 65d7fb35d1SSean Bruno #include <netinet/tcp_seq.h> 666c5087a8SJack F Vogel #include <netinet/tcp_lro.h> 678452c1b3SSepherosa Ziehau #include <netinet/tcp_var.h> 6869a34e8dSRandall Stewart #include <netinet/tcpip.h> 69e57b2d0eSRandall Stewart #include <netinet/tcp_hpts.h> 70e57b2d0eSRandall Stewart #include <netinet/tcp_log_buf.h> 719ca874cfSHans Petter Selasky #include <netinet/udp.h> 7231bfc56eSBjoern A. Zeeb #include <netinet6/ip6_var.h> 7331bfc56eSBjoern A. Zeeb 746c5087a8SJack F Vogel #include <machine/in_cksum.h> 756c5087a8SJack F Vogel 76e936121dSHans Petter Selasky static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures"); 776c5087a8SJack F Vogel 789ca874cfSHans Petter Selasky #define TCP_LRO_TS_OPTION \ 799ca874cfSHans Petter Selasky ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | \ 809ca874cfSHans Petter Selasky (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP) 816c5087a8SJack F Vogel 826dd38b87SSepherosa Ziehau static void tcp_lro_rx_done(struct lro_ctrl *lc); 839ca874cfSHans Petter Selasky static int tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m, 849ca874cfSHans Petter Selasky uint32_t csum, bool use_hash); 859ca874cfSHans Petter Selasky 869ca874cfSHans Petter Selasky #ifdef TCPHPTS 879ca874cfSHans Petter Selasky static bool do_bpf_strip_and_compress(struct inpcb *, struct lro_ctrl *, 889ca874cfSHans Petter Selasky struct lro_entry *, struct mbuf **, struct mbuf **, struct mbuf **, bool *, bool); 899ca874cfSHans Petter Selasky 909ca874cfSHans Petter Selasky #endif 916dd38b87SSepherosa Ziehau 928452c1b3SSepherosa Ziehau SYSCTL_NODE(_net_inet_tcp, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 938452c1b3SSepherosa Ziehau "TCP LRO"); 948452c1b3SSepherosa Ziehau 959ca874cfSHans Petter Selasky static long tcplro_stacks_wanting_mbufq; 96e57b2d0eSRandall Stewart counter_u64_t tcp_inp_lro_direct_queue; 97e57b2d0eSRandall Stewart counter_u64_t tcp_inp_lro_wokeup_queue; 98e57b2d0eSRandall Stewart counter_u64_t tcp_inp_lro_compressed; 99e57b2d0eSRandall Stewart counter_u64_t tcp_inp_lro_locks_taken; 10069a34e8dSRandall Stewart counter_u64_t tcp_extra_mbuf; 10169a34e8dSRandall Stewart counter_u64_t tcp_would_have_but; 10269a34e8dSRandall Stewart counter_u64_t tcp_comp_total; 10369a34e8dSRandall Stewart counter_u64_t tcp_uncomp_total; 104e57b2d0eSRandall Stewart 1058452c1b3SSepherosa Ziehau static unsigned tcp_lro_entries = TCP_LRO_ENTRIES; 1068452c1b3SSepherosa Ziehau SYSCTL_UINT(_net_inet_tcp_lro, OID_AUTO, entries, 1078452c1b3SSepherosa Ziehau CTLFLAG_RDTUN | CTLFLAG_MPSAFE, &tcp_lro_entries, 0, 1088452c1b3SSepherosa Ziehau "default number of LRO entries"); 10969a34e8dSRandall Stewart 110e57b2d0eSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, fullqueue, CTLFLAG_RD, 111e57b2d0eSRandall Stewart &tcp_inp_lro_direct_queue, "Number of lro's fully queued to transport"); 112e57b2d0eSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, wokeup, CTLFLAG_RD, 113e57b2d0eSRandall Stewart &tcp_inp_lro_wokeup_queue, "Number of lro's where we woke up transport via hpts"); 114e57b2d0eSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, compressed, CTLFLAG_RD, 115e57b2d0eSRandall Stewart &tcp_inp_lro_compressed, "Number of lro's compressed and sent to transport"); 116e57b2d0eSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, lockcnt, CTLFLAG_RD, 117e57b2d0eSRandall Stewart &tcp_inp_lro_locks_taken, "Number of lro's inp_wlocks taken"); 11869a34e8dSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, extra_mbuf, CTLFLAG_RD, 11969a34e8dSRandall Stewart &tcp_extra_mbuf, "Number of times we had an extra compressed ack dropped into the tp"); 12069a34e8dSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, would_have_but, CTLFLAG_RD, 1219ca874cfSHans Petter Selasky &tcp_would_have_but, "Number of times we would have had an extra compressed, but mget failed"); 12269a34e8dSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, with_m_ackcmp, CTLFLAG_RD, 12369a34e8dSRandall Stewart &tcp_comp_total, "Number of mbufs queued with M_ACKCMP flags set"); 12469a34e8dSRandall Stewart SYSCTL_COUNTER_U64(_net_inet_tcp_lro, OID_AUTO, without_m_ackcmp, CTLFLAG_RD, 12569a34e8dSRandall Stewart &tcp_uncomp_total, "Number of mbufs queued without M_ACKCMP"); 126e57b2d0eSRandall Stewart 127e57b2d0eSRandall Stewart void 128e57b2d0eSRandall Stewart tcp_lro_reg_mbufq(void) 129e57b2d0eSRandall Stewart { 130e57b2d0eSRandall Stewart atomic_fetchadd_long(&tcplro_stacks_wanting_mbufq, 1); 131e57b2d0eSRandall Stewart } 132e57b2d0eSRandall Stewart 133e57b2d0eSRandall Stewart void 134e57b2d0eSRandall Stewart tcp_lro_dereg_mbufq(void) 135e57b2d0eSRandall Stewart { 136e57b2d0eSRandall Stewart atomic_fetchadd_long(&tcplro_stacks_wanting_mbufq, -1); 137e57b2d0eSRandall Stewart } 1388452c1b3SSepherosa Ziehau 13951e3c20dSSepherosa Ziehau static __inline void 14005cde7efSSepherosa Ziehau tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_head *bucket, 14105cde7efSSepherosa Ziehau struct lro_entry *le) 14251e3c20dSSepherosa Ziehau { 14351e3c20dSSepherosa Ziehau 14451e3c20dSSepherosa Ziehau LIST_INSERT_HEAD(&lc->lro_active, le, next); 14505cde7efSSepherosa Ziehau LIST_INSERT_HEAD(bucket, le, hash_next); 14651e3c20dSSepherosa Ziehau } 14751e3c20dSSepherosa Ziehau 14851e3c20dSSepherosa Ziehau static __inline void 14951e3c20dSSepherosa Ziehau tcp_lro_active_remove(struct lro_entry *le) 15051e3c20dSSepherosa Ziehau { 15151e3c20dSSepherosa Ziehau 15205cde7efSSepherosa Ziehau LIST_REMOVE(le, next); /* active list */ 15305cde7efSSepherosa Ziehau LIST_REMOVE(le, hash_next); /* hash bucket */ 15451e3c20dSSepherosa Ziehau } 15551e3c20dSSepherosa Ziehau 1566c5087a8SJack F Vogel int 15762b5b6ecSBjoern A. Zeeb tcp_lro_init(struct lro_ctrl *lc) 1586c5087a8SJack F Vogel { 1598452c1b3SSepherosa Ziehau return (tcp_lro_init_args(lc, NULL, tcp_lro_entries, 0)); 160e936121dSHans Petter Selasky } 161e936121dSHans Petter Selasky 162e936121dSHans Petter Selasky int 163e936121dSHans Petter Selasky tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp, 164e936121dSHans Petter Selasky unsigned lro_entries, unsigned lro_mbufs) 165e936121dSHans Petter Selasky { 16662b5b6ecSBjoern A. Zeeb struct lro_entry *le; 167e936121dSHans Petter Selasky size_t size; 16805cde7efSSepherosa Ziehau unsigned i, elements; 1696c5087a8SJack F Vogel 17062b5b6ecSBjoern A. Zeeb lc->lro_bad_csum = 0; 17162b5b6ecSBjoern A. Zeeb lc->lro_queued = 0; 17262b5b6ecSBjoern A. Zeeb lc->lro_flushed = 0; 173e936121dSHans Petter Selasky lc->lro_mbuf_count = 0; 174e936121dSHans Petter Selasky lc->lro_mbuf_max = lro_mbufs; 175e936121dSHans Petter Selasky lc->lro_cnt = lro_entries; 1767ae3d4bfSSepherosa Ziehau lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX; 1777ae3d4bfSSepherosa Ziehau lc->lro_length_lim = TCP_LRO_LENGTH_MAX; 178e936121dSHans Petter Selasky lc->ifp = ifp; 1791ea44822SSepherosa Ziehau LIST_INIT(&lc->lro_free); 1801ea44822SSepherosa Ziehau LIST_INIT(&lc->lro_active); 1816c5087a8SJack F Vogel 18205cde7efSSepherosa Ziehau /* create hash table to accelerate entry lookup */ 18305cde7efSSepherosa Ziehau if (lro_entries > lro_mbufs) 18405cde7efSSepherosa Ziehau elements = lro_entries; 18505cde7efSSepherosa Ziehau else 18605cde7efSSepherosa Ziehau elements = lro_mbufs; 18705cde7efSSepherosa Ziehau lc->lro_hash = phashinit_flags(elements, M_LRO, &lc->lro_hashsz, 18805cde7efSSepherosa Ziehau HASH_NOWAIT); 18905cde7efSSepherosa Ziehau if (lc->lro_hash == NULL) { 19005cde7efSSepherosa Ziehau memset(lc, 0, sizeof(*lc)); 19105cde7efSSepherosa Ziehau return (ENOMEM); 19205cde7efSSepherosa Ziehau } 19305cde7efSSepherosa Ziehau 194e936121dSHans Petter Selasky /* compute size to allocate */ 195fc271df3SHans Petter Selasky size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) + 196e936121dSHans Petter Selasky (lro_entries * sizeof(*le)); 197fc271df3SHans Petter Selasky lc->lro_mbuf_data = (struct lro_mbuf_sort *) 198e936121dSHans Petter Selasky malloc(size, M_LRO, M_NOWAIT | M_ZERO); 1996c5087a8SJack F Vogel 200e936121dSHans Petter Selasky /* check for out of memory */ 201e936121dSHans Petter Selasky if (lc->lro_mbuf_data == NULL) { 202a3927369SNavdeep Parhar free(lc->lro_hash, M_LRO); 203e936121dSHans Petter Selasky memset(lc, 0, sizeof(*lc)); 204e936121dSHans Petter Selasky return (ENOMEM); 205e936121dSHans Petter Selasky } 206e936121dSHans Petter Selasky /* compute offset for LRO entries */ 207e936121dSHans Petter Selasky le = (struct lro_entry *) 208e936121dSHans Petter Selasky (lc->lro_mbuf_data + lro_mbufs); 209e936121dSHans Petter Selasky 210e936121dSHans Petter Selasky /* setup linked list */ 211e936121dSHans Petter Selasky for (i = 0; i != lro_entries; i++) 2121ea44822SSepherosa Ziehau LIST_INSERT_HEAD(&lc->lro_free, le + i, next); 213e936121dSHans Petter Selasky 214e936121dSHans Petter Selasky return (0); 2156c5087a8SJack F Vogel } 2166c5087a8SJack F Vogel 2179ca874cfSHans Petter Selasky struct vxlan_header { 2189ca874cfSHans Petter Selasky uint32_t vxlh_flags; 2199ca874cfSHans Petter Selasky uint32_t vxlh_vni; 2209ca874cfSHans Petter Selasky }; 221e57b2d0eSRandall Stewart 2229ca874cfSHans Petter Selasky static inline void * 2239ca874cfSHans Petter Selasky tcp_lro_low_level_parser(void *ptr, struct lro_parser *parser, bool update_data, bool is_vxlan) 2249ca874cfSHans Petter Selasky { 2259ca874cfSHans Petter Selasky const struct ether_vlan_header *eh; 2269ca874cfSHans Petter Selasky void *old; 2279ca874cfSHans Petter Selasky uint16_t eth_type; 2289ca874cfSHans Petter Selasky 2299ca874cfSHans Petter Selasky if (update_data) 2309ca874cfSHans Petter Selasky memset(parser, 0, sizeof(*parser)); 2319ca874cfSHans Petter Selasky 2329ca874cfSHans Petter Selasky old = ptr; 2339ca874cfSHans Petter Selasky 2349ca874cfSHans Petter Selasky if (is_vxlan) { 2359ca874cfSHans Petter Selasky const struct vxlan_header *vxh; 2369ca874cfSHans Petter Selasky vxh = ptr; 2379ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + sizeof(*vxh); 2389ca874cfSHans Petter Selasky if (update_data) { 2399ca874cfSHans Petter Selasky parser->data.vxlan_vni = 2409ca874cfSHans Petter Selasky vxh->vxlh_vni & htonl(0xffffff00); 241e57b2d0eSRandall Stewart } 2429ca874cfSHans Petter Selasky } 2439ca874cfSHans Petter Selasky 2449ca874cfSHans Petter Selasky eh = ptr; 2459ca874cfSHans Petter Selasky if (__predict_false(eh->evl_encap_proto == htons(ETHERTYPE_VLAN))) { 2469ca874cfSHans Petter Selasky eth_type = eh->evl_proto; 2479ca874cfSHans Petter Selasky if (update_data) { 2489ca874cfSHans Petter Selasky /* strip priority and keep VLAN ID only */ 2499ca874cfSHans Petter Selasky parser->data.vlan_id = eh->evl_tag & htons(EVL_VLID_MASK); 2509ca874cfSHans Petter Selasky } 2519ca874cfSHans Petter Selasky /* advance to next header */ 2529ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2539ca874cfSHans Petter Selasky } else { 2549ca874cfSHans Petter Selasky eth_type = eh->evl_encap_proto; 2559ca874cfSHans Petter Selasky /* advance to next header */ 2569ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + ETHER_HDR_LEN; 2579ca874cfSHans Petter Selasky } 2589ca874cfSHans Petter Selasky 2599ca874cfSHans Petter Selasky switch (eth_type) { 2609ca874cfSHans Petter Selasky #ifdef INET 2619ca874cfSHans Petter Selasky case htons(ETHERTYPE_IP): 2629ca874cfSHans Petter Selasky parser->ip4 = ptr; 2639ca874cfSHans Petter Selasky /* Ensure there are no IPv4 options. */ 2649ca874cfSHans Petter Selasky if ((parser->ip4->ip_hl << 2) != sizeof (*parser->ip4)) 2659ca874cfSHans Petter Selasky break; 2669ca874cfSHans Petter Selasky /* .. and the packet is not fragmented. */ 2679ca874cfSHans Petter Selasky if (parser->ip4->ip_off & htons(IP_MF|IP_OFFMASK)) 2689ca874cfSHans Petter Selasky break; 2699ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + (parser->ip4->ip_hl << 2); 2709ca874cfSHans Petter Selasky if (update_data) { 2719ca874cfSHans Petter Selasky parser->data.s_addr.v4 = parser->ip4->ip_src; 2729ca874cfSHans Petter Selasky parser->data.d_addr.v4 = parser->ip4->ip_dst; 2739ca874cfSHans Petter Selasky } 2749ca874cfSHans Petter Selasky switch (parser->ip4->ip_p) { 2759ca874cfSHans Petter Selasky case IPPROTO_UDP: 2769ca874cfSHans Petter Selasky parser->udp = ptr; 2779ca874cfSHans Petter Selasky if (update_data) { 2789ca874cfSHans Petter Selasky parser->data.lro_type = LRO_TYPE_IPV4_UDP; 2799ca874cfSHans Petter Selasky parser->data.s_port = parser->udp->uh_sport; 2809ca874cfSHans Petter Selasky parser->data.d_port = parser->udp->uh_dport; 2819ca874cfSHans Petter Selasky } else { 2829ca874cfSHans Petter Selasky MPASS(parser->data.lro_type == LRO_TYPE_IPV4_UDP); 2839ca874cfSHans Petter Selasky } 2849ca874cfSHans Petter Selasky ptr = ((uint8_t *)ptr + sizeof(*parser->udp)); 2859ca874cfSHans Petter Selasky parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; 2869ca874cfSHans Petter Selasky return (ptr); 2879ca874cfSHans Petter Selasky case IPPROTO_TCP: 2889ca874cfSHans Petter Selasky parser->tcp = ptr; 2899ca874cfSHans Petter Selasky if (update_data) { 2909ca874cfSHans Petter Selasky parser->data.lro_type = LRO_TYPE_IPV4_TCP; 2919ca874cfSHans Petter Selasky parser->data.s_port = parser->tcp->th_sport; 2929ca874cfSHans Petter Selasky parser->data.d_port = parser->tcp->th_dport; 2939ca874cfSHans Petter Selasky } else { 2949ca874cfSHans Petter Selasky MPASS(parser->data.lro_type == LRO_TYPE_IPV4_TCP); 2959ca874cfSHans Petter Selasky } 2969ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + (parser->tcp->th_off << 2); 2979ca874cfSHans Petter Selasky parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; 2989ca874cfSHans Petter Selasky return (ptr); 2999ca874cfSHans Petter Selasky default: 3009ca874cfSHans Petter Selasky break; 3019ca874cfSHans Petter Selasky } 3029ca874cfSHans Petter Selasky break; 3039ca874cfSHans Petter Selasky #endif 3049ca874cfSHans Petter Selasky #ifdef INET6 3059ca874cfSHans Petter Selasky case htons(ETHERTYPE_IPV6): 3069ca874cfSHans Petter Selasky parser->ip6 = ptr; 3079ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + sizeof(*parser->ip6); 3089ca874cfSHans Petter Selasky if (update_data) { 3099ca874cfSHans Petter Selasky parser->data.s_addr.v6 = parser->ip6->ip6_src; 3109ca874cfSHans Petter Selasky parser->data.d_addr.v6 = parser->ip6->ip6_dst; 3119ca874cfSHans Petter Selasky } 3129ca874cfSHans Petter Selasky switch (parser->ip6->ip6_nxt) { 3139ca874cfSHans Petter Selasky case IPPROTO_UDP: 3149ca874cfSHans Petter Selasky parser->udp = ptr; 3159ca874cfSHans Petter Selasky if (update_data) { 3169ca874cfSHans Petter Selasky parser->data.lro_type = LRO_TYPE_IPV6_UDP; 3179ca874cfSHans Petter Selasky parser->data.s_port = parser->udp->uh_sport; 3189ca874cfSHans Petter Selasky parser->data.d_port = parser->udp->uh_dport; 3199ca874cfSHans Petter Selasky } else { 3209ca874cfSHans Petter Selasky MPASS(parser->data.lro_type == LRO_TYPE_IPV6_UDP); 3219ca874cfSHans Petter Selasky } 3229ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + sizeof(*parser->udp); 3239ca874cfSHans Petter Selasky parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; 3249ca874cfSHans Petter Selasky return (ptr); 3259ca874cfSHans Petter Selasky case IPPROTO_TCP: 3269ca874cfSHans Petter Selasky parser->tcp = ptr; 3279ca874cfSHans Petter Selasky if (update_data) { 3289ca874cfSHans Petter Selasky parser->data.lro_type = LRO_TYPE_IPV6_TCP; 3299ca874cfSHans Petter Selasky parser->data.s_port = parser->tcp->th_sport; 3309ca874cfSHans Petter Selasky parser->data.d_port = parser->tcp->th_dport; 3319ca874cfSHans Petter Selasky } else { 3329ca874cfSHans Petter Selasky MPASS(parser->data.lro_type == LRO_TYPE_IPV6_TCP); 3339ca874cfSHans Petter Selasky } 3349ca874cfSHans Petter Selasky ptr = (uint8_t *)ptr + (parser->tcp->th_off << 2); 3359ca874cfSHans Petter Selasky parser->total_hdr_len = (uint8_t *)ptr - (uint8_t *)old; 3369ca874cfSHans Petter Selasky return (ptr); 3379ca874cfSHans Petter Selasky default: 3389ca874cfSHans Petter Selasky break; 3399ca874cfSHans Petter Selasky } 3409ca874cfSHans Petter Selasky break; 3419ca874cfSHans Petter Selasky #endif 3429ca874cfSHans Petter Selasky default: 3439ca874cfSHans Petter Selasky break; 3449ca874cfSHans Petter Selasky } 3459ca874cfSHans Petter Selasky /* Invalid packet - cannot parse */ 3469ca874cfSHans Petter Selasky return (NULL); 3479ca874cfSHans Petter Selasky } 3489ca874cfSHans Petter Selasky 3499ca874cfSHans Petter Selasky static const int vxlan_csum = CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID | 3509ca874cfSHans Petter Selasky CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID; 3519ca874cfSHans Petter Selasky 3529ca874cfSHans Petter Selasky static inline struct lro_parser * 3539ca874cfSHans Petter Selasky tcp_lro_parser(struct mbuf *m, struct lro_parser *po, struct lro_parser *pi, bool update_data) 3549ca874cfSHans Petter Selasky { 3559ca874cfSHans Petter Selasky void *data_ptr; 3569ca874cfSHans Petter Selasky 3579ca874cfSHans Petter Selasky /* Try to parse outer headers first. */ 3589ca874cfSHans Petter Selasky data_ptr = tcp_lro_low_level_parser(m->m_data, po, update_data, false); 3599ca874cfSHans Petter Selasky if (data_ptr == NULL || po->total_hdr_len > m->m_len) 3609ca874cfSHans Petter Selasky return (NULL); 3619ca874cfSHans Petter Selasky 3629ca874cfSHans Petter Selasky if (update_data) { 3639ca874cfSHans Petter Selasky /* Store VLAN ID, if any. */ 3649ca874cfSHans Petter Selasky if (__predict_false(m->m_flags & M_VLANTAG)) { 3659ca874cfSHans Petter Selasky po->data.vlan_id = 3669ca874cfSHans Petter Selasky htons(m->m_pkthdr.ether_vtag) & htons(EVL_VLID_MASK); 3679ca874cfSHans Petter Selasky } 3689ca874cfSHans Petter Selasky } 3699ca874cfSHans Petter Selasky 3709ca874cfSHans Petter Selasky switch (po->data.lro_type) { 3719ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_UDP: 3729ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_UDP: 3739ca874cfSHans Petter Selasky /* Check for VXLAN headers. */ 3749ca874cfSHans Petter Selasky if ((m->m_pkthdr.csum_flags & vxlan_csum) != vxlan_csum) 3759ca874cfSHans Petter Selasky break; 3769ca874cfSHans Petter Selasky 3779ca874cfSHans Petter Selasky /* Try to parse inner headers. */ 3789ca874cfSHans Petter Selasky data_ptr = tcp_lro_low_level_parser(data_ptr, pi, update_data, true); 3799ca874cfSHans Petter Selasky if (data_ptr == NULL || pi->total_hdr_len > m->m_len) 3809ca874cfSHans Petter Selasky break; 3819ca874cfSHans Petter Selasky 3829ca874cfSHans Petter Selasky /* Verify supported header types. */ 3839ca874cfSHans Petter Selasky switch (pi->data.lro_type) { 3849ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 3859ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 3869ca874cfSHans Petter Selasky return (pi); 3879ca874cfSHans Petter Selasky default: 3889ca874cfSHans Petter Selasky break; 3899ca874cfSHans Petter Selasky } 3909ca874cfSHans Petter Selasky break; 3919ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 3929ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 3939ca874cfSHans Petter Selasky if (update_data) 3949ca874cfSHans Petter Selasky memset(pi, 0, sizeof(*pi)); 3959ca874cfSHans Petter Selasky return (po); 3969ca874cfSHans Petter Selasky default: 3979ca874cfSHans Petter Selasky break; 3989ca874cfSHans Petter Selasky } 3999ca874cfSHans Petter Selasky return (NULL); 4009ca874cfSHans Petter Selasky } 4019ca874cfSHans Petter Selasky 4029ca874cfSHans Petter Selasky static inline int 4039ca874cfSHans Petter Selasky tcp_lro_trim_mbuf_chain(struct mbuf *m, const struct lro_parser *po) 4049ca874cfSHans Petter Selasky { 4059ca874cfSHans Petter Selasky int len; 4069ca874cfSHans Petter Selasky 4079ca874cfSHans Petter Selasky switch (po->data.lro_type) { 4089ca874cfSHans Petter Selasky #ifdef INET 4099ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 4109ca874cfSHans Petter Selasky len = ((uint8_t *)po->ip4 - (uint8_t *)m->m_data) + 4119ca874cfSHans Petter Selasky ntohs(po->ip4->ip_len); 4129ca874cfSHans Petter Selasky break; 4139ca874cfSHans Petter Selasky #endif 4149ca874cfSHans Petter Selasky #ifdef INET6 4159ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 4169ca874cfSHans Petter Selasky len = ((uint8_t *)po->ip6 - (uint8_t *)m->m_data) + 4179ca874cfSHans Petter Selasky ntohs(po->ip6->ip6_plen) + sizeof(*po->ip6); 4189ca874cfSHans Petter Selasky break; 4199ca874cfSHans Petter Selasky #endif 4209ca874cfSHans Petter Selasky default: 4219ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 4229ca874cfSHans Petter Selasky } 4239ca874cfSHans Petter Selasky 4249ca874cfSHans Petter Selasky /* 4259ca874cfSHans Petter Selasky * If the frame is padded beyond the end of the IP packet, 4269ca874cfSHans Petter Selasky * then trim the extra bytes off: 4279ca874cfSHans Petter Selasky */ 4289ca874cfSHans Petter Selasky if (__predict_true(m->m_pkthdr.len == len)) { 4299ca874cfSHans Petter Selasky return (0); 4309ca874cfSHans Petter Selasky } else if (m->m_pkthdr.len > len) { 4319ca874cfSHans Petter Selasky m_adj(m, len - m->m_pkthdr.len); 4329ca874cfSHans Petter Selasky return (0); 4339ca874cfSHans Petter Selasky } 4349ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 4359ca874cfSHans Petter Selasky } 4369ca874cfSHans Petter Selasky 4379ca874cfSHans Petter Selasky static struct tcphdr * 4389ca874cfSHans Petter Selasky tcp_lro_get_th(struct mbuf *m) 4399ca874cfSHans Petter Selasky { 4409ca874cfSHans Petter Selasky return ((struct tcphdr *)((uint8_t *)m->m_data + m->m_pkthdr.lro_tcp_h_off)); 441e57b2d0eSRandall Stewart } 442e57b2d0eSRandall Stewart 44369a34e8dSRandall Stewart static void 44469a34e8dSRandall Stewart lro_free_mbuf_chain(struct mbuf *m) 44569a34e8dSRandall Stewart { 44669a34e8dSRandall Stewart struct mbuf *save; 44769a34e8dSRandall Stewart 44869a34e8dSRandall Stewart while (m) { 44969a34e8dSRandall Stewart save = m->m_nextpkt; 45069a34e8dSRandall Stewart m->m_nextpkt = NULL; 45169a34e8dSRandall Stewart m_freem(m); 45269a34e8dSRandall Stewart m = save; 45369a34e8dSRandall Stewart } 45469a34e8dSRandall Stewart } 45569a34e8dSRandall Stewart 4566c5087a8SJack F Vogel void 45762b5b6ecSBjoern A. Zeeb tcp_lro_free(struct lro_ctrl *lc) 4586c5087a8SJack F Vogel { 45962b5b6ecSBjoern A. Zeeb struct lro_entry *le; 460e936121dSHans Petter Selasky unsigned x; 4616c5087a8SJack F Vogel 462e936121dSHans Petter Selasky /* reset LRO free list */ 4631ea44822SSepherosa Ziehau LIST_INIT(&lc->lro_free); 464e936121dSHans Petter Selasky 465e936121dSHans Petter Selasky /* free active mbufs, if any */ 4661ea44822SSepherosa Ziehau while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { 46751e3c20dSSepherosa Ziehau tcp_lro_active_remove(le); 46869a34e8dSRandall Stewart lro_free_mbuf_chain(le->m_head); 4696c5087a8SJack F Vogel } 470e936121dSHans Petter Selasky 47105cde7efSSepherosa Ziehau /* free hash table */ 47205cde7efSSepherosa Ziehau free(lc->lro_hash, M_LRO); 47305cde7efSSepherosa Ziehau lc->lro_hash = NULL; 47405cde7efSSepherosa Ziehau lc->lro_hashsz = 0; 47505cde7efSSepherosa Ziehau 476e936121dSHans Petter Selasky /* free mbuf array, if any */ 477e936121dSHans Petter Selasky for (x = 0; x != lc->lro_mbuf_count; x++) 478fc271df3SHans Petter Selasky m_freem(lc->lro_mbuf_data[x].mb); 479e936121dSHans Petter Selasky lc->lro_mbuf_count = 0; 480e936121dSHans Petter Selasky 481e936121dSHans Petter Selasky /* free allocated memory, if any */ 482e936121dSHans Petter Selasky free(lc->lro_mbuf_data, M_LRO); 483e936121dSHans Petter Selasky lc->lro_mbuf_data = NULL; 4846c5087a8SJack F Vogel } 4856c5087a8SJack F Vogel 48662b5b6ecSBjoern A. Zeeb static uint16_t 4879ca874cfSHans Petter Selasky tcp_lro_rx_csum_tcphdr(const struct tcphdr *th) 48862b5b6ecSBjoern A. Zeeb { 4899ca874cfSHans Petter Selasky const uint16_t *ptr; 4909ca874cfSHans Petter Selasky uint32_t csum; 4919ca874cfSHans Petter Selasky uint16_t len; 49262b5b6ecSBjoern A. Zeeb 4939ca874cfSHans Petter Selasky csum = -th->th_sum; /* exclude checksum field */ 4949ca874cfSHans Petter Selasky len = th->th_off; 4959ca874cfSHans Petter Selasky ptr = (const uint16_t *)th; 4969ca874cfSHans Petter Selasky while (len--) { 4979ca874cfSHans Petter Selasky csum += *ptr; 4989ca874cfSHans Petter Selasky ptr++; 4999ca874cfSHans Petter Selasky csum += *ptr; 5009ca874cfSHans Petter Selasky ptr++; 50162b5b6ecSBjoern A. Zeeb } 5029ca874cfSHans Petter Selasky while (csum > 0xffff) 5039ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 50462b5b6ecSBjoern A. Zeeb 5059ca874cfSHans Petter Selasky return (csum); 50662b5b6ecSBjoern A. Zeeb } 50762b5b6ecSBjoern A. Zeeb 50862b5b6ecSBjoern A. Zeeb static uint16_t 5099ca874cfSHans Petter Selasky tcp_lro_rx_csum_data(const struct lro_parser *pa, uint16_t tcp_csum) 51062b5b6ecSBjoern A. Zeeb { 51162b5b6ecSBjoern A. Zeeb uint32_t c; 51262b5b6ecSBjoern A. Zeeb uint16_t cs; 51362b5b6ecSBjoern A. Zeeb 5149ca874cfSHans Petter Selasky c = tcp_csum; 51562b5b6ecSBjoern A. Zeeb 5169ca874cfSHans Petter Selasky switch (pa->data.lro_type) { 51762b5b6ecSBjoern A. Zeeb #ifdef INET6 5189ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 5199ca874cfSHans Petter Selasky /* Compute full pseudo IPv6 header checksum. */ 5209ca874cfSHans Petter Selasky cs = in6_cksum_pseudo(pa->ip6, ntohs(pa->ip6->ip6_plen), pa->ip6->ip6_nxt, 0); 52162b5b6ecSBjoern A. Zeeb break; 52262b5b6ecSBjoern A. Zeeb #endif 52362b5b6ecSBjoern A. Zeeb #ifdef INET 5249ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 5259ca874cfSHans Petter Selasky /* Compute full pseudo IPv4 header checsum. */ 5269ca874cfSHans Petter Selasky cs = in_addword(ntohs(pa->ip4->ip_len) - sizeof(*pa->ip4), IPPROTO_TCP); 5279ca874cfSHans Petter Selasky cs = in_pseudo(pa->ip4->ip_src.s_addr, pa->ip4->ip_dst.s_addr, htons(cs)); 52862b5b6ecSBjoern A. Zeeb break; 52962b5b6ecSBjoern A. Zeeb #endif 53062b5b6ecSBjoern A. Zeeb default: 53162b5b6ecSBjoern A. Zeeb cs = 0; /* Keep compiler happy. */ 5329ca874cfSHans Petter Selasky break; 53362b5b6ecSBjoern A. Zeeb } 53462b5b6ecSBjoern A. Zeeb 5359ca874cfSHans Petter Selasky /* Complement checksum. */ 53662b5b6ecSBjoern A. Zeeb cs = ~cs; 53762b5b6ecSBjoern A. Zeeb c += cs; 53862b5b6ecSBjoern A. Zeeb 5399ca874cfSHans Petter Selasky /* Remove TCP header checksum. */ 5409ca874cfSHans Petter Selasky cs = ~tcp_lro_rx_csum_tcphdr(pa->tcp); 54162b5b6ecSBjoern A. Zeeb c += cs; 5429ca874cfSHans Petter Selasky 5439ca874cfSHans Petter Selasky /* Compute checksum remainder. */ 54462b5b6ecSBjoern A. Zeeb while (c > 0xffff) 54562b5b6ecSBjoern A. Zeeb c = (c >> 16) + (c & 0xffff); 54662b5b6ecSBjoern A. Zeeb 5479ca874cfSHans Petter Selasky return (c); 54862b5b6ecSBjoern A. Zeeb } 54962b5b6ecSBjoern A. Zeeb 5506dd38b87SSepherosa Ziehau static void 5516dd38b87SSepherosa Ziehau tcp_lro_rx_done(struct lro_ctrl *lc) 5526dd38b87SSepherosa Ziehau { 5536dd38b87SSepherosa Ziehau struct lro_entry *le; 5546dd38b87SSepherosa Ziehau 5551ea44822SSepherosa Ziehau while ((le = LIST_FIRST(&lc->lro_active)) != NULL) { 55651e3c20dSSepherosa Ziehau tcp_lro_active_remove(le); 5576dd38b87SSepherosa Ziehau tcp_lro_flush(lc, le); 5586dd38b87SSepherosa Ziehau } 5596dd38b87SSepherosa Ziehau } 5606dd38b87SSepherosa Ziehau 5616c5087a8SJack F Vogel void 5627127e6acSNavdeep Parhar tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout) 5637127e6acSNavdeep Parhar { 5647127e6acSNavdeep Parhar struct lro_entry *le, *le_tmp; 565*b45daaeaSRandall Stewart uint64_t now, tov; 566*b45daaeaSRandall Stewart struct bintime bt; 5677127e6acSNavdeep Parhar 5681ea44822SSepherosa Ziehau if (LIST_EMPTY(&lc->lro_active)) 5697127e6acSNavdeep Parhar return; 5707127e6acSNavdeep Parhar 571*b45daaeaSRandall Stewart /* get timeout time and current time in ns */ 572*b45daaeaSRandall Stewart binuptime(&bt); 573*b45daaeaSRandall Stewart now = bintime2ns(&bt); 574*b45daaeaSRandall Stewart tov = ((timeout->tv_sec * 1000000000) + (timeout->tv_usec * 1000)); 5751ea44822SSepherosa Ziehau LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) { 576*b45daaeaSRandall Stewart if (now >= (bintime2ns(&le->alloc_time) + tov)) { 57751e3c20dSSepherosa Ziehau tcp_lro_active_remove(le); 5787127e6acSNavdeep Parhar tcp_lro_flush(lc, le); 5797127e6acSNavdeep Parhar } 5807127e6acSNavdeep Parhar } 5817127e6acSNavdeep Parhar } 5827127e6acSNavdeep Parhar 583e57b2d0eSRandall Stewart #ifdef INET 584e57b2d0eSRandall Stewart static int 5859ca874cfSHans Petter Selasky tcp_lro_rx_ipv4(struct lro_ctrl *lc, struct mbuf *m, struct ip *ip4) 586e57b2d0eSRandall Stewart { 587e57b2d0eSRandall Stewart uint16_t csum; 588e57b2d0eSRandall Stewart 589e57b2d0eSRandall Stewart /* Legacy IP has a header checksum that needs to be correct. */ 5909ca874cfSHans Petter Selasky if (m->m_pkthdr.csum_flags & CSUM_IP_CHECKED) { 5919ca874cfSHans Petter Selasky if (__predict_false((m->m_pkthdr.csum_flags & CSUM_IP_VALID) == 0)) { 592e57b2d0eSRandall Stewart lc->lro_bad_csum++; 593e57b2d0eSRandall Stewart return (TCP_LRO_CANNOT); 594e57b2d0eSRandall Stewart } 595e57b2d0eSRandall Stewart } else { 596e57b2d0eSRandall Stewart csum = in_cksum_hdr(ip4); 5979ca874cfSHans Petter Selasky if (__predict_false(csum != 0)) { 598e57b2d0eSRandall Stewart lc->lro_bad_csum++; 599e57b2d0eSRandall Stewart return (TCP_LRO_CANNOT); 600e57b2d0eSRandall Stewart } 601e57b2d0eSRandall Stewart } 602e57b2d0eSRandall Stewart return (0); 603e57b2d0eSRandall Stewart } 604e57b2d0eSRandall Stewart #endif 605e57b2d0eSRandall Stewart 6060a4f8510SRandall Stewart #ifdef TCPHPTS 607e57b2d0eSRandall Stewart static void 6089ca874cfSHans Petter Selasky tcp_lro_log(struct tcpcb *tp, const struct lro_ctrl *lc, 6099ca874cfSHans Petter Selasky const struct lro_entry *le, const struct mbuf *m, 6109ca874cfSHans Petter Selasky int frm, int32_t tcp_data_len, uint32_t th_seq, 6119ca874cfSHans Petter Selasky uint32_t th_ack, uint16_t th_win) 612e57b2d0eSRandall Stewart { 613e57b2d0eSRandall Stewart if (tp->t_logstate != TCP_LOG_STATE_OFF) { 614e57b2d0eSRandall Stewart union tcp_log_stackspecific log; 615*b45daaeaSRandall Stewart struct timeval tv, btv; 616e57b2d0eSRandall Stewart uint32_t cts; 617e57b2d0eSRandall Stewart 618e57b2d0eSRandall Stewart cts = tcp_get_usecs(&tv); 619e57b2d0eSRandall Stewart memset(&log, 0, sizeof(union tcp_log_stackspecific)); 620e57b2d0eSRandall Stewart log.u_bbr.flex8 = frm; 621e57b2d0eSRandall Stewart log.u_bbr.flex1 = tcp_data_len; 622e57b2d0eSRandall Stewart if (m) 623e57b2d0eSRandall Stewart log.u_bbr.flex2 = m->m_pkthdr.len; 624e57b2d0eSRandall Stewart else 625e57b2d0eSRandall Stewart log.u_bbr.flex2 = 0; 6269ca874cfSHans Petter Selasky log.u_bbr.flex3 = le->m_head->m_pkthdr.lro_nsegs; 6279ca874cfSHans Petter Selasky log.u_bbr.flex4 = le->m_head->m_pkthdr.lro_tcp_d_len; 62869a34e8dSRandall Stewart if (le->m_head) { 629e57b2d0eSRandall Stewart log.u_bbr.flex5 = le->m_head->m_pkthdr.len; 630e57b2d0eSRandall Stewart log.u_bbr.delRate = le->m_head->m_flags; 631e57b2d0eSRandall Stewart log.u_bbr.rttProp = le->m_head->m_pkthdr.rcv_tstmp; 63269a34e8dSRandall Stewart } 633e57b2d0eSRandall Stewart log.u_bbr.inflight = th_seq; 634e57b2d0eSRandall Stewart log.u_bbr.timeStamp = cts; 635e57b2d0eSRandall Stewart log.u_bbr.epoch = le->next_seq; 636e57b2d0eSRandall Stewart log.u_bbr.delivered = th_ack; 637e57b2d0eSRandall Stewart log.u_bbr.lt_epoch = le->ack_seq; 638e57b2d0eSRandall Stewart log.u_bbr.pacing_gain = th_win; 639e57b2d0eSRandall Stewart log.u_bbr.cwnd_gain = le->window; 640b23b156eSWarner Losh log.u_bbr.cur_del_rate = (uintptr_t)m; 641b23b156eSWarner Losh log.u_bbr.bw_inuse = (uintptr_t)le->m_head; 642*b45daaeaSRandall Stewart bintime2timeval(&lc->lro_last_queue_time, &btv); 643*b45daaeaSRandall Stewart log.u_bbr.flex6 = tcp_tv_to_usectick(&btv); 6449ca874cfSHans Petter Selasky log.u_bbr.flex7 = le->compressed; 6459ca874cfSHans Petter Selasky log.u_bbr.pacing_gain = le->uncompressed; 64669a34e8dSRandall Stewart if (in_epoch(net_epoch_preempt)) 64769a34e8dSRandall Stewart log.u_bbr.inhpts = 1; 64869a34e8dSRandall Stewart else 64969a34e8dSRandall Stewart log.u_bbr.inhpts = 0; 650e57b2d0eSRandall Stewart TCP_LOG_EVENTP(tp, NULL, 651e57b2d0eSRandall Stewart &tp->t_inpcb->inp_socket->so_rcv, 652e57b2d0eSRandall Stewart &tp->t_inpcb->inp_socket->so_snd, 653e57b2d0eSRandall Stewart TCP_LOG_LRO, 0, 654e57b2d0eSRandall Stewart 0, &log, false, &tv); 655e57b2d0eSRandall Stewart } 656e57b2d0eSRandall Stewart } 6570a4f8510SRandall Stewart #endif 658e57b2d0eSRandall Stewart 6599ca874cfSHans Petter Selasky static inline void 6609ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(uint16_t *ptr, uint16_t value, uint16_t *psum) 661e57b2d0eSRandall Stewart { 6629ca874cfSHans Petter Selasky uint32_t csum; 6636c5087a8SJack F Vogel 6649ca874cfSHans Petter Selasky csum = 0xffff - *ptr + value; 6659ca874cfSHans Petter Selasky while (csum > 0xffff) 6669ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 6679ca874cfSHans Petter Selasky *ptr = value; 6689ca874cfSHans Petter Selasky *psum = csum; 66962b5b6ecSBjoern A. Zeeb } 67062b5b6ecSBjoern A. Zeeb 6719ca874cfSHans Petter Selasky static uint16_t 6729ca874cfSHans Petter Selasky tcp_lro_update_checksum(const struct lro_parser *pa, const struct lro_entry *le, 6739ca874cfSHans Petter Selasky uint16_t payload_len, uint16_t delta_sum) 6749ca874cfSHans Petter Selasky { 6759ca874cfSHans Petter Selasky uint32_t csum; 6769ca874cfSHans Petter Selasky uint16_t tlen; 6779ca874cfSHans Petter Selasky uint16_t temp[5] = {}; 6789ca874cfSHans Petter Selasky 6799ca874cfSHans Petter Selasky switch (pa->data.lro_type) { 6809ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 6819ca874cfSHans Petter Selasky /* Compute new IPv4 length. */ 6829ca874cfSHans Petter Selasky tlen = (pa->ip4->ip_hl << 2) + (pa->tcp->th_off << 2) + payload_len; 6839ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]); 6849ca874cfSHans Petter Selasky 6859ca874cfSHans Petter Selasky /* Subtract delta from current IPv4 checksum. */ 6869ca874cfSHans Petter Selasky csum = pa->ip4->ip_sum + 0xffff - temp[0]; 6879ca874cfSHans Petter Selasky while (csum > 0xffff) 6889ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 6899ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->ip4->ip_sum, csum, &temp[1]); 6909ca874cfSHans Petter Selasky goto update_tcp_header; 6919ca874cfSHans Petter Selasky 6929ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 6939ca874cfSHans Petter Selasky /* Compute new IPv6 length. */ 6949ca874cfSHans Petter Selasky tlen = (pa->tcp->th_off << 2) + payload_len; 6959ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]); 6969ca874cfSHans Petter Selasky goto update_tcp_header; 6979ca874cfSHans Petter Selasky 6989ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_UDP: 6999ca874cfSHans Petter Selasky /* Compute new IPv4 length. */ 7009ca874cfSHans Petter Selasky tlen = (pa->ip4->ip_hl << 2) + sizeof(*pa->udp) + payload_len; 7019ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->ip4->ip_len, htons(tlen), &temp[0]); 7029ca874cfSHans Petter Selasky 7039ca874cfSHans Petter Selasky /* Subtract delta from current IPv4 checksum. */ 7049ca874cfSHans Petter Selasky csum = pa->ip4->ip_sum + 0xffff - temp[0]; 7059ca874cfSHans Petter Selasky while (csum > 0xffff) 7069ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 7079ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->ip4->ip_sum, csum, &temp[1]); 7089ca874cfSHans Petter Selasky goto update_udp_header; 7099ca874cfSHans Petter Selasky 7109ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_UDP: 7119ca874cfSHans Petter Selasky /* Compute new IPv6 length. */ 7129ca874cfSHans Petter Selasky tlen = sizeof(*pa->udp) + payload_len; 7139ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->ip6->ip6_plen, htons(tlen), &temp[0]); 7149ca874cfSHans Petter Selasky goto update_udp_header; 7159ca874cfSHans Petter Selasky 71662b5b6ecSBjoern A. Zeeb default: 7179ca874cfSHans Petter Selasky return (0); 71862b5b6ecSBjoern A. Zeeb } 7199ca874cfSHans Petter Selasky 7209ca874cfSHans Petter Selasky update_tcp_header: 7219ca874cfSHans Petter Selasky /* Compute current TCP header checksum. */ 7229ca874cfSHans Petter Selasky temp[2] = tcp_lro_rx_csum_tcphdr(pa->tcp); 72362b5b6ecSBjoern A. Zeeb 72462b5b6ecSBjoern A. Zeeb /* Incorporate the latest ACK into the TCP header. */ 7259ca874cfSHans Petter Selasky pa->tcp->th_ack = le->ack_seq; 7269ca874cfSHans Petter Selasky pa->tcp->th_win = le->window; 7279ca874cfSHans Petter Selasky 72862b5b6ecSBjoern A. Zeeb /* Incorporate latest timestamp into the TCP header. */ 72962b5b6ecSBjoern A. Zeeb if (le->timestamp != 0) { 7306c5087a8SJack F Vogel uint32_t *ts_ptr; 7316c5087a8SJack F Vogel 7329ca874cfSHans Petter Selasky ts_ptr = (uint32_t *)(pa->tcp + 1); 73362b5b6ecSBjoern A. Zeeb ts_ptr[1] = htonl(le->tsval); 73462b5b6ecSBjoern A. Zeeb ts_ptr[2] = le->tsecr; 73562b5b6ecSBjoern A. Zeeb } 7369ca874cfSHans Petter Selasky 7379ca874cfSHans Petter Selasky /* Compute new TCP header checksum. */ 7389ca874cfSHans Petter Selasky temp[3] = tcp_lro_rx_csum_tcphdr(pa->tcp); 7399ca874cfSHans Petter Selasky 7409ca874cfSHans Petter Selasky /* Compute new TCP checksum. */ 7419ca874cfSHans Petter Selasky csum = pa->tcp->th_sum + 0xffff - delta_sum + 7429ca874cfSHans Petter Selasky 0xffff - temp[0] + 0xffff - temp[3] + temp[2]; 7439ca874cfSHans Petter Selasky while (csum > 0xffff) 7449ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 7459ca874cfSHans Petter Selasky 7469ca874cfSHans Petter Selasky /* Assign new TCP checksum. */ 7479ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->tcp->th_sum, csum, &temp[4]); 7489ca874cfSHans Petter Selasky 7499ca874cfSHans Petter Selasky /* Compute all modififications affecting next checksum. */ 7509ca874cfSHans Petter Selasky csum = temp[0] + temp[1] + 0xffff - temp[2] + 7519ca874cfSHans Petter Selasky temp[3] + temp[4] + delta_sum; 7529ca874cfSHans Petter Selasky while (csum > 0xffff) 7539ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 7549ca874cfSHans Petter Selasky 7559ca874cfSHans Petter Selasky /* Return delta checksum to next stage, if any. */ 7569ca874cfSHans Petter Selasky return (csum); 7579ca874cfSHans Petter Selasky 7589ca874cfSHans Petter Selasky update_udp_header: 7599ca874cfSHans Petter Selasky tlen = sizeof(*pa->udp) + payload_len; 7609ca874cfSHans Petter Selasky /* Assign new UDP length and compute checksum delta. */ 7619ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->udp->uh_ulen, htons(tlen), &temp[2]); 7629ca874cfSHans Petter Selasky 7639ca874cfSHans Petter Selasky /* Check if there is a UDP checksum. */ 7649ca874cfSHans Petter Selasky if (__predict_false(pa->udp->uh_sum != 0)) { 7659ca874cfSHans Petter Selasky /* Compute new UDP checksum. */ 7669ca874cfSHans Petter Selasky csum = pa->udp->uh_sum + 0xffff - delta_sum + 7679ca874cfSHans Petter Selasky 0xffff - temp[0] + 0xffff - temp[2]; 7689ca874cfSHans Petter Selasky while (csum > 0xffff) 7699ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 7709ca874cfSHans Petter Selasky /* Assign new UDP checksum. */ 7719ca874cfSHans Petter Selasky tcp_lro_assign_and_checksum_16(&pa->udp->uh_sum, csum, &temp[3]); 772e57b2d0eSRandall Stewart } 7739ca874cfSHans Petter Selasky 7749ca874cfSHans Petter Selasky /* Compute all modififications affecting next checksum. */ 7759ca874cfSHans Petter Selasky csum = temp[0] + temp[1] + temp[2] + temp[3] + delta_sum; 7769ca874cfSHans Petter Selasky while (csum > 0xffff) 7779ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 7789ca874cfSHans Petter Selasky 7799ca874cfSHans Petter Selasky /* Return delta checksum to next stage, if any. */ 7809ca874cfSHans Petter Selasky return (csum); 7819ca874cfSHans Petter Selasky } 7829ca874cfSHans Petter Selasky 7839ca874cfSHans Petter Selasky static void 7849ca874cfSHans Petter Selasky tcp_flush_out_entry(struct lro_ctrl *lc, struct lro_entry *le) 7859ca874cfSHans Petter Selasky { 7869ca874cfSHans Petter Selasky /* Check if we need to recompute any checksums. */ 7879ca874cfSHans Petter Selasky if (le->m_head->m_pkthdr.lro_nsegs > 1) { 7889ca874cfSHans Petter Selasky uint16_t csum; 7899ca874cfSHans Petter Selasky 7909ca874cfSHans Petter Selasky switch (le->inner.data.lro_type) { 7919ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 7929ca874cfSHans Petter Selasky csum = tcp_lro_update_checksum(&le->inner, le, 7939ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_len, 7949ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_csum); 7959ca874cfSHans Petter Selasky csum = tcp_lro_update_checksum(&le->outer, NULL, 7969ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_len + 7979ca874cfSHans Petter Selasky le->inner.total_hdr_len, csum); 7989ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | 7999ca874cfSHans Petter Selasky CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID; 8009ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_data = 0xffff; 8019ca874cfSHans Petter Selasky break; 8029ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 8039ca874cfSHans Petter Selasky csum = tcp_lro_update_checksum(&le->inner, le, 8049ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_len, 8059ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_csum); 8069ca874cfSHans Petter Selasky csum = tcp_lro_update_checksum(&le->outer, NULL, 8079ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_len + 8089ca874cfSHans Petter Selasky le->inner.total_hdr_len, csum); 8099ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | 8109ca874cfSHans Petter Selasky CSUM_PSEUDO_HDR; 8119ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_data = 0xffff; 8129ca874cfSHans Petter Selasky break; 8139ca874cfSHans Petter Selasky case LRO_TYPE_NONE: 8149ca874cfSHans Petter Selasky switch (le->outer.data.lro_type) { 8159ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 8169ca874cfSHans Petter Selasky csum = tcp_lro_update_checksum(&le->outer, le, 8179ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_len, 8189ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_csum); 8199ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | 8209ca874cfSHans Petter Selasky CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID; 8219ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_data = 0xffff; 8229ca874cfSHans Petter Selasky break; 8239ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 8249ca874cfSHans Petter Selasky csum = tcp_lro_update_checksum(&le->outer, le, 8259ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_len, 8269ca874cfSHans Petter Selasky le->m_head->m_pkthdr.lro_tcp_d_csum); 8279ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID | 8289ca874cfSHans Petter Selasky CSUM_PSEUDO_HDR; 8299ca874cfSHans Petter Selasky le->m_head->m_pkthdr.csum_data = 0xffff; 8309ca874cfSHans Petter Selasky break; 8319ca874cfSHans Petter Selasky default: 8329ca874cfSHans Petter Selasky break; 8339ca874cfSHans Petter Selasky } 8349ca874cfSHans Petter Selasky break; 8359ca874cfSHans Petter Selasky default: 8369ca874cfSHans Petter Selasky break; 8379ca874cfSHans Petter Selasky } 8389ca874cfSHans Petter Selasky } 8399ca874cfSHans Petter Selasky 840e57b2d0eSRandall Stewart /* 841e57b2d0eSRandall Stewart * Break any chain, this is not set to NULL on the singleton 842e57b2d0eSRandall Stewart * case m_nextpkt points to m_head. Other case set them 843e57b2d0eSRandall Stewart * m_nextpkt to NULL in push_and_replace. 844e57b2d0eSRandall Stewart */ 845e57b2d0eSRandall Stewart le->m_head->m_nextpkt = NULL; 8469ca874cfSHans Petter Selasky lc->lro_queued += le->m_head->m_pkthdr.lro_nsegs; 847e57b2d0eSRandall Stewart (*lc->ifp->if_input)(lc->ifp, le->m_head); 84862b5b6ecSBjoern A. Zeeb } 8496c5087a8SJack F Vogel 850e57b2d0eSRandall Stewart static void 8519ca874cfSHans Petter Selasky tcp_set_entry_to_mbuf(struct lro_ctrl *lc, struct lro_entry *le, 8529ca874cfSHans Petter Selasky struct mbuf *m, struct tcphdr *th) 853e57b2d0eSRandall Stewart { 854e57b2d0eSRandall Stewart uint32_t *ts_ptr; 855e57b2d0eSRandall Stewart uint16_t tcp_data_len; 8569ca874cfSHans Petter Selasky uint16_t tcp_opt_len; 857e57b2d0eSRandall Stewart 858e57b2d0eSRandall Stewart ts_ptr = (uint32_t *)(th + 1); 8599ca874cfSHans Petter Selasky tcp_opt_len = (th->th_off << 2); 8609ca874cfSHans Petter Selasky tcp_opt_len -= sizeof(*th); 8619ca874cfSHans Petter Selasky 8629ca874cfSHans Petter Selasky /* Check if there is a timestamp option. */ 8639ca874cfSHans Petter Selasky if (tcp_opt_len == 0 || 8649ca874cfSHans Petter Selasky __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA || 8659ca874cfSHans Petter Selasky *ts_ptr != TCP_LRO_TS_OPTION)) { 8669ca874cfSHans Petter Selasky /* We failed to find the timestamp option. */ 8679ca874cfSHans Petter Selasky le->timestamp = 0; 8689ca874cfSHans Petter Selasky } else { 869e57b2d0eSRandall Stewart le->timestamp = 1; 870e57b2d0eSRandall Stewart le->tsval = ntohl(*(ts_ptr + 1)); 871e57b2d0eSRandall Stewart le->tsecr = *(ts_ptr + 2); 8729ca874cfSHans Petter Selasky } 8739ca874cfSHans Petter Selasky 8749ca874cfSHans Petter Selasky tcp_data_len = m->m_pkthdr.lro_tcp_d_len; 8759ca874cfSHans Petter Selasky 8769ca874cfSHans Petter Selasky /* Pull out TCP sequence numbers and window size. */ 877e57b2d0eSRandall Stewart le->next_seq = ntohl(th->th_seq) + tcp_data_len; 878e57b2d0eSRandall Stewart le->ack_seq = th->th_ack; 879e57b2d0eSRandall Stewart le->window = th->th_win; 8809ca874cfSHans Petter Selasky 8819ca874cfSHans Petter Selasky /* Setup new data pointers. */ 882e57b2d0eSRandall Stewart le->m_head = m; 883e57b2d0eSRandall Stewart le->m_tail = m_last(m); 884e57b2d0eSRandall Stewart } 885e57b2d0eSRandall Stewart 886e57b2d0eSRandall Stewart static void 8879ca874cfSHans Petter Selasky tcp_push_and_replace(struct lro_ctrl *lc, struct lro_entry *le, struct mbuf *m) 888e57b2d0eSRandall Stewart { 8899ca874cfSHans Petter Selasky struct lro_parser *pa; 8909ca874cfSHans Petter Selasky 891e57b2d0eSRandall Stewart /* 8929ca874cfSHans Petter Selasky * Push up the stack of the current entry 8939ca874cfSHans Petter Selasky * and replace it with "m". 894e57b2d0eSRandall Stewart */ 895e57b2d0eSRandall Stewart struct mbuf *msave; 896e57b2d0eSRandall Stewart 897e57b2d0eSRandall Stewart /* Grab off the next and save it */ 898e57b2d0eSRandall Stewart msave = le->m_head->m_nextpkt; 899e57b2d0eSRandall Stewart le->m_head->m_nextpkt = NULL; 9009ca874cfSHans Petter Selasky 9019ca874cfSHans Petter Selasky /* Now push out the old entry */ 9029ca874cfSHans Petter Selasky tcp_flush_out_entry(lc, le); 9039ca874cfSHans Petter Selasky 9049ca874cfSHans Petter Selasky /* Re-parse new header, should not fail. */ 9059ca874cfSHans Petter Selasky pa = tcp_lro_parser(m, &le->outer, &le->inner, false); 9069ca874cfSHans Petter Selasky KASSERT(pa != NULL, 9079ca874cfSHans Petter Selasky ("tcp_push_and_replace: LRO parser failed on m=%p\n", m)); 9089ca874cfSHans Petter Selasky 909e57b2d0eSRandall Stewart /* 9109ca874cfSHans Petter Selasky * Now to replace the data properly in the entry 9119ca874cfSHans Petter Selasky * we have to reset the TCP header and 912e57b2d0eSRandall Stewart * other fields. 913e57b2d0eSRandall Stewart */ 9149ca874cfSHans Petter Selasky tcp_set_entry_to_mbuf(lc, le, m, pa->tcp); 9159ca874cfSHans Petter Selasky 916e57b2d0eSRandall Stewart /* Restore the next list */ 917e57b2d0eSRandall Stewart m->m_nextpkt = msave; 918e57b2d0eSRandall Stewart } 919e57b2d0eSRandall Stewart 920e57b2d0eSRandall Stewart static void 9219ca874cfSHans Petter Selasky tcp_lro_mbuf_append_pkthdr(struct mbuf *m, const struct mbuf *p) 9229ca874cfSHans Petter Selasky { 9239ca874cfSHans Petter Selasky uint32_t csum; 9249ca874cfSHans Petter Selasky 9259ca874cfSHans Petter Selasky if (m->m_pkthdr.lro_nsegs == 1) { 9269ca874cfSHans Petter Selasky /* Compute relative checksum. */ 9279ca874cfSHans Petter Selasky csum = p->m_pkthdr.lro_tcp_d_csum; 9289ca874cfSHans Petter Selasky } else { 9299ca874cfSHans Petter Selasky /* Merge TCP data checksums. */ 9309ca874cfSHans Petter Selasky csum = (uint32_t)m->m_pkthdr.lro_tcp_d_csum + 9319ca874cfSHans Petter Selasky (uint32_t)p->m_pkthdr.lro_tcp_d_csum; 9329ca874cfSHans Petter Selasky while (csum > 0xffff) 9339ca874cfSHans Petter Selasky csum = (csum >> 16) + (csum & 0xffff); 9349ca874cfSHans Petter Selasky } 9359ca874cfSHans Petter Selasky 9369ca874cfSHans Petter Selasky /* Update various counters. */ 9379ca874cfSHans Petter Selasky m->m_pkthdr.len += p->m_pkthdr.lro_tcp_d_len; 9389ca874cfSHans Petter Selasky m->m_pkthdr.lro_tcp_d_csum = csum; 9399ca874cfSHans Petter Selasky m->m_pkthdr.lro_tcp_d_len += p->m_pkthdr.lro_tcp_d_len; 9409ca874cfSHans Petter Selasky m->m_pkthdr.lro_nsegs += p->m_pkthdr.lro_nsegs; 9419ca874cfSHans Petter Selasky } 9429ca874cfSHans Petter Selasky 9439ca874cfSHans Petter Selasky static void 9449ca874cfSHans Petter Selasky tcp_lro_condense(struct lro_ctrl *lc, struct lro_entry *le) 945e57b2d0eSRandall Stewart { 946e57b2d0eSRandall Stewart /* 947e57b2d0eSRandall Stewart * Walk through the mbuf chain we 948e57b2d0eSRandall Stewart * have on tap and compress/condense 949e57b2d0eSRandall Stewart * as required. 950e57b2d0eSRandall Stewart */ 951e57b2d0eSRandall Stewart uint32_t *ts_ptr; 952e57b2d0eSRandall Stewart struct mbuf *m; 953e57b2d0eSRandall Stewart struct tcphdr *th; 9549ca874cfSHans Petter Selasky uint32_t tcp_data_len_total; 9559ca874cfSHans Petter Selasky uint32_t tcp_data_seg_total; 9569ca874cfSHans Petter Selasky uint16_t tcp_data_len; 9579ca874cfSHans Petter Selasky uint16_t tcp_opt_len; 958e57b2d0eSRandall Stewart 959e57b2d0eSRandall Stewart /* 960e57b2d0eSRandall Stewart * First we must check the lead (m_head) 961e57b2d0eSRandall Stewart * we must make sure that it is *not* 962e57b2d0eSRandall Stewart * something that should be sent up 963e57b2d0eSRandall Stewart * right away (sack etc). 964e57b2d0eSRandall Stewart */ 965e57b2d0eSRandall Stewart again: 966e57b2d0eSRandall Stewart m = le->m_head->m_nextpkt; 967e57b2d0eSRandall Stewart if (m == NULL) { 9689ca874cfSHans Petter Selasky /* Just one left. */ 969e57b2d0eSRandall Stewart return; 970e57b2d0eSRandall Stewart } 9719ca874cfSHans Petter Selasky 9729ca874cfSHans Petter Selasky th = tcp_lro_get_th(m); 9739ca874cfSHans Petter Selasky tcp_opt_len = (th->th_off << 2); 9749ca874cfSHans Petter Selasky tcp_opt_len -= sizeof(*th); 975e57b2d0eSRandall Stewart ts_ptr = (uint32_t *)(th + 1); 9769ca874cfSHans Petter Selasky 9779ca874cfSHans Petter Selasky if (tcp_opt_len != 0 && __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA || 9789ca874cfSHans Petter Selasky *ts_ptr != TCP_LRO_TS_OPTION)) { 979e57b2d0eSRandall Stewart /* 980e57b2d0eSRandall Stewart * Its not the timestamp. We can't 981e57b2d0eSRandall Stewart * use this guy as the head. 982e57b2d0eSRandall Stewart */ 983e57b2d0eSRandall Stewart le->m_head->m_nextpkt = m->m_nextpkt; 9849ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 985e57b2d0eSRandall Stewart goto again; 986e57b2d0eSRandall Stewart } 987e57b2d0eSRandall Stewart if ((th->th_flags & ~(TH_ACK | TH_PUSH)) != 0) { 988e57b2d0eSRandall Stewart /* 989e57b2d0eSRandall Stewart * Make sure that previously seen segements/ACKs are delivered 990e57b2d0eSRandall Stewart * before this segment, e.g. FIN. 991e57b2d0eSRandall Stewart */ 992e57b2d0eSRandall Stewart le->m_head->m_nextpkt = m->m_nextpkt; 9939ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 994e57b2d0eSRandall Stewart goto again; 995e57b2d0eSRandall Stewart } 996e57b2d0eSRandall Stewart while((m = le->m_head->m_nextpkt) != NULL) { 997e57b2d0eSRandall Stewart /* 998e57b2d0eSRandall Stewart * condense m into le, first 999e57b2d0eSRandall Stewart * pull m out of the list. 1000e57b2d0eSRandall Stewart */ 1001e57b2d0eSRandall Stewart le->m_head->m_nextpkt = m->m_nextpkt; 1002e57b2d0eSRandall Stewart m->m_nextpkt = NULL; 1003e57b2d0eSRandall Stewart /* Setup my data */ 10049ca874cfSHans Petter Selasky tcp_data_len = m->m_pkthdr.lro_tcp_d_len; 10059ca874cfSHans Petter Selasky th = tcp_lro_get_th(m); 1006e57b2d0eSRandall Stewart ts_ptr = (uint32_t *)(th + 1); 10079ca874cfSHans Petter Selasky tcp_opt_len = (th->th_off << 2); 10089ca874cfSHans Petter Selasky tcp_opt_len -= sizeof(*th); 10099ca874cfSHans Petter Selasky tcp_data_len_total = le->m_head->m_pkthdr.lro_tcp_d_len + tcp_data_len; 10109ca874cfSHans Petter Selasky tcp_data_seg_total = le->m_head->m_pkthdr.lro_nsegs + m->m_pkthdr.lro_nsegs; 10119ca874cfSHans Petter Selasky 10129ca874cfSHans Petter Selasky if (tcp_data_seg_total >= lc->lro_ackcnt_lim || 10139ca874cfSHans Petter Selasky tcp_data_len_total >= lc->lro_length_lim) { 1014e57b2d0eSRandall Stewart /* Flush now if appending will result in overflow. */ 10159ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 1016e57b2d0eSRandall Stewart goto again; 1017e57b2d0eSRandall Stewart } 10189ca874cfSHans Petter Selasky if (tcp_opt_len != 0 && 10199ca874cfSHans Petter Selasky __predict_false(tcp_opt_len != TCPOLEN_TSTAMP_APPA || 10209ca874cfSHans Petter Selasky *ts_ptr != TCP_LRO_TS_OPTION)) { 1021e57b2d0eSRandall Stewart /* 1022e57b2d0eSRandall Stewart * Maybe a sack in the new one? We need to 1023e57b2d0eSRandall Stewart * start all over after flushing the 1024e57b2d0eSRandall Stewart * current le. We will go up to the beginning 1025e57b2d0eSRandall Stewart * and flush it (calling the replace again possibly 1026e57b2d0eSRandall Stewart * or just returning). 1027e57b2d0eSRandall Stewart */ 10289ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 1029e57b2d0eSRandall Stewart goto again; 1030e57b2d0eSRandall Stewart } 1031e57b2d0eSRandall Stewart if ((th->th_flags & ~(TH_ACK | TH_PUSH)) != 0) { 10329ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 1033e57b2d0eSRandall Stewart goto again; 1034e57b2d0eSRandall Stewart } 10359ca874cfSHans Petter Selasky if (tcp_opt_len != 0) { 1036e57b2d0eSRandall Stewart uint32_t tsval = ntohl(*(ts_ptr + 1)); 1037e57b2d0eSRandall Stewart /* Make sure timestamp values are increasing. */ 1038e57b2d0eSRandall Stewart if (TSTMP_GT(le->tsval, tsval)) { 10399ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 1040e57b2d0eSRandall Stewart goto again; 1041e57b2d0eSRandall Stewart } 1042e57b2d0eSRandall Stewart le->tsval = tsval; 1043e57b2d0eSRandall Stewart le->tsecr = *(ts_ptr + 2); 1044e57b2d0eSRandall Stewart } 1045e57b2d0eSRandall Stewart /* Try to append the new segment. */ 1046e57b2d0eSRandall Stewart if (__predict_false(ntohl(th->th_seq) != le->next_seq || 1047e57b2d0eSRandall Stewart (tcp_data_len == 0 && 1048e57b2d0eSRandall Stewart le->ack_seq == th->th_ack && 1049e57b2d0eSRandall Stewart le->window == th->th_win))) { 1050e57b2d0eSRandall Stewart /* Out of order packet or duplicate ACK. */ 10519ca874cfSHans Petter Selasky tcp_push_and_replace(lc, le, m); 1052e57b2d0eSRandall Stewart goto again; 1053e57b2d0eSRandall Stewart } 10549ca874cfSHans Petter Selasky if (tcp_data_len != 0 || 10559ca874cfSHans Petter Selasky SEQ_GT(ntohl(th->th_ack), ntohl(le->ack_seq))) { 1056e57b2d0eSRandall Stewart le->next_seq += tcp_data_len; 1057e57b2d0eSRandall Stewart le->ack_seq = th->th_ack; 1058e57b2d0eSRandall Stewart le->window = th->th_win; 1059e57b2d0eSRandall Stewart } else if (th->th_ack == le->ack_seq) { 1060e57b2d0eSRandall Stewart le->window = WIN_MAX(le->window, th->th_win); 1061e57b2d0eSRandall Stewart } 10629ca874cfSHans Petter Selasky 1063e57b2d0eSRandall Stewart if (tcp_data_len == 0) { 1064e57b2d0eSRandall Stewart m_freem(m); 1065e57b2d0eSRandall Stewart continue; 1066e57b2d0eSRandall Stewart } 10679ca874cfSHans Petter Selasky 10689ca874cfSHans Petter Selasky /* Merge TCP data checksum and length to head mbuf. */ 10699ca874cfSHans Petter Selasky tcp_lro_mbuf_append_pkthdr(le->m_head, m); 10709ca874cfSHans Petter Selasky 1071e57b2d0eSRandall Stewart /* 1072e57b2d0eSRandall Stewart * Adjust the mbuf so that m_data points to the first byte of 1073e57b2d0eSRandall Stewart * the ULP payload. Adjust the mbuf to avoid complications and 1074e57b2d0eSRandall Stewart * append new segment to existing mbuf chain. 1075e57b2d0eSRandall Stewart */ 1076e57b2d0eSRandall Stewart m_adj(m, m->m_pkthdr.len - tcp_data_len); 1077e57b2d0eSRandall Stewart m_demote_pkthdr(m); 1078e57b2d0eSRandall Stewart le->m_tail->m_next = m; 1079e57b2d0eSRandall Stewart le->m_tail = m_last(m); 1080e57b2d0eSRandall Stewart } 1081e57b2d0eSRandall Stewart } 1082e57b2d0eSRandall Stewart 1083373013b0SConrad Meyer #ifdef TCPHPTS 1084e57b2d0eSRandall Stewart static void 10859ca874cfSHans Petter Selasky tcp_queue_pkts(struct inpcb *inp, struct tcpcb *tp, struct lro_entry *le) 1086e57b2d0eSRandall Stewart { 10879ca874cfSHans Petter Selasky INP_WLOCK_ASSERT(inp); 1088e57b2d0eSRandall Stewart if (tp->t_in_pkt == NULL) { 1089e57b2d0eSRandall Stewart /* Nothing yet there */ 1090e57b2d0eSRandall Stewart tp->t_in_pkt = le->m_head; 1091e57b2d0eSRandall Stewart tp->t_tail_pkt = le->m_last_mbuf; 1092e57b2d0eSRandall Stewart } else { 1093e57b2d0eSRandall Stewart /* Already some there */ 1094e57b2d0eSRandall Stewart tp->t_tail_pkt->m_nextpkt = le->m_head; 1095e57b2d0eSRandall Stewart tp->t_tail_pkt = le->m_last_mbuf; 1096e57b2d0eSRandall Stewart } 1097e57b2d0eSRandall Stewart le->m_head = NULL; 1098e57b2d0eSRandall Stewart le->m_last_mbuf = NULL; 1099e57b2d0eSRandall Stewart } 1100e57b2d0eSRandall Stewart 110169a34e8dSRandall Stewart static struct mbuf * 11029ca874cfSHans Petter Selasky tcp_lro_get_last_if_ackcmp(struct lro_ctrl *lc, struct lro_entry *le, 11039ca874cfSHans Petter Selasky struct inpcb *inp, int32_t *new_m) 1104e57b2d0eSRandall Stewart { 110569a34e8dSRandall Stewart struct tcpcb *tp; 11069ca874cfSHans Petter Selasky struct mbuf *m; 1107e57b2d0eSRandall Stewart 110869a34e8dSRandall Stewart tp = intotcpcb(inp); 11099ca874cfSHans Petter Selasky if (__predict_false(tp == NULL)) 11109ca874cfSHans Petter Selasky return (NULL); 11119ca874cfSHans Petter Selasky 111269a34e8dSRandall Stewart /* Look at the last mbuf if any in queue */ 111369a34e8dSRandall Stewart m = tp->t_tail_pkt; 11149ca874cfSHans Petter Selasky if (m != NULL && (m->m_flags & M_ACKCMP) != 0) { 11159ca874cfSHans Petter Selasky if (M_TRAILINGSPACE(m) >= sizeof(struct tcp_ackent)) { 11169ca874cfSHans Petter Selasky tcp_lro_log(tp, lc, le, NULL, 23, 0, 0, 0, 0); 11179ca874cfSHans Petter Selasky *new_m = 0; 11189ca874cfSHans Petter Selasky counter_u64_add(tcp_extra_mbuf, 1); 11199ca874cfSHans Petter Selasky return (m); 112069a34e8dSRandall Stewart } else { 11219ca874cfSHans Petter Selasky /* Mark we ran out of space */ 112269a34e8dSRandall Stewart inp->inp_flags2 |= INP_MBUF_L_ACKS; 112369a34e8dSRandall Stewart } 112469a34e8dSRandall Stewart } 11259ca874cfSHans Petter Selasky /* Decide mbuf size. */ 11269ca874cfSHans Petter Selasky if (inp->inp_flags2 & INP_MBUF_L_ACKS) 11279ca874cfSHans Petter Selasky m = m_getcl(M_NOWAIT, MT_DATA, M_ACKCMP | M_PKTHDR); 11289ca874cfSHans Petter Selasky else 11299ca874cfSHans Petter Selasky m = m_gethdr(M_NOWAIT, MT_DATA); 11309ca874cfSHans Petter Selasky 11319ca874cfSHans Petter Selasky if (__predict_false(m == NULL)) { 11329ca874cfSHans Petter Selasky counter_u64_add(tcp_would_have_but, 1); 11339ca874cfSHans Petter Selasky return (NULL); 113469a34e8dSRandall Stewart } 11359ca874cfSHans Petter Selasky counter_u64_add(tcp_comp_total, 1); 11369ca874cfSHans Petter Selasky m->m_flags |= M_ACKCMP; 11379ca874cfSHans Petter Selasky *new_m = 1; 113869a34e8dSRandall Stewart return (m); 113969a34e8dSRandall Stewart } 114069a34e8dSRandall Stewart 114169a34e8dSRandall Stewart static struct inpcb * 11429ca874cfSHans Petter Selasky tcp_lro_lookup(struct ifnet *ifp, struct lro_parser *pa) 114369a34e8dSRandall Stewart { 11449ca874cfSHans Petter Selasky struct inpcb *inp; 114569a34e8dSRandall Stewart 114669a34e8dSRandall Stewart NET_EPOCH_ASSERT(); 11479ca874cfSHans Petter Selasky 11489ca874cfSHans Petter Selasky switch (pa->data.lro_type) { 1149e57b2d0eSRandall Stewart #ifdef INET6 11509ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 11519ca874cfSHans Petter Selasky inp = in6_pcblookup(&V_tcbinfo, 11529ca874cfSHans Petter Selasky &pa->data.s_addr.v6, 11539ca874cfSHans Petter Selasky pa->data.s_port, 11549ca874cfSHans Petter Selasky &pa->data.d_addr.v6, 11559ca874cfSHans Petter Selasky pa->data.d_port, 1156e57b2d0eSRandall Stewart INPLOOKUP_WLOCKPCB, 11579ca874cfSHans Petter Selasky ifp); 1158e57b2d0eSRandall Stewart break; 1159e57b2d0eSRandall Stewart #endif 1160e57b2d0eSRandall Stewart #ifdef INET 11619ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 11629ca874cfSHans Petter Selasky inp = in_pcblookup(&V_tcbinfo, 11639ca874cfSHans Petter Selasky pa->data.s_addr.v4, 11649ca874cfSHans Petter Selasky pa->data.s_port, 11659ca874cfSHans Petter Selasky pa->data.d_addr.v4, 11669ca874cfSHans Petter Selasky pa->data.d_port, 1167e57b2d0eSRandall Stewart INPLOOKUP_WLOCKPCB, 11689ca874cfSHans Petter Selasky ifp); 1169e57b2d0eSRandall Stewart break; 1170e57b2d0eSRandall Stewart #endif 11719ca874cfSHans Petter Selasky default: 11729ca874cfSHans Petter Selasky inp = NULL; 11739ca874cfSHans Petter Selasky break; 1174e57b2d0eSRandall Stewart } 117569a34e8dSRandall Stewart return (inp); 117669a34e8dSRandall Stewart } 117769a34e8dSRandall Stewart 11789ca874cfSHans Petter Selasky static inline bool 11799ca874cfSHans Petter Selasky tcp_lro_ack_valid(struct mbuf *m, struct tcphdr *th, uint32_t **ppts, bool *other_opts) 11809ca874cfSHans Petter Selasky { 11819ca874cfSHans Petter Selasky /* 11829ca874cfSHans Petter Selasky * This function returns two bits of valuable information. 11839ca874cfSHans Petter Selasky * a) Is what is present capable of being ack-compressed, 11849ca874cfSHans Petter Selasky * we can ack-compress if there is no options or just 11859ca874cfSHans Petter Selasky * a timestamp option, and of course the th_flags must 11869ca874cfSHans Petter Selasky * be correct as well. 11879ca874cfSHans Petter Selasky * b) Our other options present such as SACK. This is 11889ca874cfSHans Petter Selasky * used to determine if we want to wakeup or not. 11899ca874cfSHans Petter Selasky */ 11909ca874cfSHans Petter Selasky bool ret = true; 11919ca874cfSHans Petter Selasky 11929ca874cfSHans Petter Selasky switch (th->th_off << 2) { 11939ca874cfSHans Petter Selasky case (sizeof(*th) + TCPOLEN_TSTAMP_APPA): 11949ca874cfSHans Petter Selasky *ppts = (uint32_t *)(th + 1); 11959ca874cfSHans Petter Selasky /* Check if we have only one timestamp option. */ 11969ca874cfSHans Petter Selasky if (**ppts == TCP_LRO_TS_OPTION) 11979ca874cfSHans Petter Selasky *other_opts = false; 11989ca874cfSHans Petter Selasky else { 11999ca874cfSHans Petter Selasky *other_opts = true; 12009ca874cfSHans Petter Selasky ret = false; 12019ca874cfSHans Petter Selasky } 12029ca874cfSHans Petter Selasky break; 12039ca874cfSHans Petter Selasky case (sizeof(*th)): 12049ca874cfSHans Petter Selasky /* No options. */ 12059ca874cfSHans Petter Selasky *ppts = NULL; 12069ca874cfSHans Petter Selasky *other_opts = false; 12079ca874cfSHans Petter Selasky break; 12089ca874cfSHans Petter Selasky default: 12099ca874cfSHans Petter Selasky *ppts = NULL; 12109ca874cfSHans Petter Selasky *other_opts = true; 12119ca874cfSHans Petter Selasky ret = false; 12129ca874cfSHans Petter Selasky break; 12139ca874cfSHans Petter Selasky } 12149ca874cfSHans Petter Selasky /* For ACKCMP we only accept ACK, PUSH, ECE and CWR. */ 12159ca874cfSHans Petter Selasky if ((th->th_flags & ~(TH_ACK | TH_PUSH | TH_ECE | TH_CWR)) != 0) 12169ca874cfSHans Petter Selasky ret = false; 12179ca874cfSHans Petter Selasky /* If it has data on it we cannot compress it */ 12189ca874cfSHans Petter Selasky if (m->m_pkthdr.lro_tcp_d_len) 12199ca874cfSHans Petter Selasky ret = false; 12209ca874cfSHans Petter Selasky 12219ca874cfSHans Petter Selasky /* ACK flag must be set. */ 12229ca874cfSHans Petter Selasky if (!(th->th_flags & TH_ACK)) 12239ca874cfSHans Petter Selasky ret = false; 12249ca874cfSHans Petter Selasky return (ret); 12259ca874cfSHans Petter Selasky } 12269ca874cfSHans Petter Selasky 12279ca874cfSHans Petter Selasky static int 12289ca874cfSHans Petter Selasky tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le) 12299ca874cfSHans Petter Selasky { 12309ca874cfSHans Petter Selasky struct inpcb *inp; 12319ca874cfSHans Petter Selasky struct tcpcb *tp; 12329ca874cfSHans Petter Selasky struct mbuf **pp, *cmp, *mv_to; 12339ca874cfSHans Petter Selasky bool bpf_req, should_wake; 12349ca874cfSHans Petter Selasky 12359ca874cfSHans Petter Selasky /* Check if packet doesn't belongs to our network interface. */ 12369ca874cfSHans Petter Selasky if ((tcplro_stacks_wanting_mbufq == 0) || 12379ca874cfSHans Petter Selasky (le->outer.data.vlan_id != 0) || 12389ca874cfSHans Petter Selasky (le->inner.data.lro_type != LRO_TYPE_NONE)) 12399ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 12409ca874cfSHans Petter Selasky 12419ca874cfSHans Petter Selasky #ifdef INET6 12429ca874cfSHans Petter Selasky /* 12439ca874cfSHans Petter Selasky * Be proactive about unspecified IPv6 address in source. As 12449ca874cfSHans Petter Selasky * we use all-zero to indicate unbounded/unconnected pcb, 12459ca874cfSHans Petter Selasky * unspecified IPv6 address can be used to confuse us. 12469ca874cfSHans Petter Selasky * 12479ca874cfSHans Petter Selasky * Note that packets with unspecified IPv6 destination is 12489ca874cfSHans Petter Selasky * already dropped in ip6_input. 12499ca874cfSHans Petter Selasky */ 12509ca874cfSHans Petter Selasky if (__predict_false(le->outer.data.lro_type == LRO_TYPE_IPV6_TCP && 12519ca874cfSHans Petter Selasky IN6_IS_ADDR_UNSPECIFIED(&le->outer.data.s_addr.v6))) 12529ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 12539ca874cfSHans Petter Selasky 12549ca874cfSHans Petter Selasky if (__predict_false(le->inner.data.lro_type == LRO_TYPE_IPV6_TCP && 12559ca874cfSHans Petter Selasky IN6_IS_ADDR_UNSPECIFIED(&le->inner.data.s_addr.v6))) 12569ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 125769a34e8dSRandall Stewart #endif 12589ca874cfSHans Petter Selasky /* Lookup inp, if any. */ 12599ca874cfSHans Petter Selasky inp = tcp_lro_lookup(lc->ifp, 12609ca874cfSHans Petter Selasky (le->inner.data.lro_type == LRO_TYPE_NONE) ? &le->outer : &le->inner); 12619ca874cfSHans Petter Selasky if (inp == NULL) 12629ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 126369a34e8dSRandall Stewart 12649ca874cfSHans Petter Selasky counter_u64_add(tcp_inp_lro_locks_taken, 1); 12659ca874cfSHans Petter Selasky 12669ca874cfSHans Petter Selasky /* Get TCP control structure. */ 12679ca874cfSHans Petter Selasky tp = intotcpcb(inp); 12689ca874cfSHans Petter Selasky 12699ca874cfSHans Petter Selasky /* Check if the inp is dead, Jim. */ 12709ca874cfSHans Petter Selasky if (tp == NULL || 12719ca874cfSHans Petter Selasky (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) || 12729ca874cfSHans Petter Selasky (inp->inp_flags2 & INP_FREED)) { 12739ca874cfSHans Petter Selasky INP_WUNLOCK(inp); 12749ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 127569a34e8dSRandall Stewart } 127669a34e8dSRandall Stewart 12779ca874cfSHans Petter Selasky /* Check if the transport doesn't support the needed optimizations. */ 12789ca874cfSHans Petter Selasky if ((inp->inp_flags2 & (INP_SUPPORTS_MBUFQ | INP_MBUF_ACKCMP)) == 0) { 12799ca874cfSHans Petter Selasky INP_WUNLOCK(inp); 12809ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 128169a34e8dSRandall Stewart } 12829ca874cfSHans Petter Selasky 12839ca874cfSHans Petter Selasky if (inp->inp_flags2 & INP_MBUF_QUEUE_READY) 12849ca874cfSHans Petter Selasky should_wake = false; 12859ca874cfSHans Petter Selasky else 12869ca874cfSHans Petter Selasky should_wake = true; 12879ca874cfSHans Petter Selasky /* Check if packets should be tapped to BPF. */ 12889ca874cfSHans Petter Selasky bpf_req = bpf_peers_present(lc->ifp->if_bpf); 12899ca874cfSHans Petter Selasky 12909ca874cfSHans Petter Selasky /* Strip and compress all the incoming packets. */ 12919ca874cfSHans Petter Selasky cmp = NULL; 12929ca874cfSHans Petter Selasky for (pp = &le->m_head; *pp != NULL; ) { 12939ca874cfSHans Petter Selasky mv_to = NULL; 12949ca874cfSHans Petter Selasky if (do_bpf_strip_and_compress(inp, lc, le, pp, 12959ca874cfSHans Petter Selasky &cmp, &mv_to, &should_wake, bpf_req ) == false) { 12969ca874cfSHans Petter Selasky /* Advance to next mbuf. */ 12979ca874cfSHans Petter Selasky pp = &(*pp)->m_nextpkt; 12989ca874cfSHans Petter Selasky } else if (mv_to != NULL) { 12999ca874cfSHans Petter Selasky /* We are asked to move pp up */ 13009ca874cfSHans Petter Selasky pp = &mv_to->m_nextpkt; 13019ca874cfSHans Petter Selasky } 13029ca874cfSHans Petter Selasky } 13039ca874cfSHans Petter Selasky /* Update "m_last_mbuf", if any. */ 13049ca874cfSHans Petter Selasky if (pp == &le->m_head) 13059ca874cfSHans Petter Selasky le->m_last_mbuf = *pp; 13069ca874cfSHans Petter Selasky else 13079ca874cfSHans Petter Selasky le->m_last_mbuf = __containerof(pp, struct mbuf, m_nextpkt); 13089ca874cfSHans Petter Selasky 13099ca874cfSHans Petter Selasky /* Check if any data mbufs left. */ 13109ca874cfSHans Petter Selasky if (le->m_head != NULL) { 13119ca874cfSHans Petter Selasky counter_u64_add(tcp_inp_lro_direct_queue, 1); 13129ca874cfSHans Petter Selasky tcp_lro_log(tp, lc, le, NULL, 22, 1, 13139ca874cfSHans Petter Selasky inp->inp_flags2, inp->inp_in_input, 1); 13149ca874cfSHans Petter Selasky tcp_queue_pkts(inp, tp, le); 13159ca874cfSHans Petter Selasky } 13169ca874cfSHans Petter Selasky if (should_wake) { 13179ca874cfSHans Petter Selasky /* Wakeup */ 13189ca874cfSHans Petter Selasky counter_u64_add(tcp_inp_lro_wokeup_queue, 1); 13199ca874cfSHans Petter Selasky if ((*tp->t_fb->tfb_do_queued_segments)(inp->inp_socket, tp, 0)) 13209ca874cfSHans Petter Selasky inp = NULL; 13219ca874cfSHans Petter Selasky } 13229ca874cfSHans Petter Selasky if (inp != NULL) 13239ca874cfSHans Petter Selasky INP_WUNLOCK(inp); 13249ca874cfSHans Petter Selasky return (0); /* Success. */ 132569a34e8dSRandall Stewart } 132669a34e8dSRandall Stewart #endif 132769a34e8dSRandall Stewart 132869a34e8dSRandall Stewart void 132969a34e8dSRandall Stewart tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le) 133069a34e8dSRandall Stewart { 13319ca874cfSHans Petter Selasky /* Only optimise if there are multiple packets waiting. */ 133269a34e8dSRandall Stewart #ifdef TCPHPTS 13339ca874cfSHans Petter Selasky int error; 133469a34e8dSRandall Stewart 133569a34e8dSRandall Stewart CURVNET_SET(lc->ifp->if_vnet); 13369ca874cfSHans Petter Selasky error = tcp_lro_flush_tcphpts(lc, le); 13379ca874cfSHans Petter Selasky CURVNET_RESTORE(); 13389ca874cfSHans Petter Selasky if (error != 0) { 13399ca874cfSHans Petter Selasky #endif 13409ca874cfSHans Petter Selasky tcp_lro_condense(lc, le); 13419ca874cfSHans Petter Selasky tcp_flush_out_entry(lc, le); 1342e57b2d0eSRandall Stewart #ifdef TCPHPTS 1343e57b2d0eSRandall Stewart } 1344e57b2d0eSRandall Stewart #endif 134562b5b6ecSBjoern A. Zeeb lc->lro_flushed++; 134662b5b6ecSBjoern A. Zeeb bzero(le, sizeof(*le)); 13471ea44822SSepherosa Ziehau LIST_INSERT_HEAD(&lc->lro_free, le, next); 134862b5b6ecSBjoern A. Zeeb } 13496c5087a8SJack F Vogel 1350fc271df3SHans Petter Selasky #ifdef HAVE_INLINE_FLSLL 1351fc271df3SHans Petter Selasky #define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1)) 1352fc271df3SHans Petter Selasky #else 1353fc271df3SHans Petter Selasky static inline uint64_t 1354fc271df3SHans Petter Selasky tcp_lro_msb_64(uint64_t x) 1355e936121dSHans Petter Selasky { 1356fc271df3SHans Petter Selasky x |= (x >> 1); 1357fc271df3SHans Petter Selasky x |= (x >> 2); 1358fc271df3SHans Petter Selasky x |= (x >> 4); 1359fc271df3SHans Petter Selasky x |= (x >> 8); 1360fc271df3SHans Petter Selasky x |= (x >> 16); 1361fc271df3SHans Petter Selasky x |= (x >> 32); 1362fc271df3SHans Petter Selasky return (x & ~(x >> 1)); 1363fc271df3SHans Petter Selasky } 1364fc271df3SHans Petter Selasky #endif 1365e936121dSHans Petter Selasky 1366fc271df3SHans Petter Selasky /* 1367fc271df3SHans Petter Selasky * The tcp_lro_sort() routine is comparable to qsort(), except it has 1368fc271df3SHans Petter Selasky * a worst case complexity limit of O(MIN(N,64)*N), where N is the 1369fc271df3SHans Petter Selasky * number of elements to sort and 64 is the number of sequence bits 1370fc271df3SHans Petter Selasky * available. The algorithm is bit-slicing the 64-bit sequence number, 1371fc271df3SHans Petter Selasky * sorting one bit at a time from the most significant bit until the 1372ec668905SHans Petter Selasky * least significant one, skipping the constant bits. This is 1373ec668905SHans Petter Selasky * typically called a radix sort. 1374fc271df3SHans Petter Selasky */ 1375fc271df3SHans Petter Selasky static void 1376fc271df3SHans Petter Selasky tcp_lro_sort(struct lro_mbuf_sort *parray, uint32_t size) 1377fc271df3SHans Petter Selasky { 1378fc271df3SHans Petter Selasky struct lro_mbuf_sort temp; 1379fc271df3SHans Petter Selasky uint64_t ones; 1380fc271df3SHans Petter Selasky uint64_t zeros; 1381fc271df3SHans Petter Selasky uint32_t x; 1382fc271df3SHans Petter Selasky uint32_t y; 1383e936121dSHans Petter Selasky 1384fc271df3SHans Petter Selasky repeat: 1385ec668905SHans Petter Selasky /* for small arrays insertion sort is faster */ 1386fc271df3SHans Petter Selasky if (size <= 12) { 1387ec668905SHans Petter Selasky for (x = 1; x < size; x++) { 1388fc271df3SHans Petter Selasky temp = parray[x]; 1389ec668905SHans Petter Selasky for (y = x; y > 0 && temp.seq < parray[y - 1].seq; y--) 1390ec668905SHans Petter Selasky parray[y] = parray[y - 1]; 1391fc271df3SHans Petter Selasky parray[y] = temp; 1392fc271df3SHans Petter Selasky } 1393fc271df3SHans Petter Selasky return; 1394fc271df3SHans Petter Selasky } 1395e936121dSHans Petter Selasky 1396fc271df3SHans Petter Selasky /* compute sequence bits which are constant */ 1397fc271df3SHans Petter Selasky ones = 0; 1398fc271df3SHans Petter Selasky zeros = 0; 1399fc271df3SHans Petter Selasky for (x = 0; x != size; x++) { 1400fc271df3SHans Petter Selasky ones |= parray[x].seq; 1401fc271df3SHans Petter Selasky zeros |= ~parray[x].seq; 1402fc271df3SHans Petter Selasky } 1403fc271df3SHans Petter Selasky 1404fc271df3SHans Petter Selasky /* compute bits which are not constant into "ones" */ 1405fc271df3SHans Petter Selasky ones &= zeros; 1406fc271df3SHans Petter Selasky if (ones == 0) 1407fc271df3SHans Petter Selasky return; 1408fc271df3SHans Petter Selasky 1409fc271df3SHans Petter Selasky /* pick the most significant bit which is not constant */ 1410fc271df3SHans Petter Selasky ones = tcp_lro_msb_64(ones); 1411fc271df3SHans Petter Selasky 1412fc271df3SHans Petter Selasky /* 1413fc271df3SHans Petter Selasky * Move entries having cleared sequence bits to the beginning 1414fc271df3SHans Petter Selasky * of the array: 1415fc271df3SHans Petter Selasky */ 1416fc271df3SHans Petter Selasky for (x = y = 0; y != size; y++) { 1417fc271df3SHans Petter Selasky /* skip set bits */ 1418fc271df3SHans Petter Selasky if (parray[y].seq & ones) 1419fc271df3SHans Petter Selasky continue; 1420fc271df3SHans Petter Selasky /* swap entries */ 1421fc271df3SHans Petter Selasky temp = parray[x]; 1422fc271df3SHans Petter Selasky parray[x] = parray[y]; 1423fc271df3SHans Petter Selasky parray[y] = temp; 1424fc271df3SHans Petter Selasky x++; 1425fc271df3SHans Petter Selasky } 1426fc271df3SHans Petter Selasky 1427fc271df3SHans Petter Selasky KASSERT(x != 0 && x != size, ("Memory is corrupted\n")); 1428fc271df3SHans Petter Selasky 1429fc271df3SHans Petter Selasky /* sort zeros */ 1430fc271df3SHans Petter Selasky tcp_lro_sort(parray, x); 1431fc271df3SHans Petter Selasky 1432fc271df3SHans Petter Selasky /* sort ones */ 1433fc271df3SHans Petter Selasky parray += x; 1434fc271df3SHans Petter Selasky size -= x; 1435fc271df3SHans Petter Selasky goto repeat; 1436e936121dSHans Petter Selasky } 1437e936121dSHans Petter Selasky 1438e936121dSHans Petter Selasky void 1439e936121dSHans Petter Selasky tcp_lro_flush_all(struct lro_ctrl *lc) 1440e936121dSHans Petter Selasky { 1441fc271df3SHans Petter Selasky uint64_t seq; 1442fc271df3SHans Petter Selasky uint64_t nseq; 1443e936121dSHans Petter Selasky unsigned x; 1444e936121dSHans Petter Selasky 1445e936121dSHans Petter Selasky /* check if no mbufs to flush */ 14466dd38b87SSepherosa Ziehau if (lc->lro_mbuf_count == 0) 1447e936121dSHans Petter Selasky goto done; 1448e936121dSHans Petter Selasky 1449a9b66dbdSHans Petter Selasky CURVNET_SET(lc->ifp->if_vnet); 1450a9b66dbdSHans Petter Selasky 14519ca874cfSHans Petter Selasky /* get current time */ 1452*b45daaeaSRandall Stewart binuptime(&lc->lro_last_queue_time); 14539ca874cfSHans Petter Selasky 1454e936121dSHans Petter Selasky /* sort all mbufs according to stream */ 1455fc271df3SHans Petter Selasky tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count); 1456e936121dSHans Petter Selasky 1457e936121dSHans Petter Selasky /* input data into LRO engine, stream by stream */ 1458fc271df3SHans Petter Selasky seq = 0; 1459e936121dSHans Petter Selasky for (x = 0; x != lc->lro_mbuf_count; x++) { 1460e936121dSHans Petter Selasky struct mbuf *mb; 1461e936121dSHans Petter Selasky 1462fc271df3SHans Petter Selasky /* get mbuf */ 1463fc271df3SHans Petter Selasky mb = lc->lro_mbuf_data[x].mb; 1464fc271df3SHans Petter Selasky 1465fc271df3SHans Petter Selasky /* get sequence number, masking away the packet index */ 1466fc271df3SHans Petter Selasky nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24); 1467e936121dSHans Petter Selasky 1468e936121dSHans Petter Selasky /* check for new stream */ 1469fc271df3SHans Petter Selasky if (seq != nseq) { 1470fc271df3SHans Petter Selasky seq = nseq; 1471e936121dSHans Petter Selasky 1472e936121dSHans Petter Selasky /* flush active streams */ 14736dd38b87SSepherosa Ziehau tcp_lro_rx_done(lc); 1474e936121dSHans Petter Selasky } 1475fc271df3SHans Petter Selasky 1476e936121dSHans Petter Selasky /* add packet to LRO engine */ 14779ca874cfSHans Petter Selasky if (tcp_lro_rx_common(lc, mb, 0, false) != 0) { 1478e936121dSHans Petter Selasky /* input packet to network layer */ 1479e936121dSHans Petter Selasky (*lc->ifp->if_input)(lc->ifp, mb); 1480e936121dSHans Petter Selasky lc->lro_queued++; 1481e936121dSHans Petter Selasky lc->lro_flushed++; 1482e936121dSHans Petter Selasky } 1483e936121dSHans Petter Selasky } 1484a9b66dbdSHans Petter Selasky CURVNET_RESTORE(); 1485e936121dSHans Petter Selasky done: 1486e936121dSHans Petter Selasky /* flush active streams */ 14876dd38b87SSepherosa Ziehau tcp_lro_rx_done(lc); 14886dd38b87SSepherosa Ziehau 1489e936121dSHans Petter Selasky lc->lro_mbuf_count = 0; 149062b5b6ecSBjoern A. Zeeb } 14916c5087a8SJack F Vogel 1492ab4fad4bSRandall Stewart #ifdef TCPHPTS 149369a34e8dSRandall Stewart static void 14949ca874cfSHans Petter Selasky build_ack_entry(struct tcp_ackent *ae, struct tcphdr *th, struct mbuf *m, 14959ca874cfSHans Petter Selasky uint32_t *ts_ptr, uint16_t iptos) 149669a34e8dSRandall Stewart { 149769a34e8dSRandall Stewart /* 14989ca874cfSHans Petter Selasky * Given a TCP ACK, summarize it down into the small TCP ACK 14999ca874cfSHans Petter Selasky * entry. 150069a34e8dSRandall Stewart */ 150169a34e8dSRandall Stewart ae->timestamp = m->m_pkthdr.rcv_tstmp; 150269a34e8dSRandall Stewart if (m->m_flags & M_TSTMP_LRO) 150369a34e8dSRandall Stewart ae->flags = TSTMP_LRO; 150469a34e8dSRandall Stewart else if (m->m_flags & M_TSTMP) 150569a34e8dSRandall Stewart ae->flags = TSTMP_HDWR; 150669a34e8dSRandall Stewart ae->seq = ntohl(th->th_seq); 150769a34e8dSRandall Stewart ae->ack = ntohl(th->th_ack); 150869a34e8dSRandall Stewart ae->flags |= th->th_flags; 15099ca874cfSHans Petter Selasky if (ts_ptr != NULL) { 15109ca874cfSHans Petter Selasky ae->ts_value = ntohl(ts_ptr[1]); 15119ca874cfSHans Petter Selasky ae->ts_echo = ntohl(ts_ptr[2]); 151269a34e8dSRandall Stewart ae->flags |= HAS_TSTMP; 151369a34e8dSRandall Stewart } 151469a34e8dSRandall Stewart ae->win = ntohs(th->th_win); 151569a34e8dSRandall Stewart ae->codepoint = iptos; 151669a34e8dSRandall Stewart } 151769a34e8dSRandall Stewart 151869a34e8dSRandall Stewart /* 15199ca874cfSHans Petter Selasky * Do BPF tap for either ACK_CMP packets or MBUF QUEUE type packets 15209ca874cfSHans Petter Selasky * and strip all, but the IPv4/IPv6 header. 152169a34e8dSRandall Stewart */ 15229ca874cfSHans Petter Selasky static bool 15239ca874cfSHans Petter Selasky do_bpf_strip_and_compress(struct inpcb *inp, struct lro_ctrl *lc, 15249ca874cfSHans Petter Selasky struct lro_entry *le, struct mbuf **pp, struct mbuf **cmp, struct mbuf **mv_to, 15259ca874cfSHans Petter Selasky bool *should_wake, bool bpf_req) 15269ca874cfSHans Petter Selasky { 15279ca874cfSHans Petter Selasky union { 15289ca874cfSHans Petter Selasky void *ptr; 15299ca874cfSHans Petter Selasky struct ip *ip4; 15309ca874cfSHans Petter Selasky struct ip6_hdr *ip6; 15319ca874cfSHans Petter Selasky } l3; 15329ca874cfSHans Petter Selasky struct mbuf *m; 15339ca874cfSHans Petter Selasky struct mbuf *nm; 153469a34e8dSRandall Stewart struct tcphdr *th; 15359ca874cfSHans Petter Selasky struct tcp_ackent *ack_ent; 15369ca874cfSHans Petter Selasky uint32_t *ts_ptr; 15379ca874cfSHans Petter Selasky int32_t n_mbuf; 15389ca874cfSHans Petter Selasky bool other_opts, can_compress; 15399ca874cfSHans Petter Selasky uint16_t lro_type; 15409ca874cfSHans Petter Selasky uint16_t iptos; 15419ca874cfSHans Petter Selasky int tcp_hdr_offset; 15429ca874cfSHans Petter Selasky int idx; 154369a34e8dSRandall Stewart 15449ca874cfSHans Petter Selasky /* Get current mbuf. */ 15459ca874cfSHans Petter Selasky m = *pp; 154669a34e8dSRandall Stewart 154769a34e8dSRandall Stewart /* Let the BPF see the packet */ 15489ca874cfSHans Petter Selasky if (__predict_false(bpf_req)) 154969a34e8dSRandall Stewart ETHER_BPF_MTAP(lc->ifp, m); 15509ca874cfSHans Petter Selasky 15519ca874cfSHans Petter Selasky tcp_hdr_offset = m->m_pkthdr.lro_tcp_h_off; 15529ca874cfSHans Petter Selasky lro_type = le->inner.data.lro_type; 15539ca874cfSHans Petter Selasky switch (lro_type) { 15549ca874cfSHans Petter Selasky case LRO_TYPE_NONE: 15559ca874cfSHans Petter Selasky lro_type = le->outer.data.lro_type; 15569ca874cfSHans Petter Selasky switch (lro_type) { 15579ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 15589ca874cfSHans Petter Selasky tcp_hdr_offset -= sizeof(*le->outer.ip4); 15599ca874cfSHans Petter Selasky m->m_pkthdr.lro_etype = ETHERTYPE_IP; 15609ca874cfSHans Petter Selasky break; 15619ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 15629ca874cfSHans Petter Selasky tcp_hdr_offset -= sizeof(*le->outer.ip6); 15639ca874cfSHans Petter Selasky m->m_pkthdr.lro_etype = ETHERTYPE_IPV6; 15649ca874cfSHans Petter Selasky break; 15659ca874cfSHans Petter Selasky default: 15669ca874cfSHans Petter Selasky goto compressed; 15679ca874cfSHans Petter Selasky } 15689ca874cfSHans Petter Selasky break; 15699ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 15709ca874cfSHans Petter Selasky tcp_hdr_offset -= sizeof(*le->outer.ip4); 15719ca874cfSHans Petter Selasky m->m_pkthdr.lro_etype = ETHERTYPE_IP; 15729ca874cfSHans Petter Selasky break; 15739ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 15749ca874cfSHans Petter Selasky tcp_hdr_offset -= sizeof(*le->outer.ip6); 15759ca874cfSHans Petter Selasky m->m_pkthdr.lro_etype = ETHERTYPE_IPV6; 15769ca874cfSHans Petter Selasky break; 15779ca874cfSHans Petter Selasky default: 15789ca874cfSHans Petter Selasky goto compressed; 15799ca874cfSHans Petter Selasky } 15809ca874cfSHans Petter Selasky 15819ca874cfSHans Petter Selasky MPASS(tcp_hdr_offset >= 0); 15829ca874cfSHans Petter Selasky 15839ca874cfSHans Petter Selasky m_adj(m, tcp_hdr_offset); 158469a34e8dSRandall Stewart m->m_flags |= M_LRO_EHDRSTRP; 15859ca874cfSHans Petter Selasky m->m_flags &= ~M_ACKCMP; 15869ca874cfSHans Petter Selasky m->m_pkthdr.lro_tcp_h_off -= tcp_hdr_offset; 158769a34e8dSRandall Stewart 15889ca874cfSHans Petter Selasky th = tcp_lro_get_th(m); 158969a34e8dSRandall Stewart 15909ca874cfSHans Petter Selasky th->th_sum = 0; /* TCP checksum is valid. */ 159169a34e8dSRandall Stewart 15929ca874cfSHans Petter Selasky /* Check if ACK can be compressed */ 15939ca874cfSHans Petter Selasky can_compress = tcp_lro_ack_valid(m, th, &ts_ptr, &other_opts); 159469a34e8dSRandall Stewart 15959ca874cfSHans Petter Selasky /* Now lets look at the should wake states */ 15969ca874cfSHans Petter Selasky if ((other_opts == true) && 15979ca874cfSHans Petter Selasky ((inp->inp_flags2 & INP_DONT_SACK_QUEUE) == 0)) { 159869a34e8dSRandall Stewart /* 15999ca874cfSHans Petter Selasky * If there are other options (SACK?) and the 16009ca874cfSHans Petter Selasky * tcp endpoint has not expressly told us it does 16019ca874cfSHans Petter Selasky * not care about SACKS, then we should wake up. 160269a34e8dSRandall Stewart */ 16039ca874cfSHans Petter Selasky *should_wake = true; 160469a34e8dSRandall Stewart } 16059ca874cfSHans Petter Selasky /* Is the ack compressable? */ 16069ca874cfSHans Petter Selasky if (can_compress == false) 16079ca874cfSHans Petter Selasky goto done; 16089ca874cfSHans Petter Selasky /* Does the TCP endpoint support ACK compression? */ 16099ca874cfSHans Petter Selasky if ((inp->inp_flags2 & INP_MBUF_ACKCMP) == 0) 16109ca874cfSHans Petter Selasky goto done; 16119ca874cfSHans Petter Selasky 16129ca874cfSHans Petter Selasky /* Lets get the TOS/traffic class field */ 16139ca874cfSHans Petter Selasky l3.ptr = mtod(m, void *); 16149ca874cfSHans Petter Selasky switch (lro_type) { 16159ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 16169ca874cfSHans Petter Selasky iptos = l3.ip4->ip_tos; 16179ca874cfSHans Petter Selasky break; 16189ca874cfSHans Petter Selasky case LRO_TYPE_IPV6_TCP: 16199ca874cfSHans Petter Selasky iptos = IPV6_TRAFFIC_CLASS(l3.ip6); 16209ca874cfSHans Petter Selasky break; 16219ca874cfSHans Petter Selasky default: 16229ca874cfSHans Petter Selasky iptos = 0; /* Keep compiler happy. */ 162369a34e8dSRandall Stewart break; 162469a34e8dSRandall Stewart } 16259ca874cfSHans Petter Selasky /* Now lets get space if we don't have some already */ 16269ca874cfSHans Petter Selasky if (*cmp == NULL) { 16279ca874cfSHans Petter Selasky new_one: 16289ca874cfSHans Petter Selasky nm = tcp_lro_get_last_if_ackcmp(lc, le, inp, &n_mbuf); 16299ca874cfSHans Petter Selasky if (__predict_false(nm == NULL)) 16309ca874cfSHans Petter Selasky goto done; 16319ca874cfSHans Petter Selasky *cmp = nm; 16329ca874cfSHans Petter Selasky if (n_mbuf) { 163369a34e8dSRandall Stewart /* 16349ca874cfSHans Petter Selasky * Link in the new cmp ack to our in-order place, 16359ca874cfSHans Petter Selasky * first set our cmp ack's next to where we are. 163669a34e8dSRandall Stewart */ 16379ca874cfSHans Petter Selasky nm->m_nextpkt = m; 16389ca874cfSHans Petter Selasky (*pp) = nm; 16399ca874cfSHans Petter Selasky /* 16409ca874cfSHans Petter Selasky * Set it up so mv_to is advanced to our 16419ca874cfSHans Petter Selasky * compressed ack. This way the caller can 16429ca874cfSHans Petter Selasky * advance pp to the right place. 16439ca874cfSHans Petter Selasky */ 16449ca874cfSHans Petter Selasky *mv_to = nm; 16459ca874cfSHans Petter Selasky /* 16469ca874cfSHans Petter Selasky * Advance it here locally as well. 16479ca874cfSHans Petter Selasky */ 16489ca874cfSHans Petter Selasky pp = &nm->m_nextpkt; 164969a34e8dSRandall Stewart } 16509ca874cfSHans Petter Selasky } else { 16519ca874cfSHans Petter Selasky /* We have one already we are working on */ 16529ca874cfSHans Petter Selasky nm = *cmp; 16539ca874cfSHans Petter Selasky if (M_TRAILINGSPACE(nm) < sizeof(struct tcp_ackent)) { 16549ca874cfSHans Petter Selasky /* We ran out of space */ 16559ca874cfSHans Petter Selasky inp->inp_flags2 |= INP_MBUF_L_ACKS; 16569ca874cfSHans Petter Selasky goto new_one; 16579ca874cfSHans Petter Selasky } 16589ca874cfSHans Petter Selasky } 16599ca874cfSHans Petter Selasky MPASS(M_TRAILINGSPACE(nm) >= sizeof(struct tcp_ackent)); 16609ca874cfSHans Petter Selasky counter_u64_add(tcp_inp_lro_compressed, 1); 16619ca874cfSHans Petter Selasky le->compressed++; 16629ca874cfSHans Petter Selasky /* We can add in to the one on the tail */ 16639ca874cfSHans Petter Selasky ack_ent = mtod(nm, struct tcp_ackent *); 16649ca874cfSHans Petter Selasky idx = (nm->m_len / sizeof(struct tcp_ackent)); 16659ca874cfSHans Petter Selasky build_ack_entry(&ack_ent[idx], th, m, ts_ptr, iptos); 166669a34e8dSRandall Stewart 16679ca874cfSHans Petter Selasky /* Bump the size of both pkt-hdr and len */ 16689ca874cfSHans Petter Selasky nm->m_len += sizeof(struct tcp_ackent); 16699ca874cfSHans Petter Selasky nm->m_pkthdr.len += sizeof(struct tcp_ackent); 16709ca874cfSHans Petter Selasky compressed: 16719ca874cfSHans Petter Selasky /* Advance to next mbuf before freeing. */ 16729ca874cfSHans Petter Selasky *pp = m->m_nextpkt; 16739ca874cfSHans Petter Selasky m->m_nextpkt = NULL; 167469a34e8dSRandall Stewart m_freem(m); 16759ca874cfSHans Petter Selasky return (true); 16769ca874cfSHans Petter Selasky done: 16779ca874cfSHans Petter Selasky counter_u64_add(tcp_uncomp_total, 1); 16789ca874cfSHans Petter Selasky le->uncompressed++; 16799ca874cfSHans Petter Selasky return (false); 168069a34e8dSRandall Stewart } 168169a34e8dSRandall Stewart #endif 16829ca874cfSHans Petter Selasky 16839ca874cfSHans Petter Selasky static struct lro_head * 16849ca874cfSHans Petter Selasky tcp_lro_rx_get_bucket(struct lro_ctrl *lc, struct mbuf *m, struct lro_parser *parser) 16859ca874cfSHans Petter Selasky { 16869ca874cfSHans Petter Selasky u_long hash; 16879ca874cfSHans Petter Selasky 16889ca874cfSHans Petter Selasky if (M_HASHTYPE_ISHASH(m)) { 16899ca874cfSHans Petter Selasky hash = m->m_pkthdr.flowid; 16909ca874cfSHans Petter Selasky } else { 16919ca874cfSHans Petter Selasky for (unsigned i = hash = 0; i != LRO_RAW_ADDRESS_MAX; i++) 16929ca874cfSHans Petter Selasky hash += parser->data.raw[i]; 169369a34e8dSRandall Stewart } 16949ca874cfSHans Petter Selasky return (&lc->lro_hash[hash % lc->lro_hashsz]); 16959ca874cfSHans Petter Selasky } 169669a34e8dSRandall Stewart 169705cde7efSSepherosa Ziehau static int 16989ca874cfSHans Petter Selasky tcp_lro_rx_common(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum, bool use_hash) 169962b5b6ecSBjoern A. Zeeb { 17009ca874cfSHans Petter Selasky struct lro_parser pi; /* inner address data */ 17019ca874cfSHans Petter Selasky struct lro_parser po; /* outer address data */ 17029ca874cfSHans Petter Selasky struct lro_parser *pa; /* current parser for TCP stream */ 170362b5b6ecSBjoern A. Zeeb struct lro_entry *le; 170405cde7efSSepherosa Ziehau struct lro_head *bucket; 17059ca874cfSHans Petter Selasky struct tcphdr *th; 17069ca874cfSHans Petter Selasky int tcp_data_len; 17079ca874cfSHans Petter Selasky int tcp_opt_len; 17089ca874cfSHans Petter Selasky int error; 17099ca874cfSHans Petter Selasky uint16_t tcp_data_sum; 17106c5087a8SJack F Vogel 17119ca874cfSHans Petter Selasky #ifdef INET 17129ca874cfSHans Petter Selasky /* Quickly decide if packet cannot be LRO'ed */ 17139ca874cfSHans Petter Selasky if (__predict_false(V_ipforwarding != 0)) 17149ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 17159ca874cfSHans Petter Selasky #endif 17169ca874cfSHans Petter Selasky #ifdef INET6 17179ca874cfSHans Petter Selasky /* Quickly decide if packet cannot be LRO'ed */ 17189ca874cfSHans Petter Selasky if (__predict_false(V_ip6_forwarding != 0)) 17199ca874cfSHans Petter Selasky return (TCP_LRO_CANNOT); 17209ca874cfSHans Petter Selasky #endif 172169a34e8dSRandall Stewart 172262b5b6ecSBjoern A. Zeeb /* We expect a contiguous header [eh, ip, tcp]. */ 17239ca874cfSHans Petter Selasky pa = tcp_lro_parser(m, &po, &pi, true); 17249ca874cfSHans Petter Selasky if (__predict_false(pa == NULL)) 17259ca874cfSHans Petter Selasky return (TCP_LRO_NOT_SUPPORTED); 17269ca874cfSHans Petter Selasky 17279ca874cfSHans Petter Selasky /* We don't expect any padding. */ 17289ca874cfSHans Petter Selasky error = tcp_lro_trim_mbuf_chain(m, pa); 17299ca874cfSHans Petter Selasky if (__predict_false(error != 0)) 17309ca874cfSHans Petter Selasky return (error); 17319ca874cfSHans Petter Selasky 17329ca874cfSHans Petter Selasky #ifdef INET 17339ca874cfSHans Petter Selasky switch (pa->data.lro_type) { 17349ca874cfSHans Petter Selasky case LRO_TYPE_IPV4_TCP: 17359ca874cfSHans Petter Selasky error = tcp_lro_rx_ipv4(lc, m, pa->ip4); 17369ca874cfSHans Petter Selasky if (__predict_false(error != 0)) 17379ca874cfSHans Petter Selasky return (error); 17389ca874cfSHans Petter Selasky break; 17399ca874cfSHans Petter Selasky default: 17409ca874cfSHans Petter Selasky break; 17419ca874cfSHans Petter Selasky } 17429ca874cfSHans Petter Selasky #endif 17439ca874cfSHans Petter Selasky /* If no hardware or arrival stamp on the packet add timestamp */ 1744e57b2d0eSRandall Stewart if ((m->m_flags & (M_TSTMP_LRO | M_TSTMP)) == 0) { 1745*b45daaeaSRandall Stewart m->m_pkthdr.rcv_tstmp = bintime2ns(&lc->lro_last_queue_time); 1746e57b2d0eSRandall Stewart m->m_flags |= M_TSTMP_LRO; 1747e57b2d0eSRandall Stewart } 17486c5087a8SJack F Vogel 17499ca874cfSHans Petter Selasky /* Get pointer to TCP header. */ 17509ca874cfSHans Petter Selasky th = pa->tcp; 17519ca874cfSHans Petter Selasky 17529ca874cfSHans Petter Selasky /* Don't process SYN packets. */ 17539ca874cfSHans Petter Selasky if (__predict_false(th->th_flags & TH_SYN)) 175462b5b6ecSBjoern A. Zeeb return (TCP_LRO_CANNOT); 175562b5b6ecSBjoern A. Zeeb 17569ca874cfSHans Petter Selasky /* Get total TCP header length and compute payload length. */ 17579ca874cfSHans Petter Selasky tcp_opt_len = (th->th_off << 2); 17589ca874cfSHans Petter Selasky tcp_data_len = m->m_pkthdr.len - ((uint8_t *)th - 17599ca874cfSHans Petter Selasky (uint8_t *)m->m_data) - tcp_opt_len; 17609ca874cfSHans Petter Selasky tcp_opt_len -= sizeof(*th); 17619ca874cfSHans Petter Selasky 17629ca874cfSHans Petter Selasky /* Don't process invalid TCP headers. */ 17639ca874cfSHans Petter Selasky if (__predict_false(tcp_opt_len < 0 || tcp_data_len < 0)) 176462b5b6ecSBjoern A. Zeeb return (TCP_LRO_CANNOT); 17659ca874cfSHans Petter Selasky 17669ca874cfSHans Petter Selasky /* Compute TCP data only checksum. */ 17679ca874cfSHans Petter Selasky if (tcp_data_len == 0) 17689ca874cfSHans Petter Selasky tcp_data_sum = 0; /* no data, no checksum */ 17699ca874cfSHans Petter Selasky else if (__predict_false(csum != 0)) 17709ca874cfSHans Petter Selasky tcp_data_sum = tcp_lro_rx_csum_data(pa, ~csum); 17719ca874cfSHans Petter Selasky else 17729ca874cfSHans Petter Selasky tcp_data_sum = tcp_lro_rx_csum_data(pa, ~th->th_sum); 17739ca874cfSHans Petter Selasky 17749ca874cfSHans Petter Selasky /* Save TCP info in mbuf. */ 17759ca874cfSHans Petter Selasky m->m_nextpkt = NULL; 17769ca874cfSHans Petter Selasky m->m_pkthdr.rcvif = lc->ifp; 17779ca874cfSHans Petter Selasky m->m_pkthdr.lro_tcp_d_csum = tcp_data_sum; 17789ca874cfSHans Petter Selasky m->m_pkthdr.lro_tcp_d_len = tcp_data_len; 17799ca874cfSHans Petter Selasky m->m_pkthdr.lro_tcp_h_off = ((uint8_t *)th - (uint8_t *)m->m_data); 17809ca874cfSHans Petter Selasky m->m_pkthdr.lro_nsegs = 1; 17819ca874cfSHans Petter Selasky 17829ca874cfSHans Petter Selasky /* Get hash bucket. */ 178305cde7efSSepherosa Ziehau if (!use_hash) { 178405cde7efSSepherosa Ziehau bucket = &lc->lro_hash[0]; 178505cde7efSSepherosa Ziehau } else { 17869ca874cfSHans Petter Selasky bucket = tcp_lro_rx_get_bucket(lc, m, pa); 178705cde7efSSepherosa Ziehau } 178805cde7efSSepherosa Ziehau 178962b5b6ecSBjoern A. Zeeb /* Try to find a matching previous segment. */ 179005cde7efSSepherosa Ziehau LIST_FOREACH(le, bucket, hash_next) { 17919ca874cfSHans Petter Selasky /* Compare addresses and ports. */ 17929ca874cfSHans Petter Selasky if (lro_address_compare(&po.data, &le->outer.data) == false || 17939ca874cfSHans Petter Selasky lro_address_compare(&pi.data, &le->inner.data) == false) 179462b5b6ecSBjoern A. Zeeb continue; 17959ca874cfSHans Petter Selasky 17969ca874cfSHans Petter Selasky /* Check if no data and old ACK. */ 17979ca874cfSHans Petter Selasky if (tcp_data_len == 0 && 17989ca874cfSHans Petter Selasky SEQ_LT(ntohl(th->th_ack), ntohl(le->ack_seq))) { 1799d7fb35d1SSean Bruno m_freem(m); 1800d7fb35d1SSean Bruno return (0); 1801d7fb35d1SSean Bruno } 180269a34e8dSRandall Stewart 18039ca874cfSHans Petter Selasky /* Mark "m" in the last spot. */ 1804e57b2d0eSRandall Stewart le->m_last_mbuf->m_nextpkt = m; 18059ca874cfSHans Petter Selasky /* Now set the tail to "m". */ 1806e57b2d0eSRandall Stewart le->m_last_mbuf = m; 180762b5b6ecSBjoern A. Zeeb return (0); 18086c5087a8SJack F Vogel } 18099ca874cfSHans Petter Selasky 181062b5b6ecSBjoern A. Zeeb /* Try to find an empty slot. */ 18111ea44822SSepherosa Ziehau if (LIST_EMPTY(&lc->lro_free)) 1812489f0c3cSSepherosa Ziehau return (TCP_LRO_NO_ENTRIES); 181362b5b6ecSBjoern A. Zeeb 181462b5b6ecSBjoern A. Zeeb /* Start a new segment chain. */ 18151ea44822SSepherosa Ziehau le = LIST_FIRST(&lc->lro_free); 18161ea44822SSepherosa Ziehau LIST_REMOVE(le, next); 181705cde7efSSepherosa Ziehau tcp_lro_active_insert(lc, bucket, le); 181862b5b6ecSBjoern A. Zeeb 18199ca874cfSHans Petter Selasky /* Make sure the headers are set. */ 18209ca874cfSHans Petter Selasky le->inner = pi; 18219ca874cfSHans Petter Selasky le->outer = po; 182262b5b6ecSBjoern A. Zeeb 18239ca874cfSHans Petter Selasky /* Store time this entry was allocated. */ 18249ca874cfSHans Petter Selasky le->alloc_time = lc->lro_last_queue_time; 182569a34e8dSRandall Stewart 18269ca874cfSHans Petter Selasky tcp_set_entry_to_mbuf(lc, le, m, th); 182769a34e8dSRandall Stewart 18289ca874cfSHans Petter Selasky /* Now set the tail to "m". */ 1829e57b2d0eSRandall Stewart le->m_last_mbuf = m; 18309ca874cfSHans Petter Selasky 183162b5b6ecSBjoern A. Zeeb return (0); 183262b5b6ecSBjoern A. Zeeb } 183362b5b6ecSBjoern A. Zeeb 183405cde7efSSepherosa Ziehau int 183505cde7efSSepherosa Ziehau tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum) 183605cde7efSSepherosa Ziehau { 18379ca874cfSHans Petter Selasky int error; 183805cde7efSSepherosa Ziehau 18399ca874cfSHans Petter Selasky /* get current time */ 1840*b45daaeaSRandall Stewart binuptime(&lc->lro_last_queue_time); 18419ca874cfSHans Petter Selasky 18429ca874cfSHans Petter Selasky CURVNET_SET(lc->ifp->if_vnet); 18439ca874cfSHans Petter Selasky error = tcp_lro_rx_common(lc, m, csum, true); 18449ca874cfSHans Petter Selasky CURVNET_RESTORE(); 18459ca874cfSHans Petter Selasky 18469ca874cfSHans Petter Selasky return (error); 184705cde7efSSepherosa Ziehau } 184805cde7efSSepherosa Ziehau 1849e936121dSHans Petter Selasky void 1850e936121dSHans Petter Selasky tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb) 1851e936121dSHans Petter Selasky { 1852e936121dSHans Petter Selasky /* sanity checks */ 1853e936121dSHans Petter Selasky if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL || 1854e936121dSHans Petter Selasky lc->lro_mbuf_max == 0)) { 1855e936121dSHans Petter Selasky /* packet drop */ 1856e936121dSHans Petter Selasky m_freem(mb); 1857e936121dSHans Petter Selasky return; 1858e936121dSHans Petter Selasky } 1859e936121dSHans Petter Selasky 1860e936121dSHans Petter Selasky /* check if packet is not LRO capable */ 1861e936121dSHans Petter Selasky if (__predict_false(mb->m_pkthdr.csum_flags == 0 || 1862e936121dSHans Petter Selasky (lc->ifp->if_capenable & IFCAP_LRO) == 0)) { 1863e936121dSHans Petter Selasky /* input packet to network layer */ 1864e936121dSHans Petter Selasky (*lc->ifp->if_input) (lc->ifp, mb); 1865e936121dSHans Petter Selasky return; 1866e936121dSHans Petter Selasky } 1867e936121dSHans Petter Selasky 1868fc271df3SHans Petter Selasky /* create sequence number */ 1869fc271df3SHans Petter Selasky lc->lro_mbuf_data[lc->lro_mbuf_count].seq = 1870fc271df3SHans Petter Selasky (((uint64_t)M_HASHTYPE_GET(mb)) << 56) | 1871fc271df3SHans Petter Selasky (((uint64_t)mb->m_pkthdr.flowid) << 24) | 1872fc271df3SHans Petter Selasky ((uint64_t)lc->lro_mbuf_count); 1873e936121dSHans Petter Selasky 1874e936121dSHans Petter Selasky /* enter mbuf */ 1875f8acc03eSNavdeep Parhar lc->lro_mbuf_data[lc->lro_mbuf_count].mb = mb; 1876f8acc03eSNavdeep Parhar 1877f8acc03eSNavdeep Parhar /* flush if array is full */ 1878f8acc03eSNavdeep Parhar if (__predict_false(++lc->lro_mbuf_count == lc->lro_mbuf_max)) 1879f8acc03eSNavdeep Parhar tcp_lro_flush_all(lc); 1880e936121dSHans Petter Selasky } 1881e936121dSHans Petter Selasky 188262b5b6ecSBjoern A. Zeeb /* end */ 1883