1 /*- 2 * Copyright (c) 2016-2018 Netflix, Inc. 3 * Copyright (c) 2016-2021 Mellanox Technologies. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 #include <sys/cdefs.h> 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/malloc.h> 35 #include <sys/mbuf.h> 36 #include <sys/socket.h> 37 #include <sys/socketvar.h> 38 #include <sys/sysctl.h> 39 40 #include <net/if.h> 41 #include <net/if_var.h> 42 #include <net/if_private.h> 43 #include <net/ethernet.h> 44 #include <net/bpf.h> 45 #include <net/vnet.h> 46 #include <net/if_dl.h> 47 #include <net/if_media.h> 48 #include <net/if_types.h> 49 #include <net/infiniband.h> 50 #include <net/if_lagg.h> 51 #include <net/pfil.h> 52 53 #include <netinet/in.h> 54 #include <netinet/in_kdtrace.h> 55 #include <netinet/ip6.h> 56 #include <netinet/ip.h> 57 #include <netinet/ip_var.h> 58 #include <netinet/in_pcb.h> 59 #include <netinet6/in6_pcb.h> 60 #include <netinet6/ip6_var.h> 61 #include <netinet/tcp.h> 62 #include <netinet/tcp_lro.h> 63 #include <netinet/tcp_var.h> 64 #include <netinet/tcp_hpts.h> 65 #ifdef TCP_BLACKBOX 66 #include <netinet/tcp_log_buf.h> 67 #endif 68 69 static void 70 build_ack_entry(struct tcp_ackent *ae, struct tcphdr *th, struct mbuf *m, 71 uint32_t *ts_ptr, uint16_t iptos) 72 { 73 /* 74 * Given a TCP ACK, summarize it down into the small TCP ACK 75 * entry. 76 */ 77 ae->timestamp = m->m_pkthdr.rcv_tstmp; 78 ae->flags = 0; 79 if (m->m_flags & M_TSTMP_LRO) 80 ae->flags |= TSTMP_LRO; 81 else if (m->m_flags & M_TSTMP) 82 ae->flags |= TSTMP_HDWR; 83 ae->seq = th->th_seq; 84 ae->ack = th->th_ack; 85 ae->flags |= tcp_get_flags(th); 86 if (ts_ptr != NULL) { 87 ae->ts_value = ntohl(ts_ptr[1]); 88 ae->ts_echo = ntohl(ts_ptr[2]); 89 ae->flags |= HAS_TSTMP; 90 } 91 ae->win = th->th_win; 92 ae->codepoint = iptos; 93 } 94 95 static inline bool 96 tcp_lro_ack_valid(struct mbuf *m, struct tcphdr *th, uint32_t **ppts, bool *other_opts) 97 { 98 /* 99 * This function returns two bits of valuable information. 100 * a) Is what is present capable of being ack-compressed, 101 * we can ack-compress if there is no options or just 102 * a timestamp option, and of course the th_flags must 103 * be correct as well. 104 * b) Our other options present such as SACK. This is 105 * used to determine if we want to wakeup or not. 106 */ 107 bool ret = true; 108 109 switch (th->th_off << 2) { 110 case (sizeof(*th) + TCPOLEN_TSTAMP_APPA): 111 *ppts = (uint32_t *)(th + 1); 112 /* Check if we have only one timestamp option. */ 113 if (**ppts == TCP_LRO_TS_OPTION) 114 *other_opts = false; 115 else { 116 *other_opts = true; 117 ret = false; 118 } 119 break; 120 case (sizeof(*th)): 121 /* No options. */ 122 *ppts = NULL; 123 *other_opts = false; 124 break; 125 default: 126 *ppts = NULL; 127 *other_opts = true; 128 ret = false; 129 break; 130 } 131 /* For ACKCMP we only accept ACK, PUSH, ECE and CWR. */ 132 if ((tcp_get_flags(th) & ~(TH_ACK | TH_PUSH | TH_ECE | TH_CWR)) != 0) 133 ret = false; 134 /* If it has data on it we cannot compress it */ 135 if (m->m_pkthdr.lro_tcp_d_len) 136 ret = false; 137 138 /* ACK flag must be set. */ 139 if (!(tcp_get_flags(th) & TH_ACK)) 140 ret = false; 141 return (ret); 142 } 143 144 static bool 145 tcp_lro_check_wake_status(struct tcpcb *tp) 146 { 147 148 if (tp->t_fb->tfb_early_wake_check != NULL) 149 return ((tp->t_fb->tfb_early_wake_check)(tp)); 150 return (false); 151 } 152 153 #ifdef TCP_BLACKBOX 154 static void 155 tcp_lro_log(struct tcpcb *tp, const struct lro_ctrl *lc, 156 const struct lro_entry *le, const struct mbuf *m, 157 int frm, int32_t tcp_data_len, uint32_t th_seq, 158 uint32_t th_ack, uint16_t th_win) 159 { 160 if (tcp_bblogging_on(tp)) { 161 union tcp_log_stackspecific log; 162 struct timeval tv, btv; 163 uint32_t cts; 164 165 cts = tcp_get_usecs(&tv); 166 memset(&log, 0, sizeof(union tcp_log_stackspecific)); 167 log.u_bbr.flex8 = frm; 168 log.u_bbr.flex1 = tcp_data_len; 169 if (m) 170 log.u_bbr.flex2 = m->m_pkthdr.len; 171 else 172 log.u_bbr.flex2 = 0; 173 if (le->m_head) { 174 log.u_bbr.flex3 = le->m_head->m_pkthdr.lro_nsegs; 175 log.u_bbr.flex4 = le->m_head->m_pkthdr.lro_tcp_d_len; 176 log.u_bbr.flex5 = le->m_head->m_pkthdr.len; 177 log.u_bbr.delRate = le->m_head->m_flags; 178 log.u_bbr.rttProp = le->m_head->m_pkthdr.rcv_tstmp; 179 } 180 log.u_bbr.inflight = th_seq; 181 log.u_bbr.delivered = th_ack; 182 log.u_bbr.timeStamp = cts; 183 log.u_bbr.epoch = le->next_seq; 184 log.u_bbr.lt_epoch = le->ack_seq; 185 log.u_bbr.pacing_gain = th_win; 186 log.u_bbr.cwnd_gain = le->window; 187 log.u_bbr.lost = curcpu; 188 log.u_bbr.cur_del_rate = (uintptr_t)m; 189 log.u_bbr.bw_inuse = (uintptr_t)le->m_head; 190 bintime2timeval(&lc->lro_last_queue_time, &btv); 191 log.u_bbr.flex6 = tcp_tv_to_usectick(&btv); 192 log.u_bbr.flex7 = le->compressed; 193 log.u_bbr.pacing_gain = le->uncompressed; 194 if (in_epoch(net_epoch_preempt)) 195 log.u_bbr.inhpts = 1; 196 else 197 log.u_bbr.inhpts = 0; 198 TCP_LOG_EVENTP(tp, NULL, &tptosocket(tp)->so_rcv, 199 &tptosocket(tp)->so_snd, 200 TCP_LOG_LRO, 0, 0, &log, false, &tv); 201 } 202 } 203 #endif 204 205 static struct mbuf * 206 tcp_lro_get_last_if_ackcmp(struct lro_ctrl *lc, struct lro_entry *le, 207 struct tcpcb *tp, int32_t *new_m, bool can_append_old_cmp) 208 { 209 struct mbuf *m; 210 211 /* Look at the last mbuf if any in queue */ 212 if (can_append_old_cmp) { 213 m = STAILQ_LAST(&tp->t_inqueue, mbuf, m_stailqpkt); 214 if (m != NULL && (m->m_flags & M_ACKCMP) != 0) { 215 if (M_TRAILINGSPACE(m) >= sizeof(struct tcp_ackent)) { 216 #ifdef TCP_BLACKBOX 217 tcp_lro_log(tp, lc, le, NULL, 23, 0, 0, 0, 0); 218 #endif 219 *new_m = 0; 220 counter_u64_add(tcp_extra_mbuf, 1); 221 return (m); 222 } else { 223 /* Mark we ran out of space */ 224 tp->t_flags2 |= TF2_MBUF_L_ACKS; 225 } 226 } 227 } 228 /* Decide mbuf size. */ 229 #ifdef TCP_BLACKBOX 230 tcp_lro_log(tp, lc, le, NULL, 21, 0, 0, 0, 0); 231 #endif 232 if (tp->t_flags2 & TF2_MBUF_L_ACKS) 233 m = m_getcl(M_NOWAIT, MT_DATA, M_ACKCMP | M_PKTHDR); 234 else 235 m = m_gethdr(M_NOWAIT, MT_DATA); 236 237 if (__predict_false(m == NULL)) { 238 counter_u64_add(tcp_would_have_but, 1); 239 return (NULL); 240 } 241 counter_u64_add(tcp_comp_total, 1); 242 m->m_pkthdr.rcvif = lc->ifp; 243 m->m_flags |= M_ACKCMP; 244 *new_m = 1; 245 return (m); 246 } 247 248 /* 249 * Do BPF tap for either ACK_CMP packets or MBUF QUEUE type packets 250 * and strip all, but the IPv4/IPv6 header. 251 */ 252 static bool 253 do_bpf_strip_and_compress(struct tcpcb *tp, struct lro_ctrl *lc, 254 struct lro_entry *le, struct mbuf **pp, struct mbuf **cmp, 255 struct mbuf **mv_to, bool *should_wake, bool bpf_req, bool lagg_bpf_req, 256 struct ifnet *lagg_ifp, bool can_append_old_cmp) 257 { 258 union { 259 void *ptr; 260 struct ip *ip4; 261 struct ip6_hdr *ip6; 262 } l3; 263 struct mbuf *m; 264 struct mbuf *nm; 265 struct tcphdr *th; 266 struct tcp_ackent *ack_ent; 267 uint32_t *ts_ptr; 268 int32_t n_mbuf; 269 bool other_opts, can_compress; 270 uint8_t lro_type; 271 uint16_t iptos; 272 int tcp_hdr_offset; 273 int idx; 274 275 /* Get current mbuf. */ 276 m = *pp; 277 278 /* Let the BPF see the packet */ 279 if (__predict_false(bpf_req)) 280 ETHER_BPF_MTAP(lc->ifp, m); 281 282 if (__predict_false(lagg_bpf_req)) 283 ETHER_BPF_MTAP(lagg_ifp, m); 284 285 tcp_hdr_offset = m->m_pkthdr.lro_tcp_h_off; 286 lro_type = le->inner.data.lro_type; 287 switch (lro_type) { 288 case LRO_TYPE_NONE: 289 lro_type = le->outer.data.lro_type; 290 switch (lro_type) { 291 case LRO_TYPE_IPV4_TCP: 292 tcp_hdr_offset -= sizeof(*le->outer.ip4); 293 m->m_pkthdr.lro_etype = ETHERTYPE_IP; 294 IP_PROBE(receive, NULL, NULL, le->outer.ip4, lc->ifp, 295 le->outer.ip4, NULL); 296 break; 297 case LRO_TYPE_IPV6_TCP: 298 tcp_hdr_offset -= sizeof(*le->outer.ip6); 299 m->m_pkthdr.lro_etype = ETHERTYPE_IPV6; 300 IP_PROBE(receive, NULL, NULL, le->outer.ip6, lc->ifp, 301 NULL, le->outer.ip6); 302 break; 303 default: 304 goto compressed; 305 } 306 break; 307 case LRO_TYPE_IPV4_TCP: 308 switch (le->outer.data.lro_type) { 309 case LRO_TYPE_IPV4_UDP: 310 IP_PROBE(receive, NULL, NULL, le->outer.ip4, lc->ifp, 311 le->outer.ip4, NULL); 312 UDP_PROBE(receive, NULL, NULL, le->outer.ip4, NULL, 313 le->outer.udp); 314 break; 315 case LRO_TYPE_IPV6_UDP: 316 IP_PROBE(receive, NULL, NULL, le->outer.ip6, lc->ifp, 317 NULL, le->outer.ip6); 318 UDP_PROBE(receive, NULL, NULL, le->outer.ip6, NULL, 319 le->outer.udp); 320 break; 321 default: 322 __assert_unreachable(); 323 break; 324 } 325 tcp_hdr_offset -= sizeof(*le->outer.ip4); 326 m->m_pkthdr.lro_etype = ETHERTYPE_IP; 327 IP_PROBE(receive, NULL, NULL, le->inner.ip4, NULL, 328 le->inner.ip4, NULL); 329 break; 330 case LRO_TYPE_IPV6_TCP: 331 switch (le->outer.data.lro_type) { 332 case LRO_TYPE_IPV4_UDP: 333 IP_PROBE(receive, NULL, NULL, le->outer.ip4, lc->ifp, 334 le->outer.ip4, NULL); 335 UDP_PROBE(receive, NULL, NULL, le->outer.ip4, NULL, 336 le->outer.udp); 337 break; 338 case LRO_TYPE_IPV6_UDP: 339 IP_PROBE(receive, NULL, NULL, le->outer.ip6, lc->ifp, 340 NULL, le->outer.ip6); 341 UDP_PROBE(receive, NULL, NULL, le->outer.ip6, NULL, 342 le->outer.udp); 343 break; 344 default: 345 __assert_unreachable(); 346 break; 347 } 348 tcp_hdr_offset -= sizeof(*le->outer.ip6); 349 m->m_pkthdr.lro_etype = ETHERTYPE_IPV6; 350 IP_PROBE(receive, NULL, NULL, le->inner.ip6, NULL, NULL, 351 le->inner.ip6); 352 break; 353 default: 354 goto compressed; 355 } 356 357 MPASS(tcp_hdr_offset >= 0); 358 359 m_adj(m, tcp_hdr_offset); 360 m->m_flags |= M_LRO_EHDRSTRP; 361 m->m_flags &= ~M_ACKCMP; 362 m->m_pkthdr.lro_tcp_h_off -= tcp_hdr_offset; 363 364 th = tcp_lro_get_th(m); 365 366 th->th_sum = 0; /* TCP checksum is valid. */ 367 tcp_fields_to_host(th); 368 TCP_PROBE5(receive, NULL, tp, m, tp, th); 369 370 /* Check if ACK can be compressed */ 371 can_compress = tcp_lro_ack_valid(m, th, &ts_ptr, &other_opts); 372 373 /* Now lets look at the should wake states */ 374 if ((other_opts == true) && 375 ((tp->t_flags2 & TF2_DONT_SACK_QUEUE) == 0)) { 376 /* 377 * If there are other options (SACK?) and the 378 * tcp endpoint has not expressly told us it does 379 * not care about SACKS, then we should wake up. 380 */ 381 *should_wake = true; 382 } else if (*should_wake == false) { 383 /* Wakeup override check if we are false here */ 384 *should_wake = tcp_lro_check_wake_status(tp); 385 } 386 /* Is the ack compressable? */ 387 if (can_compress == false) 388 goto done; 389 /* Does the TCP endpoint support ACK compression? */ 390 if ((tp->t_flags2 & TF2_MBUF_ACKCMP) == 0) 391 goto done; 392 393 /* Lets get the TOS/traffic class field */ 394 l3.ptr = mtod(m, void *); 395 switch (lro_type) { 396 case LRO_TYPE_IPV4_TCP: 397 iptos = l3.ip4->ip_tos; 398 break; 399 case LRO_TYPE_IPV6_TCP: 400 iptos = IPV6_TRAFFIC_CLASS(l3.ip6); 401 break; 402 default: 403 iptos = 0; /* Keep compiler happy. */ 404 break; 405 } 406 /* Now lets get space if we don't have some already */ 407 if (*cmp == NULL) { 408 new_one: 409 nm = tcp_lro_get_last_if_ackcmp(lc, le, tp, &n_mbuf, 410 can_append_old_cmp); 411 if (__predict_false(nm == NULL)) 412 goto done; 413 *cmp = nm; 414 if (n_mbuf) { 415 /* 416 * Link in the new cmp ack to our in-order place, 417 * first set our cmp ack's next to where we are. 418 */ 419 nm->m_nextpkt = m; 420 (*pp) = nm; 421 /* 422 * Set it up so mv_to is advanced to our 423 * compressed ack. This way the caller can 424 * advance pp to the right place. 425 */ 426 *mv_to = nm; 427 /* 428 * Advance it here locally as well. 429 */ 430 pp = &nm->m_nextpkt; 431 } 432 } else { 433 /* We have one already we are working on */ 434 nm = *cmp; 435 if (M_TRAILINGSPACE(nm) < sizeof(struct tcp_ackent)) { 436 /* We ran out of space */ 437 tp->t_flags2 |= TF2_MBUF_L_ACKS; 438 goto new_one; 439 } 440 } 441 MPASS(M_TRAILINGSPACE(nm) >= sizeof(struct tcp_ackent)); 442 counter_u64_add(tcp_inp_lro_compressed, 1); 443 le->compressed++; 444 /* We can add in to the one on the tail */ 445 ack_ent = mtod(nm, struct tcp_ackent *); 446 idx = (nm->m_len / sizeof(struct tcp_ackent)); 447 build_ack_entry(&ack_ent[idx], th, m, ts_ptr, iptos); 448 449 /* Bump the size of both pkt-hdr and len */ 450 nm->m_len += sizeof(struct tcp_ackent); 451 nm->m_pkthdr.len += sizeof(struct tcp_ackent); 452 compressed: 453 /* Advance to next mbuf before freeing. */ 454 *pp = m->m_nextpkt; 455 m->m_nextpkt = NULL; 456 m_freem(m); 457 return (true); 458 done: 459 counter_u64_add(tcp_uncomp_total, 1); 460 le->uncompressed++; 461 return (false); 462 } 463 464 static void 465 tcp_queue_pkts(struct tcpcb *tp, struct lro_entry *le) 466 { 467 468 INP_WLOCK_ASSERT(tptoinpcb(tp)); 469 470 STAILQ_HEAD(, mbuf) q = { le->m_head, 471 &STAILQ_NEXT(le->m_last_mbuf, m_stailqpkt) }; 472 STAILQ_CONCAT(&tp->t_inqueue, &q); 473 le->m_head = NULL; 474 le->m_last_mbuf = NULL; 475 } 476 477 static struct tcpcb * 478 tcp_lro_lookup(struct ifnet *ifp, struct lro_parser *pa) 479 { 480 struct inpcb *inp; 481 482 CURVNET_ASSERT_SET(); 483 switch (pa->data.lro_type) { 484 #ifdef INET6 485 case LRO_TYPE_IPV6_TCP: 486 inp = in6_pcblookup(&V_tcbinfo, 487 &pa->data.s_addr.v6, 488 pa->data.s_port, 489 &pa->data.d_addr.v6, 490 pa->data.d_port, 491 INPLOOKUP_WLOCKPCB, 492 ifp); 493 break; 494 #endif 495 #ifdef INET 496 case LRO_TYPE_IPV4_TCP: 497 inp = in_pcblookup(&V_tcbinfo, 498 pa->data.s_addr.v4, 499 pa->data.s_port, 500 pa->data.d_addr.v4, 501 pa->data.d_port, 502 INPLOOKUP_WLOCKPCB, 503 ifp); 504 break; 505 #endif 506 default: 507 return (NULL); 508 } 509 510 return (intotcpcb(inp)); 511 } 512 513 static int 514 _tcp_lro_flush_tcphpts(struct lro_ctrl *lc, struct lro_entry *le) 515 { 516 struct tcpcb *tp; 517 struct mbuf **pp, *cmp, *mv_to; 518 struct ifnet *lagg_ifp; 519 bool bpf_req, lagg_bpf_req, should_wake, can_append_old_cmp; 520 521 /* Check if packet doesn't belongs to our network interface. */ 522 if ((tcplro_stacks_wanting_mbufq == 0) || 523 (le->outer.data.vlan_id != 0) || 524 (le->inner.data.lro_type != LRO_TYPE_NONE)) 525 return (TCP_LRO_CANNOT); 526 527 #ifdef INET6 528 /* 529 * Be proactive about unspecified IPv6 address in source. As 530 * we use all-zero to indicate unbounded/unconnected pcb, 531 * unspecified IPv6 address can be used to confuse us. 532 * 533 * Note that packets with unspecified IPv6 destination is 534 * already dropped in ip6_input. 535 */ 536 if (__predict_false(le->outer.data.lro_type == LRO_TYPE_IPV6_TCP && 537 IN6_IS_ADDR_UNSPECIFIED(&le->outer.data.s_addr.v6))) 538 return (TCP_LRO_CANNOT); 539 540 if (__predict_false(le->inner.data.lro_type == LRO_TYPE_IPV6_TCP && 541 IN6_IS_ADDR_UNSPECIFIED(&le->inner.data.s_addr.v6))) 542 return (TCP_LRO_CANNOT); 543 #endif 544 545 CURVNET_SET(lc->ifp->if_vnet); 546 /* 547 * Ensure that there are no packet filter hooks which would normally 548 * being triggered in ether_demux(), ip_input(), or ip6_input(). 549 */ 550 if ( 551 #ifdef INET 552 PFIL_HOOKED_IN(V_inet_pfil_head) || 553 #endif 554 #ifdef INET6 555 PFIL_HOOKED_IN(V_inet6_pfil_head) || 556 #endif 557 PFIL_HOOKED_IN(V_link_pfil_head)) { 558 CURVNET_RESTORE(); 559 return (TCP_LRO_CANNOT); 560 } 561 562 /* Lookup inp, if any. Returns locked TCP inpcb. */ 563 tp = tcp_lro_lookup(lc->ifp, 564 (le->inner.data.lro_type == LRO_TYPE_NONE) ? &le->outer : &le->inner); 565 CURVNET_RESTORE(); 566 if (tp == NULL) 567 return (TCP_LRO_CANNOT); 568 569 counter_u64_add(tcp_inp_lro_locks_taken, 1); 570 571 /* Check if the inp is dead, Jim. */ 572 if (tp->t_state == TCPS_TIME_WAIT) { 573 INP_WUNLOCK(tptoinpcb(tp)); 574 return (TCP_LRO_CANNOT); 575 } 576 if (tp->t_lro_cpu == HPTS_CPU_NONE && lc->lro_cpu_is_set == 1) 577 tp->t_lro_cpu = lc->lro_last_cpu; 578 /* Check if the transport doesn't support the needed optimizations. */ 579 if ((tp->t_flags2 & (TF2_SUPPORTS_MBUFQ | TF2_MBUF_ACKCMP)) == 0) { 580 INP_WUNLOCK(tptoinpcb(tp)); 581 return (TCP_LRO_CANNOT); 582 } 583 584 if (tp->t_flags2 & TF2_MBUF_QUEUE_READY) 585 should_wake = false; 586 else 587 should_wake = true; 588 /* Check if packets should be tapped to BPF. */ 589 bpf_req = bpf_peers_present(lc->ifp->if_bpf); 590 lagg_bpf_req = false; 591 lagg_ifp = NULL; 592 if (lc->ifp->if_type == IFT_IEEE8023ADLAG || 593 lc->ifp->if_type == IFT_INFINIBANDLAG) { 594 struct lagg_port *lp = lc->ifp->if_lagg; 595 struct lagg_softc *sc = lp->lp_softc; 596 597 lagg_ifp = sc->sc_ifp; 598 if (lagg_ifp != NULL) 599 lagg_bpf_req = bpf_peers_present(lagg_ifp->if_bpf); 600 } 601 602 /* Strip and compress all the incoming packets. */ 603 can_append_old_cmp = true; 604 cmp = NULL; 605 for (pp = &le->m_head; *pp != NULL; ) { 606 mv_to = NULL; 607 if (do_bpf_strip_and_compress(tp, lc, le, pp, &cmp, &mv_to, 608 &should_wake, bpf_req, lagg_bpf_req, lagg_ifp, 609 can_append_old_cmp) == false) { 610 /* Advance to next mbuf. */ 611 pp = &(*pp)->m_nextpkt; 612 /* 613 * Once we have appended we can't look in the pending 614 * inbound packets for a compressed ack to append to. 615 */ 616 can_append_old_cmp = false; 617 /* 618 * Once we append we also need to stop adding to any 619 * compressed ack we were remembering. A new cmp 620 * ack will be required. 621 */ 622 cmp = NULL; 623 #ifdef TCP_BLACKBOX 624 tcp_lro_log(tp, lc, le, NULL, 25, 0, 0, 0, 0); 625 #endif 626 } else if (mv_to != NULL) { 627 /* We are asked to move pp up */ 628 pp = &mv_to->m_nextpkt; 629 #ifdef TCP_BLACKBOX 630 tcp_lro_log(tp, lc, le, NULL, 24, 0, 0, 0, 0); 631 } else 632 tcp_lro_log(tp, lc, le, NULL, 26, 0, 0, 0, 0); 633 #else 634 } 635 #endif 636 } 637 /* Update "m_last_mbuf", if any. */ 638 if (pp == &le->m_head) 639 le->m_last_mbuf = *pp; 640 else 641 le->m_last_mbuf = __containerof(pp, struct mbuf, m_nextpkt); 642 643 /* Check if any data mbufs left. */ 644 if (le->m_head != NULL) { 645 counter_u64_add(tcp_inp_lro_direct_queue, 1); 646 #ifdef TCP_BLACKBOX 647 tcp_lro_log(tp, lc, le, NULL, 22, 1, tp->t_flags2, 0, 1); 648 #endif 649 tcp_queue_pkts(tp, le); 650 } 651 if (should_wake) { 652 /* Wakeup */ 653 counter_u64_add(tcp_inp_lro_wokeup_queue, 1); 654 if ((*tp->t_fb->tfb_do_queued_segments)(tp, 0)) 655 /* TCP cb gone and unlocked. */ 656 return (0); 657 } 658 INP_WUNLOCK(tptoinpcb(tp)); 659 660 return (0); /* Success. */ 661 } 662 663 void 664 tcp_lro_hpts_init(void) 665 { 666 tcp_lro_flush_tcphpts = _tcp_lro_flush_tcphpts; 667 } 668 669 void 670 tcp_lro_hpts_uninit(void) 671 { 672 atomic_store_ptr(&tcp_lro_flush_tcphpts, NULL); 673 } 674