xref: /freebsd/sys/netinet/tcp_usrreq.c (revision 6549718b70f0e660a15685369afb4f9caf2215ce)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1988, 1993
5  *	The Regents of the University of California.
6  * Copyright (c) 2006-2007 Robert N. M. Watson
7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
8  * All rights reserved.
9  *
10  * Portions of this software were developed by Robert N. M. Watson under
11  * contract to Juniper Networks, Inc.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  */
37 
38 #include <sys/cdefs.h>
39 #include "opt_ddb.h"
40 #include "opt_inet.h"
41 #include "opt_inet6.h"
42 #include "opt_ipsec.h"
43 #include "opt_kern_tls.h"
44 
45 #include <sys/param.h>
46 #include <sys/systm.h>
47 #include <sys/arb.h>
48 #include <sys/limits.h>
49 #include <sys/malloc.h>
50 #include <sys/refcount.h>
51 #include <sys/kernel.h>
52 #include <sys/ktls.h>
53 #include <sys/qmath.h>
54 #include <sys/sysctl.h>
55 #include <sys/mbuf.h>
56 #ifdef INET6
57 #include <sys/domain.h>
58 #endif /* INET6 */
59 #include <sys/socket.h>
60 #include <sys/socketvar.h>
61 #include <sys/protosw.h>
62 #include <sys/proc.h>
63 #include <sys/jail.h>
64 #include <sys/stats.h>
65 
66 #ifdef DDB
67 #include <ddb/ddb.h>
68 #endif
69 
70 #include <net/if.h>
71 #include <net/if_var.h>
72 #include <net/route.h>
73 #include <net/vnet.h>
74 
75 #include <netinet/in.h>
76 #include <netinet/in_kdtrace.h>
77 #include <netinet/in_pcb.h>
78 #include <netinet/in_systm.h>
79 #include <netinet/in_var.h>
80 #include <netinet/ip.h>
81 #include <netinet/ip_var.h>
82 #ifdef INET6
83 #include <netinet/ip6.h>
84 #include <netinet6/in6_pcb.h>
85 #include <netinet6/ip6_var.h>
86 #include <netinet6/scope6_var.h>
87 #endif
88 #include <netinet/tcp.h>
89 #include <netinet/tcp_fsm.h>
90 #include <netinet/tcp_seq.h>
91 #include <netinet/tcp_timer.h>
92 #include <netinet/tcp_var.h>
93 #include <netinet/tcp_log_buf.h>
94 #include <netinet/tcpip.h>
95 #include <netinet/cc/cc.h>
96 #include <netinet/tcp_fastopen.h>
97 #include <netinet/tcp_hpts.h>
98 #ifdef TCPPCAP
99 #include <netinet/tcp_pcap.h>
100 #endif
101 #ifdef TCP_OFFLOAD
102 #include <netinet/tcp_offload.h>
103 #endif
104 #include <netipsec/ipsec_support.h>
105 
106 #include <vm/vm.h>
107 #include <vm/vm_param.h>
108 #include <vm/pmap.h>
109 #include <vm/vm_extern.h>
110 #include <vm/vm_map.h>
111 #include <vm/vm_page.h>
112 
113 /*
114  * TCP protocol interface to socket abstraction.
115  */
116 #ifdef INET
117 static int	tcp_connect(struct tcpcb *, struct sockaddr_in *,
118 		    struct thread *td);
119 #endif /* INET */
120 #ifdef INET6
121 static int	tcp6_connect(struct tcpcb *, struct sockaddr_in6 *,
122 		    struct thread *td);
123 #endif /* INET6 */
124 static void	tcp_disconnect(struct tcpcb *);
125 static void	tcp_usrclosed(struct tcpcb *);
126 static void	tcp_fill_info(const struct tcpcb *, struct tcp_info *);
127 
128 static int	tcp_pru_options_support(struct tcpcb *tp, int flags);
129 
130 static void
131 tcp_bblog_pru(struct tcpcb *tp, uint32_t pru, int error)
132 {
133 	struct tcp_log_buffer *lgb;
134 
135 	KASSERT(tp != NULL, ("tcp_bblog_pru: tp == NULL"));
136 	INP_WLOCK_ASSERT(tptoinpcb(tp));
137 	if (tcp_bblogging_on(tp)) {
138 		lgb = tcp_log_event(tp, NULL, NULL, NULL, TCP_LOG_PRU, error,
139 		    0, NULL, false, NULL, NULL, 0, NULL);
140 	} else {
141 		lgb = NULL;
142 	}
143 	if (lgb != NULL) {
144 		if (error >= 0) {
145 			lgb->tlb_errno = (uint32_t)error;
146 		}
147 		lgb->tlb_flex1 = pru;
148 	}
149 }
150 
151 /*
152  * TCP attaches to socket via pru_attach(), reserving space,
153  * and an internet control block.
154  */
155 static int
156 tcp_usr_attach(struct socket *so, int proto, struct thread *td)
157 {
158 	struct inpcb *inp;
159 	struct tcpcb *tp = NULL;
160 	int error;
161 
162 	inp = sotoinpcb(so);
163 	KASSERT(inp == NULL, ("tcp_usr_attach: inp != NULL"));
164 
165 	error = soreserve(so, V_tcp_sendspace, V_tcp_recvspace);
166 	if (error)
167 		goto out;
168 
169 	so->so_rcv.sb_flags |= SB_AUTOSIZE;
170 	so->so_snd.sb_flags |= SB_AUTOSIZE;
171 	error = in_pcballoc(so, &V_tcbinfo);
172 	if (error)
173 		goto out;
174 	inp = sotoinpcb(so);
175 	tp = tcp_newtcpcb(inp, NULL);
176 	if (tp == NULL) {
177 		error = ENOBUFS;
178 		in_pcbfree(inp);
179 		goto out;
180 	}
181 	tp->t_state = TCPS_CLOSED;
182 	tcp_bblog_pru(tp, PRU_ATTACH, error);
183 	INP_WUNLOCK(inp);
184 	TCPSTATES_INC(TCPS_CLOSED);
185 out:
186 	TCP_PROBE2(debug__user, tp, PRU_ATTACH);
187 	return (error);
188 }
189 
190 /*
191  * tcp_usr_detach is called when the socket layer loses its final reference
192  * to the socket, be it a file descriptor reference, a reference from TCP,
193  * etc.  At this point, there is only one case in which we will keep around
194  * inpcb state: time wait.
195  */
196 static void
197 tcp_usr_detach(struct socket *so)
198 {
199 	struct inpcb *inp;
200 	struct tcpcb *tp;
201 
202 	inp = sotoinpcb(so);
203 	KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
204 	INP_WLOCK(inp);
205 	KASSERT(so->so_pcb == inp && inp->inp_socket == so,
206 		("%s: socket %p inp %p mismatch", __func__, so, inp));
207 
208 	tp = intotcpcb(inp);
209 
210 	KASSERT(inp->inp_flags & INP_DROPPED ||
211 	    tp->t_state < TCPS_SYN_SENT,
212 	    ("%s: inp %p not dropped or embryonic", __func__, inp));
213 
214 	tcp_discardcb(tp);
215 	in_pcbfree(inp);
216 }
217 
218 #ifdef INET
219 /*
220  * Give the socket an address.
221  */
222 static int
223 tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
224 {
225 	int error = 0;
226 	struct inpcb *inp;
227 	struct tcpcb *tp;
228 	struct sockaddr_in *sinp;
229 
230 	inp = sotoinpcb(so);
231 	KASSERT(inp != NULL, ("tcp_usr_bind: inp == NULL"));
232 	INP_WLOCK(inp);
233 	if (inp->inp_flags & INP_DROPPED) {
234 		INP_WUNLOCK(inp);
235 		return (EINVAL);
236 	}
237 	tp = intotcpcb(inp);
238 
239 	sinp = (struct sockaddr_in *)nam;
240 	if (nam->sa_family != AF_INET) {
241 		/*
242 		 * Preserve compatibility with old programs.
243 		 */
244 		if (nam->sa_family != AF_UNSPEC ||
245 		    nam->sa_len < offsetof(struct sockaddr_in, sin_zero) ||
246 		    sinp->sin_addr.s_addr != INADDR_ANY) {
247 			error = EAFNOSUPPORT;
248 			goto out;
249 		}
250 		nam->sa_family = AF_INET;
251 	}
252 	if (nam->sa_len != sizeof(*sinp)) {
253 		error = EINVAL;
254 		goto out;
255 	}
256 	/*
257 	 * Must check for multicast addresses and disallow binding
258 	 * to them.
259 	 */
260 	if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
261 		error = EAFNOSUPPORT;
262 		goto out;
263 	}
264 	INP_HASH_WLOCK(&V_tcbinfo);
265 	error = in_pcbbind(inp, sinp, td->td_ucred);
266 	INP_HASH_WUNLOCK(&V_tcbinfo);
267 out:
268 	tcp_bblog_pru(tp, PRU_BIND, error);
269 	TCP_PROBE2(debug__user, tp, PRU_BIND);
270 	INP_WUNLOCK(inp);
271 
272 	return (error);
273 }
274 #endif /* INET */
275 
276 #ifdef INET6
277 static int
278 tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
279 {
280 	int error = 0;
281 	struct inpcb *inp;
282 	struct tcpcb *tp;
283 	struct sockaddr_in6 *sin6;
284 	u_char vflagsav;
285 
286 	inp = sotoinpcb(so);
287 	KASSERT(inp != NULL, ("tcp6_usr_bind: inp == NULL"));
288 	INP_WLOCK(inp);
289 	if (inp->inp_flags & INP_DROPPED) {
290 		INP_WUNLOCK(inp);
291 		return (EINVAL);
292 	}
293 	tp = intotcpcb(inp);
294 
295 	vflagsav = inp->inp_vflag;
296 
297 	sin6 = (struct sockaddr_in6 *)nam;
298 	if (nam->sa_family != AF_INET6) {
299 		error = EAFNOSUPPORT;
300 		goto out;
301 	}
302 	if (nam->sa_len != sizeof(*sin6)) {
303 		error = EINVAL;
304 		goto out;
305 	}
306 	/*
307 	 * Must check for multicast addresses and disallow binding
308 	 * to them.
309 	 */
310 	if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
311 		error = EAFNOSUPPORT;
312 		goto out;
313 	}
314 
315 	INP_HASH_WLOCK(&V_tcbinfo);
316 	inp->inp_vflag &= ~INP_IPV4;
317 	inp->inp_vflag |= INP_IPV6;
318 #ifdef INET
319 	if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0) {
320 		if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr))
321 			inp->inp_vflag |= INP_IPV4;
322 		else if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
323 			struct sockaddr_in sin;
324 
325 			in6_sin6_2_sin(&sin, sin6);
326 			if (IN_MULTICAST(ntohl(sin.sin_addr.s_addr))) {
327 				error = EAFNOSUPPORT;
328 				INP_HASH_WUNLOCK(&V_tcbinfo);
329 				goto out;
330 			}
331 			inp->inp_vflag |= INP_IPV4;
332 			inp->inp_vflag &= ~INP_IPV6;
333 			error = in_pcbbind(inp, &sin, td->td_ucred);
334 			INP_HASH_WUNLOCK(&V_tcbinfo);
335 			goto out;
336 		}
337 	}
338 #endif
339 	error = in6_pcbbind(inp, sin6, td->td_ucred);
340 	INP_HASH_WUNLOCK(&V_tcbinfo);
341 out:
342 	if (error != 0)
343 		inp->inp_vflag = vflagsav;
344 	tcp_bblog_pru(tp, PRU_BIND, error);
345 	TCP_PROBE2(debug__user, tp, PRU_BIND);
346 	INP_WUNLOCK(inp);
347 	return (error);
348 }
349 #endif /* INET6 */
350 
351 #ifdef INET
352 /*
353  * Prepare to accept connections.
354  */
355 static int
356 tcp_usr_listen(struct socket *so, int backlog, struct thread *td)
357 {
358 	int error = 0;
359 	struct inpcb *inp;
360 	struct tcpcb *tp;
361 
362 	inp = sotoinpcb(so);
363 	KASSERT(inp != NULL, ("tcp_usr_listen: inp == NULL"));
364 	INP_WLOCK(inp);
365 	if (inp->inp_flags & INP_DROPPED) {
366 		INP_WUNLOCK(inp);
367 		return (EINVAL);
368 	}
369 	tp = intotcpcb(inp);
370 
371 	SOCK_LOCK(so);
372 	error = solisten_proto_check(so);
373 	if (error != 0) {
374 		SOCK_UNLOCK(so);
375 		goto out;
376 	}
377 	if (inp->inp_lport == 0) {
378 		INP_HASH_WLOCK(&V_tcbinfo);
379 		error = in_pcbbind(inp, NULL, td->td_ucred);
380 		INP_HASH_WUNLOCK(&V_tcbinfo);
381 	}
382 	if (error == 0) {
383 		tcp_state_change(tp, TCPS_LISTEN);
384 		solisten_proto(so, backlog);
385 #ifdef TCP_OFFLOAD
386 		if ((so->so_options & SO_NO_OFFLOAD) == 0)
387 			tcp_offload_listen_start(tp);
388 #endif
389 	} else {
390 		solisten_proto_abort(so);
391 	}
392 	SOCK_UNLOCK(so);
393 
394 	if (tp->t_flags & TF_FASTOPEN)
395 		tp->t_tfo_pending = tcp_fastopen_alloc_counter();
396 
397 out:
398 	tcp_bblog_pru(tp, PRU_LISTEN, error);
399 	TCP_PROBE2(debug__user, tp, PRU_LISTEN);
400 	INP_WUNLOCK(inp);
401 	return (error);
402 }
403 #endif /* INET */
404 
405 #ifdef INET6
406 static int
407 tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
408 {
409 	int error = 0;
410 	struct inpcb *inp;
411 	struct tcpcb *tp;
412 	u_char vflagsav;
413 
414 	inp = sotoinpcb(so);
415 	KASSERT(inp != NULL, ("tcp6_usr_listen: inp == NULL"));
416 	INP_WLOCK(inp);
417 	if (inp->inp_flags & INP_DROPPED) {
418 		INP_WUNLOCK(inp);
419 		return (EINVAL);
420 	}
421 	tp = intotcpcb(inp);
422 
423 	vflagsav = inp->inp_vflag;
424 
425 	SOCK_LOCK(so);
426 	error = solisten_proto_check(so);
427 	if (error != 0) {
428 		SOCK_UNLOCK(so);
429 		goto out;
430 	}
431 	INP_HASH_WLOCK(&V_tcbinfo);
432 	if (inp->inp_lport == 0) {
433 		inp->inp_vflag &= ~INP_IPV4;
434 		if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0)
435 			inp->inp_vflag |= INP_IPV4;
436 		error = in6_pcbbind(inp, NULL, td->td_ucred);
437 	}
438 	INP_HASH_WUNLOCK(&V_tcbinfo);
439 	if (error == 0) {
440 		tcp_state_change(tp, TCPS_LISTEN);
441 		solisten_proto(so, backlog);
442 #ifdef TCP_OFFLOAD
443 		if ((so->so_options & SO_NO_OFFLOAD) == 0)
444 			tcp_offload_listen_start(tp);
445 #endif
446 	} else {
447 		solisten_proto_abort(so);
448 	}
449 	SOCK_UNLOCK(so);
450 
451 	if (tp->t_flags & TF_FASTOPEN)
452 		tp->t_tfo_pending = tcp_fastopen_alloc_counter();
453 
454 	if (error != 0)
455 		inp->inp_vflag = vflagsav;
456 
457 out:
458 	tcp_bblog_pru(tp, PRU_LISTEN, error);
459 	TCP_PROBE2(debug__user, tp, PRU_LISTEN);
460 	INP_WUNLOCK(inp);
461 	return (error);
462 }
463 #endif /* INET6 */
464 
465 #ifdef INET
466 /*
467  * Initiate connection to peer.
468  * Create a template for use in transmissions on this connection.
469  * Enter SYN_SENT state, and mark socket as connecting.
470  * Start keep-alive timer, and seed output sequence space.
471  * Send initial segment on connection.
472  */
473 static int
474 tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
475 {
476 	struct epoch_tracker et;
477 	int error = 0;
478 	struct inpcb *inp;
479 	struct tcpcb *tp;
480 	struct sockaddr_in *sinp;
481 
482 	inp = sotoinpcb(so);
483 	KASSERT(inp != NULL, ("tcp_usr_connect: inp == NULL"));
484 	INP_WLOCK(inp);
485 	if (inp->inp_flags & INP_DROPPED) {
486 		INP_WUNLOCK(inp);
487 		return (ECONNREFUSED);
488 	}
489 	tp = intotcpcb(inp);
490 
491 	sinp = (struct sockaddr_in *)nam;
492 	if (nam->sa_family != AF_INET) {
493 		error = EAFNOSUPPORT;
494 		goto out;
495 	}
496 	if (nam->sa_len != sizeof (*sinp)) {
497 		error = EINVAL;
498 		goto out;
499 	}
500 	/*
501 	 * Must disallow TCP ``connections'' to multicast addresses.
502 	 */
503 	if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
504 		error = EAFNOSUPPORT;
505 		goto out;
506 	}
507 	if (ntohl(sinp->sin_addr.s_addr) == INADDR_BROADCAST) {
508 		error = EACCES;
509 		goto out;
510 	}
511 	if ((error = prison_remote_ip4(td->td_ucred, &sinp->sin_addr)) != 0)
512 		goto out;
513 	if (SOLISTENING(so)) {
514 		error = EOPNOTSUPP;
515 		goto out;
516 	}
517 	NET_EPOCH_ENTER(et);
518 	if ((error = tcp_connect(tp, sinp, td)) != 0)
519 		goto out_in_epoch;
520 #ifdef TCP_OFFLOAD
521 	if (registered_toedevs > 0 &&
522 	    (so->so_options & SO_NO_OFFLOAD) == 0 &&
523 	    (error = tcp_offload_connect(so, nam)) == 0)
524 		goto out_in_epoch;
525 #endif
526 	tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
527 	error = tcp_output(tp);
528 	KASSERT(error >= 0, ("TCP stack %s requested tcp_drop(%p) at connect()"
529 	    ", error code %d", tp->t_fb->tfb_tcp_block_name, tp, -error));
530 out_in_epoch:
531 	NET_EPOCH_EXIT(et);
532 out:
533 	tcp_bblog_pru(tp, PRU_CONNECT, error);
534 	TCP_PROBE2(debug__user, tp, PRU_CONNECT);
535 	INP_WUNLOCK(inp);
536 	return (error);
537 }
538 #endif /* INET */
539 
540 #ifdef INET6
541 static int
542 tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
543 {
544 	struct epoch_tracker et;
545 	int error = 0;
546 	struct inpcb *inp;
547 	struct tcpcb *tp;
548 	struct sockaddr_in6 *sin6;
549 	u_int8_t incflagsav;
550 	u_char vflagsav;
551 
552 	inp = sotoinpcb(so);
553 	KASSERT(inp != NULL, ("tcp6_usr_connect: inp == NULL"));
554 	INP_WLOCK(inp);
555 	if (inp->inp_flags & INP_DROPPED) {
556 		INP_WUNLOCK(inp);
557 		return (ECONNREFUSED);
558 	}
559 	tp = intotcpcb(inp);
560 
561 	vflagsav = inp->inp_vflag;
562 	incflagsav = inp->inp_inc.inc_flags;
563 
564 	sin6 = (struct sockaddr_in6 *)nam;
565 	if (nam->sa_family != AF_INET6) {
566 		error = EAFNOSUPPORT;
567 		goto out;
568 	}
569 	if (nam->sa_len != sizeof (*sin6)) {
570 		error = EINVAL;
571 		goto out;
572 	}
573 	/*
574 	 * Must disallow TCP ``connections'' to multicast addresses.
575 	 */
576 	if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
577 		error = EAFNOSUPPORT;
578 		goto out;
579 	}
580 	if (SOLISTENING(so)) {
581 		error = EINVAL;
582 		goto out;
583 	}
584 #ifdef INET
585 	/*
586 	 * XXXRW: Some confusion: V4/V6 flags relate to binding, and
587 	 * therefore probably require the hash lock, which isn't held here.
588 	 * Is this a significant problem?
589 	 */
590 	if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
591 		struct sockaddr_in sin;
592 
593 		if ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0) {
594 			error = EINVAL;
595 			goto out;
596 		}
597 		if ((inp->inp_vflag & INP_IPV4) == 0) {
598 			error = EAFNOSUPPORT;
599 			goto out;
600 		}
601 
602 		in6_sin6_2_sin(&sin, sin6);
603 		if (IN_MULTICAST(ntohl(sin.sin_addr.s_addr))) {
604 			error = EAFNOSUPPORT;
605 			goto out;
606 		}
607 		if (ntohl(sin.sin_addr.s_addr) == INADDR_BROADCAST) {
608 			error = EACCES;
609 			goto out;
610 		}
611 		if ((error = prison_remote_ip4(td->td_ucred,
612 		    &sin.sin_addr)) != 0)
613 			goto out;
614 		inp->inp_vflag |= INP_IPV4;
615 		inp->inp_vflag &= ~INP_IPV6;
616 		NET_EPOCH_ENTER(et);
617 		if ((error = tcp_connect(tp, &sin, td)) != 0)
618 			goto out_in_epoch;
619 #ifdef TCP_OFFLOAD
620 		if (registered_toedevs > 0 &&
621 		    (so->so_options & SO_NO_OFFLOAD) == 0 &&
622 		    (error = tcp_offload_connect(so, nam)) == 0)
623 			goto out_in_epoch;
624 #endif
625 		error = tcp_output(tp);
626 		goto out_in_epoch;
627 	} else {
628 		if ((inp->inp_vflag & INP_IPV6) == 0) {
629 			error = EAFNOSUPPORT;
630 			goto out;
631 		}
632 	}
633 #endif
634 	if ((error = prison_remote_ip6(td->td_ucred, &sin6->sin6_addr)) != 0)
635 		goto out;
636 	inp->inp_vflag &= ~INP_IPV4;
637 	inp->inp_vflag |= INP_IPV6;
638 	inp->inp_inc.inc_flags |= INC_ISIPV6;
639 	NET_EPOCH_ENTER(et);
640 	if ((error = tcp6_connect(tp, sin6, td)) != 0)
641 		goto out_in_epoch;
642 #ifdef TCP_OFFLOAD
643 	if (registered_toedevs > 0 &&
644 	    (so->so_options & SO_NO_OFFLOAD) == 0 &&
645 	    (error = tcp_offload_connect(so, nam)) == 0)
646 		goto out_in_epoch;
647 #endif
648 	tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
649 	error = tcp_output(tp);
650 out_in_epoch:
651 	NET_EPOCH_EXIT(et);
652 out:
653 	KASSERT(error >= 0, ("TCP stack %s requested tcp_drop(%p) at connect()"
654 	    ", error code %d", tp->t_fb->tfb_tcp_block_name, tp, -error));
655 	/*
656 	 * If the implicit bind in the connect call fails, restore
657 	 * the flags we modified.
658 	 */
659 	if (error != 0 && inp->inp_lport == 0) {
660 		inp->inp_vflag = vflagsav;
661 		inp->inp_inc.inc_flags = incflagsav;
662 	}
663 
664 	tcp_bblog_pru(tp, PRU_CONNECT, error);
665 	TCP_PROBE2(debug__user, tp, PRU_CONNECT);
666 	INP_WUNLOCK(inp);
667 	return (error);
668 }
669 #endif /* INET6 */
670 
671 /*
672  * Initiate disconnect from peer.
673  * If connection never passed embryonic stage, just drop;
674  * else if don't need to let data drain, then can just drop anyways,
675  * else have to begin TCP shutdown process: mark socket disconnecting,
676  * drain unread data, state switch to reflect user close, and
677  * send segment (e.g. FIN) to peer.  Socket will be really disconnected
678  * when peer sends FIN and acks ours.
679  *
680  * SHOULD IMPLEMENT LATER PRU_CONNECT VIA REALLOC TCPCB.
681  */
682 static int
683 tcp_usr_disconnect(struct socket *so)
684 {
685 	struct inpcb *inp;
686 	struct tcpcb *tp = NULL;
687 	struct epoch_tracker et;
688 	int error = 0;
689 
690 	NET_EPOCH_ENTER(et);
691 	inp = sotoinpcb(so);
692 	KASSERT(inp != NULL, ("tcp_usr_disconnect: inp == NULL"));
693 	INP_WLOCK(inp);
694 	if (inp->inp_flags & INP_DROPPED) {
695 		INP_WUNLOCK(inp);
696 		NET_EPOCH_EXIT(et);
697 		return (ECONNRESET);
698 	}
699 	tp = intotcpcb(inp);
700 
701 	if (tp->t_state == TCPS_TIME_WAIT)
702 		goto out;
703 	tcp_disconnect(tp);
704 out:
705 	tcp_bblog_pru(tp, PRU_DISCONNECT, error);
706 	TCP_PROBE2(debug__user, tp, PRU_DISCONNECT);
707 	INP_WUNLOCK(inp);
708 	NET_EPOCH_EXIT(et);
709 	return (error);
710 }
711 
712 #ifdef INET
713 /*
714  * Accept a connection.  Essentially all the work is done at higher levels;
715  * just return the address of the peer, storing through addr.
716  */
717 static int
718 tcp_usr_accept(struct socket *so, struct sockaddr *sa)
719 {
720 	struct inpcb *inp;
721 	struct tcpcb *tp;
722 	int error = 0;
723 
724 	inp = sotoinpcb(so);
725 	KASSERT(inp != NULL, ("tcp_usr_accept: inp == NULL"));
726 	INP_WLOCK(inp);
727 	if (inp->inp_flags & INP_DROPPED) {
728 		INP_WUNLOCK(inp);
729 		return (ECONNABORTED);
730 	}
731 	tp = intotcpcb(inp);
732 
733 	if (so->so_state & SS_ISDISCONNECTED)
734 		error = ECONNABORTED;
735 	else
736 		*(struct sockaddr_in *)sa = (struct sockaddr_in ){
737 			.sin_family = AF_INET,
738 			.sin_len = sizeof(struct sockaddr_in),
739 			.sin_port = inp->inp_fport,
740 			.sin_addr = inp->inp_faddr,
741 		};
742 	tcp_bblog_pru(tp, PRU_ACCEPT, error);
743 	TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
744 	INP_WUNLOCK(inp);
745 
746 	return (error);
747 }
748 #endif /* INET */
749 
750 #ifdef INET6
751 static int
752 tcp6_usr_accept(struct socket *so, struct sockaddr *sa)
753 {
754 	struct inpcb *inp;
755 	struct tcpcb *tp;
756 	int error = 0;
757 
758 	inp = sotoinpcb(so);
759 	KASSERT(inp != NULL, ("tcp6_usr_accept: inp == NULL"));
760 	INP_WLOCK(inp);
761 	if (inp->inp_flags & INP_DROPPED) {
762 		INP_WUNLOCK(inp);
763 		return (ECONNABORTED);
764 	}
765 	tp = intotcpcb(inp);
766 
767 	if (so->so_state & SS_ISDISCONNECTED) {
768 		error = ECONNABORTED;
769 	} else {
770 		if (inp->inp_vflag & INP_IPV4) {
771 			struct sockaddr_in sin = {
772 				.sin_family = AF_INET,
773 				.sin_len = sizeof(struct sockaddr_in),
774 				.sin_port = inp->inp_fport,
775 				.sin_addr = inp->inp_faddr,
776 			};
777 			in6_sin_2_v4mapsin6(&sin, (struct sockaddr_in6 *)sa);
778 		} else {
779 			*(struct sockaddr_in6 *)sa = (struct sockaddr_in6 ){
780 				.sin6_family = AF_INET6,
781 				.sin6_len = sizeof(struct sockaddr_in6),
782 				.sin6_port = inp->inp_fport,
783 				.sin6_addr = inp->in6p_faddr,
784 			};
785 			/* XXX: should catch errors */
786 			(void)sa6_recoverscope((struct sockaddr_in6 *)sa);
787 		}
788 	}
789 
790 	tcp_bblog_pru(tp, PRU_ACCEPT, error);
791 	TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
792 	INP_WUNLOCK(inp);
793 
794 	return (error);
795 }
796 #endif /* INET6 */
797 
798 /*
799  * Mark the connection as being incapable of further output.
800  */
801 static int
802 tcp_usr_shutdown(struct socket *so, enum shutdown_how how)
803 {
804 	struct epoch_tracker et;
805 	struct inpcb *inp = sotoinpcb(so);
806 	struct tcpcb *tp = intotcpcb(inp);
807 	int error = 0;
808 
809 	SOCK_LOCK(so);
810 	if (SOLISTENING(so)) {
811 		if (how != SHUT_WR) {
812 			so->so_error = ECONNABORTED;
813 			solisten_wakeup(so);	/* unlocks so */
814 		} else
815 			SOCK_UNLOCK(so);
816 		return (ENOTCONN);
817 	} else if ((so->so_state &
818 	    (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) {
819 		SOCK_UNLOCK(so);
820 		return (ENOTCONN);
821 	}
822 	SOCK_UNLOCK(so);
823 
824 	switch (how) {
825 	case SHUT_RD:
826 		sorflush(so);
827 		break;
828 	case SHUT_RDWR:
829 		sorflush(so);
830 		/* FALLTHROUGH */
831 	case SHUT_WR:
832 		/*
833 		 * XXXGL: mimicing old soshutdown() here. But shouldn't we
834 		 * return ECONNRESEST for SHUT_RD as well?
835 		 */
836 		INP_WLOCK(inp);
837 		if (inp->inp_flags & INP_DROPPED) {
838 			INP_WUNLOCK(inp);
839 			return (ECONNRESET);
840 		}
841 
842 		socantsendmore(so);
843 		NET_EPOCH_ENTER(et);
844 		tcp_usrclosed(tp);
845 		error = tcp_output_nodrop(tp);
846 		tcp_bblog_pru(tp, PRU_SHUTDOWN, error);
847 		TCP_PROBE2(debug__user, tp, PRU_SHUTDOWN);
848 		error = tcp_unlock_or_drop(tp, error);
849 		NET_EPOCH_EXIT(et);
850 	}
851 	wakeup(&so->so_timeo);
852 
853 	return (error);
854 }
855 
856 /*
857  * After a receive, possibly send window update to peer.
858  */
859 static int
860 tcp_usr_rcvd(struct socket *so, int flags)
861 {
862 	struct epoch_tracker et;
863 	struct inpcb *inp;
864 	struct tcpcb *tp;
865 	int outrv = 0, error = 0;
866 
867 	inp = sotoinpcb(so);
868 	KASSERT(inp != NULL, ("tcp_usr_rcvd: inp == NULL"));
869 	INP_WLOCK(inp);
870 	if (inp->inp_flags & INP_DROPPED) {
871 		INP_WUNLOCK(inp);
872 		return (ECONNRESET);
873 	}
874 	tp = intotcpcb(inp);
875 
876 	NET_EPOCH_ENTER(et);
877 	/*
878 	 * For passively-created TFO connections, don't attempt a window
879 	 * update while still in SYN_RECEIVED as this may trigger an early
880 	 * SYN|ACK.  It is preferable to have the SYN|ACK be sent along with
881 	 * application response data, or failing that, when the DELACK timer
882 	 * expires.
883 	 */
884 	if ((tp->t_flags & TF_FASTOPEN) && (tp->t_state == TCPS_SYN_RECEIVED))
885 		goto out;
886 #ifdef TCP_OFFLOAD
887 	if (tp->t_flags & TF_TOE)
888 		tcp_offload_rcvd(tp);
889 	else
890 #endif
891 		outrv = tcp_output_nodrop(tp);
892 out:
893 	tcp_bblog_pru(tp, PRU_RCVD, error);
894 	TCP_PROBE2(debug__user, tp, PRU_RCVD);
895 	(void) tcp_unlock_or_drop(tp, outrv);
896 	NET_EPOCH_EXIT(et);
897 	return (error);
898 }
899 
900 /*
901  * Do a send by putting data in output queue and updating urgent
902  * marker if URG set.  Possibly send more data.  Unlike the other
903  * pru_*() routines, the mbuf chains are our responsibility.  We
904  * must either enqueue them or free them.  The other pru_* routines
905  * generally are caller-frees.
906  */
907 static int
908 tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
909     struct sockaddr *nam, struct mbuf *control, struct thread *td)
910 {
911 	struct epoch_tracker et;
912 	int error = 0;
913 	struct inpcb *inp;
914 	struct tcpcb *tp;
915 #ifdef INET
916 #ifdef INET6
917 	struct sockaddr_in sin;
918 #endif
919 	struct sockaddr_in *sinp;
920 #endif
921 #ifdef INET6
922 	struct sockaddr_in6 *sin6;
923 	int isipv6;
924 #endif
925 	u_int8_t incflagsav;
926 	u_char vflagsav;
927 	bool restoreflags;
928 
929 	inp = sotoinpcb(so);
930 	KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL"));
931 	INP_WLOCK(inp);
932 	if (inp->inp_flags & INP_DROPPED) {
933 		if (m != NULL && (flags & PRUS_NOTREADY) == 0)
934 			m_freem(m);
935 		INP_WUNLOCK(inp);
936 		return (ECONNRESET);
937 	}
938 	tp = intotcpcb(inp);
939 
940 	vflagsav = inp->inp_vflag;
941 	incflagsav = inp->inp_inc.inc_flags;
942 	restoreflags = false;
943 
944 	NET_EPOCH_ENTER(et);
945 	if (control != NULL) {
946 		/* TCP doesn't do control messages (rights, creds, etc) */
947 		if (control->m_len > 0) {
948 			m_freem(control);
949 			error = EINVAL;
950 			goto out;
951 		}
952 		m_freem(control);	/* empty control, just free it */
953 	}
954 
955 	if ((flags & PRUS_OOB) != 0 &&
956 	    (error = tcp_pru_options_support(tp, PRUS_OOB)) != 0)
957 		goto out;
958 
959 	if (nam != NULL && tp->t_state < TCPS_SYN_SENT) {
960 		if (tp->t_state == TCPS_LISTEN) {
961 			error = EINVAL;
962 			goto out;
963 		}
964 		switch (nam->sa_family) {
965 #ifdef INET
966 		case AF_INET:
967 			sinp = (struct sockaddr_in *)nam;
968 			if (sinp->sin_len != sizeof(struct sockaddr_in)) {
969 				error = EINVAL;
970 				goto out;
971 			}
972 			if ((inp->inp_vflag & INP_IPV6) != 0) {
973 				error = EAFNOSUPPORT;
974 				goto out;
975 			}
976 			if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
977 				error = EAFNOSUPPORT;
978 				goto out;
979 			}
980 			if (ntohl(sinp->sin_addr.s_addr) == INADDR_BROADCAST) {
981 				error = EACCES;
982 				goto out;
983 			}
984 			if ((error = prison_remote_ip4(td->td_ucred,
985 			    &sinp->sin_addr)))
986 				goto out;
987 #ifdef INET6
988 			isipv6 = 0;
989 #endif
990 			break;
991 #endif /* INET */
992 #ifdef INET6
993 		case AF_INET6:
994 			sin6 = (struct sockaddr_in6 *)nam;
995 			if (sin6->sin6_len != sizeof(*sin6)) {
996 				error = EINVAL;
997 				goto out;
998 			}
999 			if ((inp->inp_vflag & INP_IPV6PROTO) == 0) {
1000 				error = EAFNOSUPPORT;
1001 				goto out;
1002 			}
1003 			if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
1004 				error = EAFNOSUPPORT;
1005 				goto out;
1006 			}
1007 			if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
1008 #ifdef INET
1009 				if ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0) {
1010 					error = EINVAL;
1011 					goto out;
1012 				}
1013 				if ((inp->inp_vflag & INP_IPV4) == 0) {
1014 					error = EAFNOSUPPORT;
1015 					goto out;
1016 				}
1017 				restoreflags = true;
1018 				inp->inp_vflag &= ~INP_IPV6;
1019 				sinp = &sin;
1020 				in6_sin6_2_sin(sinp, sin6);
1021 				if (IN_MULTICAST(
1022 				    ntohl(sinp->sin_addr.s_addr))) {
1023 					error = EAFNOSUPPORT;
1024 					goto out;
1025 				}
1026 				if ((error = prison_remote_ip4(td->td_ucred,
1027 				    &sinp->sin_addr)))
1028 					goto out;
1029 				isipv6 = 0;
1030 #else /* !INET */
1031 				error = EAFNOSUPPORT;
1032 				goto out;
1033 #endif /* INET */
1034 			} else {
1035 				if ((inp->inp_vflag & INP_IPV6) == 0) {
1036 					error = EAFNOSUPPORT;
1037 					goto out;
1038 				}
1039 				restoreflags = true;
1040 				inp->inp_vflag &= ~INP_IPV4;
1041 				inp->inp_inc.inc_flags |= INC_ISIPV6;
1042 				if ((error = prison_remote_ip6(td->td_ucred,
1043 				    &sin6->sin6_addr)))
1044 					goto out;
1045 				isipv6 = 1;
1046 			}
1047 			break;
1048 #endif /* INET6 */
1049 		default:
1050 			error = EAFNOSUPPORT;
1051 			goto out;
1052 		}
1053 	}
1054 	if (!(flags & PRUS_OOB)) {
1055 		if (tp->t_acktime == 0)
1056 			tp->t_acktime = ticks;
1057 		sbappendstream(&so->so_snd, m, flags);
1058 		m = NULL;
1059 		if (nam && tp->t_state < TCPS_SYN_SENT) {
1060 			KASSERT(tp->t_state == TCPS_CLOSED,
1061 			    ("%s: tp %p is listening", __func__, tp));
1062 
1063 			/*
1064 			 * Do implied connect if not yet connected,
1065 			 * initialize window to default value, and
1066 			 * initialize maxseg using peer's cached MSS.
1067 			 */
1068 #ifdef INET6
1069 			if (isipv6)
1070 				error = tcp6_connect(tp, sin6, td);
1071 #endif /* INET6 */
1072 #if defined(INET6) && defined(INET)
1073 			else
1074 #endif
1075 #ifdef INET
1076 				error = tcp_connect(tp, sinp, td);
1077 #endif
1078 			/*
1079 			 * The bind operation in tcp_connect succeeded. We
1080 			 * no longer want to restore the flags if later
1081 			 * operations fail.
1082 			 */
1083 			if (error == 0 || inp->inp_lport != 0)
1084 				restoreflags = false;
1085 
1086 			if (error) {
1087 				/* m is freed if PRUS_NOTREADY is unset. */
1088 				sbflush(&so->so_snd);
1089 				goto out;
1090 			}
1091 			if (tp->t_flags & TF_FASTOPEN)
1092 				tcp_fastopen_connect(tp);
1093 			else {
1094 				tp->snd_wnd = TTCP_CLIENT_SND_WND;
1095 				tcp_mss(tp, -1);
1096 			}
1097 		}
1098 		if (flags & PRUS_EOF) {
1099 			/*
1100 			 * Close the send side of the connection after
1101 			 * the data is sent.
1102 			 */
1103 			socantsendmore(so);
1104 			tcp_usrclosed(tp);
1105 		}
1106 		if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1107 		    ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
1108 		    (tp->t_fbyte_out == 0) &&
1109 		    (so->so_snd.sb_ccc > 0)) {
1110 			tp->t_fbyte_out = ticks;
1111 			if (tp->t_fbyte_out == 0)
1112 				tp->t_fbyte_out = 1;
1113 			if (tp->t_fbyte_out && tp->t_fbyte_in)
1114 				tp->t_flags2 |= TF2_FBYTES_COMPLETE;
1115 		}
1116 		if (!(inp->inp_flags & INP_DROPPED) &&
1117 		    !(flags & PRUS_NOTREADY)) {
1118 			if (flags & PRUS_MORETOCOME)
1119 				tp->t_flags |= TF_MORETOCOME;
1120 			error = tcp_output_nodrop(tp);
1121 			if (flags & PRUS_MORETOCOME)
1122 				tp->t_flags &= ~TF_MORETOCOME;
1123 		}
1124 	} else {
1125 		/*
1126 		 * XXXRW: PRUS_EOF not implemented with PRUS_OOB?
1127 		 */
1128 		SOCKBUF_LOCK(&so->so_snd);
1129 		if (sbspace(&so->so_snd) < -512) {
1130 			SOCKBUF_UNLOCK(&so->so_snd);
1131 			error = ENOBUFS;
1132 			goto out;
1133 		}
1134 		/*
1135 		 * According to RFC961 (Assigned Protocols),
1136 		 * the urgent pointer points to the last octet
1137 		 * of urgent data.  We continue, however,
1138 		 * to consider it to indicate the first octet
1139 		 * of data past the urgent section.
1140 		 * Otherwise, snd_up should be one lower.
1141 		 */
1142 		if (tp->t_acktime == 0)
1143 			tp->t_acktime = ticks;
1144 		sbappendstream_locked(&so->so_snd, m, flags);
1145 		SOCKBUF_UNLOCK(&so->so_snd);
1146 		m = NULL;
1147 		if (nam && tp->t_state < TCPS_SYN_SENT) {
1148 			/*
1149 			 * Do implied connect if not yet connected,
1150 			 * initialize window to default value, and
1151 			 * initialize maxseg using peer's cached MSS.
1152 			 */
1153 
1154 			/*
1155 			 * Not going to contemplate SYN|URG
1156 			 */
1157 			if (tp->t_flags & TF_FASTOPEN)
1158 				tp->t_flags &= ~TF_FASTOPEN;
1159 #ifdef INET6
1160 			if (isipv6)
1161 				error = tcp6_connect(tp, sin6, td);
1162 #endif /* INET6 */
1163 #if defined(INET6) && defined(INET)
1164 			else
1165 #endif
1166 #ifdef INET
1167 				error = tcp_connect(tp, sinp, td);
1168 #endif
1169 			/*
1170 			 * The bind operation in tcp_connect succeeded. We
1171 			 * no longer want to restore the flags if later
1172 			 * operations fail.
1173 			 */
1174 			if (error == 0 || inp->inp_lport != 0)
1175 				restoreflags = false;
1176 
1177 			if (error != 0) {
1178 				/* m is freed if PRUS_NOTREADY is unset. */
1179 				sbflush(&so->so_snd);
1180 				goto out;
1181 			}
1182 			tp->snd_wnd = TTCP_CLIENT_SND_WND;
1183 			tcp_mss(tp, -1);
1184 		}
1185 		tp->snd_up = tp->snd_una + sbavail(&so->so_snd);
1186 		if ((flags & PRUS_NOTREADY) == 0) {
1187 			tp->t_flags |= TF_FORCEDATA;
1188 			error = tcp_output_nodrop(tp);
1189 			tp->t_flags &= ~TF_FORCEDATA;
1190 		}
1191 	}
1192 	TCP_LOG_EVENT(tp, NULL,
1193 	    &inp->inp_socket->so_rcv,
1194 	    &inp->inp_socket->so_snd,
1195 	    TCP_LOG_USERSEND, error,
1196 	    0, NULL, false);
1197 
1198 out:
1199 	/*
1200 	 * In case of PRUS_NOTREADY, the caller or tcp_usr_ready() is
1201 	 * responsible for freeing memory.
1202 	 */
1203 	if (m != NULL && (flags & PRUS_NOTREADY) == 0)
1204 		m_freem(m);
1205 
1206 	/*
1207 	 * If the request was unsuccessful and we changed flags,
1208 	 * restore the original flags.
1209 	 */
1210 	if (error != 0 && restoreflags) {
1211 		inp->inp_vflag = vflagsav;
1212 		inp->inp_inc.inc_flags = incflagsav;
1213 	}
1214 	tcp_bblog_pru(tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
1215 		      ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND), error);
1216 	TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
1217 		   ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
1218 	error = tcp_unlock_or_drop(tp, error);
1219 	NET_EPOCH_EXIT(et);
1220 	return (error);
1221 }
1222 
1223 static int
1224 tcp_usr_ready(struct socket *so, struct mbuf *m, int count)
1225 {
1226 	struct epoch_tracker et;
1227 	struct inpcb *inp;
1228 	struct tcpcb *tp;
1229 	int error;
1230 
1231 	inp = sotoinpcb(so);
1232 	INP_WLOCK(inp);
1233 	if (inp->inp_flags & INP_DROPPED) {
1234 		INP_WUNLOCK(inp);
1235 		mb_free_notready(m, count);
1236 		return (ECONNRESET);
1237 	}
1238 	tp = intotcpcb(inp);
1239 
1240 	SOCKBUF_LOCK(&so->so_snd);
1241 	error = sbready(&so->so_snd, m, count);
1242 	SOCKBUF_UNLOCK(&so->so_snd);
1243 	if (error) {
1244 		INP_WUNLOCK(inp);
1245 		return (error);
1246 	}
1247 	NET_EPOCH_ENTER(et);
1248 	error = tcp_output_unlock(tp);
1249 	NET_EPOCH_EXIT(et);
1250 
1251 	return (error);
1252 }
1253 
1254 /*
1255  * Abort the TCP.  Drop the connection abruptly.
1256  */
1257 static void
1258 tcp_usr_abort(struct socket *so)
1259 {
1260 	struct inpcb *inp;
1261 	struct tcpcb *tp;
1262 	struct epoch_tracker et;
1263 
1264 	inp = sotoinpcb(so);
1265 	KASSERT(inp != NULL, ("tcp_usr_abort: inp == NULL"));
1266 
1267 	NET_EPOCH_ENTER(et);
1268 	INP_WLOCK(inp);
1269 	KASSERT(inp->inp_socket != NULL,
1270 	    ("tcp_usr_abort: inp_socket == NULL"));
1271 
1272 	/*
1273 	 * If we still have full TCP state, and we're not dropped, drop.
1274 	 */
1275 	if (!(inp->inp_flags & INP_DROPPED)) {
1276 		tp = intotcpcb(inp);
1277 		tp = tcp_drop(tp, ECONNABORTED);
1278 		if (tp == NULL)
1279 			goto dropped;
1280 		tcp_bblog_pru(tp, PRU_ABORT, 0);
1281 		TCP_PROBE2(debug__user, tp, PRU_ABORT);
1282 	}
1283 	if (!(inp->inp_flags & INP_DROPPED)) {
1284 		soref(so);
1285 		inp->inp_flags |= INP_SOCKREF;
1286 	}
1287 	INP_WUNLOCK(inp);
1288 dropped:
1289 	NET_EPOCH_EXIT(et);
1290 }
1291 
1292 /*
1293  * TCP socket is closed.  Start friendly disconnect.
1294  */
1295 static void
1296 tcp_usr_close(struct socket *so)
1297 {
1298 	struct inpcb *inp;
1299 	struct tcpcb *tp;
1300 	struct epoch_tracker et;
1301 
1302 	inp = sotoinpcb(so);
1303 	KASSERT(inp != NULL, ("tcp_usr_close: inp == NULL"));
1304 
1305 	NET_EPOCH_ENTER(et);
1306 	INP_WLOCK(inp);
1307 	KASSERT(inp->inp_socket != NULL,
1308 	    ("tcp_usr_close: inp_socket == NULL"));
1309 
1310 	/*
1311 	 * If we are still connected and we're not dropped, initiate
1312 	 * a disconnect.
1313 	 */
1314 	if (!(inp->inp_flags & INP_DROPPED)) {
1315 		tp = intotcpcb(inp);
1316 		if (tp->t_state != TCPS_TIME_WAIT) {
1317 			tp->t_flags |= TF_CLOSED;
1318 			tcp_disconnect(tp);
1319 			tcp_bblog_pru(tp, PRU_CLOSE, 0);
1320 			TCP_PROBE2(debug__user, tp, PRU_CLOSE);
1321 		}
1322 	}
1323 	if (!(inp->inp_flags & INP_DROPPED)) {
1324 		soref(so);
1325 		inp->inp_flags |= INP_SOCKREF;
1326 	}
1327 	INP_WUNLOCK(inp);
1328 	NET_EPOCH_EXIT(et);
1329 }
1330 
1331 static int
1332 tcp_pru_options_support(struct tcpcb *tp, int flags)
1333 {
1334 	/*
1335 	 * If the specific TCP stack has a pru_options
1336 	 * specified then it does not always support
1337 	 * all the PRU_XX options and we must ask it.
1338 	 * If the function is not specified then all
1339 	 * of the PRU_XX options are supported.
1340 	 */
1341 	int ret = 0;
1342 
1343 	if (tp->t_fb->tfb_pru_options) {
1344 		ret = (*tp->t_fb->tfb_pru_options)(tp, flags);
1345 	}
1346 	return (ret);
1347 }
1348 
1349 /*
1350  * Receive out-of-band data.
1351  */
1352 static int
1353 tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
1354 {
1355 	int error = 0;
1356 	struct inpcb *inp;
1357 	struct tcpcb *tp;
1358 
1359 	inp = sotoinpcb(so);
1360 	KASSERT(inp != NULL, ("tcp_usr_rcvoob: inp == NULL"));
1361 	INP_WLOCK(inp);
1362 	if (inp->inp_flags & INP_DROPPED) {
1363 		INP_WUNLOCK(inp);
1364 		return (ECONNRESET);
1365 	}
1366 	tp = intotcpcb(inp);
1367 
1368 	error = tcp_pru_options_support(tp, PRUS_OOB);
1369 	if (error) {
1370 		goto out;
1371 	}
1372 	if ((so->so_oobmark == 0 &&
1373 	     (so->so_rcv.sb_state & SBS_RCVATMARK) == 0) ||
1374 	    so->so_options & SO_OOBINLINE ||
1375 	    tp->t_oobflags & TCPOOB_HADDATA) {
1376 		error = EINVAL;
1377 		goto out;
1378 	}
1379 	if ((tp->t_oobflags & TCPOOB_HAVEDATA) == 0) {
1380 		error = EWOULDBLOCK;
1381 		goto out;
1382 	}
1383 	m->m_len = 1;
1384 	*mtod(m, caddr_t) = tp->t_iobc;
1385 	if ((flags & MSG_PEEK) == 0)
1386 		tp->t_oobflags ^= (TCPOOB_HAVEDATA | TCPOOB_HADDATA);
1387 
1388 out:
1389 	tcp_bblog_pru(tp, PRU_RCVOOB, error);
1390 	TCP_PROBE2(debug__user, tp, PRU_RCVOOB);
1391 	INP_WUNLOCK(inp);
1392 	return (error);
1393 }
1394 
1395 #ifdef INET
1396 struct protosw tcp_protosw = {
1397 	.pr_type =		SOCK_STREAM,
1398 	.pr_protocol =		IPPROTO_TCP,
1399 	.pr_flags =		PR_CONNREQUIRED | PR_IMPLOPCL | PR_WANTRCVD |
1400 				    PR_CAPATTACH,
1401 	.pr_ctloutput =		tcp_ctloutput,
1402 	.pr_abort =		tcp_usr_abort,
1403 	.pr_accept =		tcp_usr_accept,
1404 	.pr_attach =		tcp_usr_attach,
1405 	.pr_bind =		tcp_usr_bind,
1406 	.pr_connect =		tcp_usr_connect,
1407 	.pr_control =		in_control,
1408 	.pr_detach =		tcp_usr_detach,
1409 	.pr_disconnect =	tcp_usr_disconnect,
1410 	.pr_listen =		tcp_usr_listen,
1411 	.pr_peeraddr =		in_getpeeraddr,
1412 	.pr_rcvd =		tcp_usr_rcvd,
1413 	.pr_rcvoob =		tcp_usr_rcvoob,
1414 	.pr_send =		tcp_usr_send,
1415 	.pr_ready =		tcp_usr_ready,
1416 	.pr_shutdown =		tcp_usr_shutdown,
1417 	.pr_sockaddr =		in_getsockaddr,
1418 	.pr_sosetlabel =	in_pcbsosetlabel,
1419 	.pr_close =		tcp_usr_close,
1420 };
1421 #endif /* INET */
1422 
1423 #ifdef INET6
1424 struct protosw tcp6_protosw = {
1425 	.pr_type =		SOCK_STREAM,
1426 	.pr_protocol =		IPPROTO_TCP,
1427 	.pr_flags =		PR_CONNREQUIRED | PR_IMPLOPCL |PR_WANTRCVD |
1428 				    PR_CAPATTACH,
1429 	.pr_ctloutput =		tcp_ctloutput,
1430 	.pr_abort =		tcp_usr_abort,
1431 	.pr_accept =		tcp6_usr_accept,
1432 	.pr_attach =		tcp_usr_attach,
1433 	.pr_bind =		tcp6_usr_bind,
1434 	.pr_connect =		tcp6_usr_connect,
1435 	.pr_control =		in6_control,
1436 	.pr_detach =		tcp_usr_detach,
1437 	.pr_disconnect =	tcp_usr_disconnect,
1438 	.pr_listen =		tcp6_usr_listen,
1439 	.pr_peeraddr =		in6_mapped_peeraddr,
1440 	.pr_rcvd =		tcp_usr_rcvd,
1441 	.pr_rcvoob =		tcp_usr_rcvoob,
1442 	.pr_send =		tcp_usr_send,
1443 	.pr_ready =		tcp_usr_ready,
1444 	.pr_shutdown =		tcp_usr_shutdown,
1445 	.pr_sockaddr =		in6_mapped_sockaddr,
1446 	.pr_sosetlabel =	in_pcbsosetlabel,
1447 	.pr_close =		tcp_usr_close,
1448 };
1449 #endif /* INET6 */
1450 
1451 #ifdef INET
1452 /*
1453  * Common subroutine to open a TCP connection to remote host specified
1454  * by struct sockaddr_in.  Call in_pcbconnect() to choose local host address
1455  * and assign a local port number and install the inpcb into the hash.
1456  * Initialize connection parameters and enter SYN-SENT state.
1457  */
1458 static int
1459 tcp_connect(struct tcpcb *tp, struct sockaddr_in *sin, struct thread *td)
1460 {
1461 	struct inpcb *inp = tptoinpcb(tp);
1462 	struct socket *so = tptosocket(tp);
1463 	int error;
1464 
1465 	NET_EPOCH_ASSERT();
1466 	INP_WLOCK_ASSERT(inp);
1467 
1468 	if (__predict_false((so->so_state &
1469 	    (SS_ISCONNECTING | SS_ISCONNECTED | SS_ISDISCONNECTING |
1470 	    SS_ISDISCONNECTED)) != 0))
1471 		return (EISCONN);
1472 
1473 	INP_HASH_WLOCK(&V_tcbinfo);
1474 	error = in_pcbconnect(inp, sin, td->td_ucred, true);
1475 	INP_HASH_WUNLOCK(&V_tcbinfo);
1476 	if (error != 0)
1477 		return (error);
1478 
1479 	/*
1480 	 * Compute window scaling to request:
1481 	 * Scale to fit into sweet spot.  See tcp_syncache.c.
1482 	 * XXX: This should move to tcp_output().
1483 	 */
1484 	while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1485 	    (TCP_MAXWIN << tp->request_r_scale) < sb_max)
1486 		tp->request_r_scale++;
1487 
1488 	soisconnecting(so);
1489 	TCPSTAT_INC(tcps_connattempt);
1490 	tcp_state_change(tp, TCPS_SYN_SENT);
1491 	tp->iss = tcp_new_isn(&inp->inp_inc);
1492 	if (tp->t_flags & TF_REQ_TSTMP)
1493 		tp->ts_offset = tcp_new_ts_offset(&inp->inp_inc);
1494 	tcp_sendseqinit(tp);
1495 
1496 	return (0);
1497 }
1498 #endif /* INET */
1499 
1500 #ifdef INET6
1501 static int
1502 tcp6_connect(struct tcpcb *tp, struct sockaddr_in6 *sin6, struct thread *td)
1503 {
1504 	struct inpcb *inp = tptoinpcb(tp);
1505 	struct socket *so = tptosocket(tp);
1506 	int error;
1507 
1508 	NET_EPOCH_ASSERT();
1509 	INP_WLOCK_ASSERT(inp);
1510 
1511 	if (__predict_false((so->so_state &
1512 	    (SS_ISCONNECTING | SS_ISCONNECTED)) != 0))
1513 		return (EISCONN);
1514 
1515 	INP_HASH_WLOCK(&V_tcbinfo);
1516 	error = in6_pcbconnect(inp, sin6, td->td_ucred, true);
1517 	INP_HASH_WUNLOCK(&V_tcbinfo);
1518 	if (error != 0)
1519 		return (error);
1520 
1521 	/* Compute window scaling to request.  */
1522 	while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1523 	    (TCP_MAXWIN << tp->request_r_scale) < sb_max)
1524 		tp->request_r_scale++;
1525 
1526 	soisconnecting(so);
1527 	TCPSTAT_INC(tcps_connattempt);
1528 	tcp_state_change(tp, TCPS_SYN_SENT);
1529 	tp->iss = tcp_new_isn(&inp->inp_inc);
1530 	if (tp->t_flags & TF_REQ_TSTMP)
1531 		tp->ts_offset = tcp_new_ts_offset(&inp->inp_inc);
1532 	tcp_sendseqinit(tp);
1533 
1534 	return (0);
1535 }
1536 #endif /* INET6 */
1537 
1538 /*
1539  * Export TCP internal state information via a struct tcp_info, based on the
1540  * Linux 2.6 API.  Not ABI compatible as our constants are mapped differently
1541  * (TCP state machine, etc).  We export all information using FreeBSD-native
1542  * constants -- for example, the numeric values for tcpi_state will differ
1543  * from Linux.
1544  */
1545 void
1546 tcp_fill_info(const struct tcpcb *tp, struct tcp_info *ti)
1547 {
1548 
1549 	INP_LOCK_ASSERT(tptoinpcb(tp));
1550 	bzero(ti, sizeof(*ti));
1551 
1552 	ti->tcpi_state = tp->t_state;
1553 	if ((tp->t_flags & TF_REQ_TSTMP) && (tp->t_flags & TF_RCVD_TSTMP))
1554 		ti->tcpi_options |= TCPI_OPT_TIMESTAMPS;
1555 	if (tp->t_flags & TF_SACK_PERMIT)
1556 		ti->tcpi_options |= TCPI_OPT_SACK;
1557 	if ((tp->t_flags & TF_REQ_SCALE) && (tp->t_flags & TF_RCVD_SCALE)) {
1558 		ti->tcpi_options |= TCPI_OPT_WSCALE;
1559 		ti->tcpi_snd_wscale = tp->snd_scale;
1560 		ti->tcpi_rcv_wscale = tp->rcv_scale;
1561 	}
1562 	switch (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT)) {
1563 		case TF2_ECN_PERMIT:
1564 			ti->tcpi_options |= TCPI_OPT_ECN;
1565 			break;
1566 		case TF2_ACE_PERMIT:
1567 			/* FALLTHROUGH */
1568 		case TF2_ECN_PERMIT | TF2_ACE_PERMIT:
1569 			ti->tcpi_options |= TCPI_OPT_ACE;
1570 			break;
1571 		default:
1572 			break;
1573 	}
1574 	if (tp->t_flags & TF_FASTOPEN)
1575 		ti->tcpi_options |= TCPI_OPT_TFO;
1576 
1577 	ti->tcpi_rto = tp->t_rxtcur * tick;
1578 	ti->tcpi_last_data_recv = ((uint32_t)ticks - tp->t_rcvtime) * tick;
1579 	ti->tcpi_rtt = ((u_int64_t)tp->t_srtt * tick) >> TCP_RTT_SHIFT;
1580 	ti->tcpi_rttvar = ((u_int64_t)tp->t_rttvar * tick) >> TCP_RTTVAR_SHIFT;
1581 
1582 	ti->tcpi_snd_ssthresh = tp->snd_ssthresh;
1583 	ti->tcpi_snd_cwnd = tp->snd_cwnd;
1584 
1585 	/*
1586 	 * FreeBSD-specific extension fields for tcp_info.
1587 	 */
1588 	ti->tcpi_rcv_space = tp->rcv_wnd;
1589 	ti->tcpi_rcv_nxt = tp->rcv_nxt;
1590 	ti->tcpi_snd_wnd = tp->snd_wnd;
1591 	ti->tcpi_snd_bwnd = 0;		/* Unused, kept for compat. */
1592 	ti->tcpi_snd_nxt = tp->snd_nxt;
1593 	ti->tcpi_snd_mss = tp->t_maxseg;
1594 	ti->tcpi_rcv_mss = tp->t_maxseg;
1595 	ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack;
1596 	ti->tcpi_rcv_ooopack = tp->t_rcvoopack;
1597 	ti->tcpi_snd_zerowin = tp->t_sndzerowin;
1598 	ti->tcpi_snd_una = tp->snd_una;
1599 	ti->tcpi_snd_max = tp->snd_max;
1600 	ti->tcpi_rcv_numsacks = tp->rcv_numsacks;
1601 	ti->tcpi_rcv_adv = tp->rcv_adv;
1602 	ti->tcpi_dupacks = tp->t_dupacks;
1603 	ti->tcpi_rttmin = tp->t_rttlow;
1604 #ifdef TCP_OFFLOAD
1605 	if (tp->t_flags & TF_TOE) {
1606 		ti->tcpi_options |= TCPI_OPT_TOE;
1607 		tcp_offload_tcp_info(tp, ti);
1608 	}
1609 #endif
1610 	/*
1611 	 * AccECN related counters.
1612 	 */
1613 	if ((tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT)) ==
1614 	    (TF2_ECN_PERMIT | TF2_ACE_PERMIT))
1615 		/*
1616 		 * Internal counter starts at 5 for AccECN
1617 		 * but 0 for RFC3168 ECN.
1618 		 */
1619 		ti->tcpi_delivered_ce = tp->t_scep - 5;
1620 	else
1621 		ti->tcpi_delivered_ce = tp->t_scep;
1622 	ti->tcpi_received_ce = tp->t_rcep;
1623 }
1624 
1625 /*
1626  * tcp_ctloutput() must drop the inpcb lock before performing copyin on
1627  * socket option arguments.  When it re-acquires the lock after the copy, it
1628  * has to revalidate that the connection is still valid for the socket
1629  * option.
1630  */
1631 #define INP_WLOCK_RECHECK_CLEANUP(inp, cleanup) do {			\
1632 	INP_WLOCK(inp);							\
1633 	if (inp->inp_flags & INP_DROPPED) {				\
1634 		INP_WUNLOCK(inp);					\
1635 		cleanup;						\
1636 		return (ECONNRESET);					\
1637 	}								\
1638 	tp = intotcpcb(inp);						\
1639 } while(0)
1640 #define INP_WLOCK_RECHECK(inp) INP_WLOCK_RECHECK_CLEANUP((inp), /* noop */)
1641 
1642 int
1643 tcp_ctloutput_set(struct inpcb *inp, struct sockopt *sopt)
1644 {
1645 	struct socket *so = inp->inp_socket;
1646 	struct tcpcb *tp = intotcpcb(inp);
1647 	int error = 0;
1648 
1649 	MPASS(sopt->sopt_dir == SOPT_SET);
1650 	INP_WLOCK_ASSERT(inp);
1651 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1652 	    ("inp_flags == %x", inp->inp_flags));
1653 	KASSERT(so != NULL, ("inp_socket == NULL"));
1654 
1655 	if (sopt->sopt_level != IPPROTO_TCP) {
1656 		INP_WUNLOCK(inp);
1657 #ifdef INET6
1658 		if (inp->inp_vflag & INP_IPV6PROTO)
1659 			error = ip6_ctloutput(so, sopt);
1660 #endif
1661 #if defined(INET6) && defined(INET)
1662 		else
1663 #endif
1664 #ifdef INET
1665 			error = ip_ctloutput(so, sopt);
1666 #endif
1667 		/*
1668 		 * When an IP-level socket option affects TCP, pass control
1669 		 * down to stack tfb_tcp_ctloutput, otherwise return what
1670 		 * IP level returned.
1671 		 */
1672 		switch (sopt->sopt_level) {
1673 #ifdef INET6
1674 		case IPPROTO_IPV6:
1675 			if ((inp->inp_vflag & INP_IPV6PROTO) == 0)
1676 				return (error);
1677 			switch (sopt->sopt_name) {
1678 			case IPV6_TCLASS:
1679 				/* Notify tcp stacks that care (e.g. RACK). */
1680 				break;
1681 			case IPV6_USE_MIN_MTU:
1682 				/* Update t_maxseg accordingly. */
1683 				break;
1684 			default:
1685 				return (error);
1686 			}
1687 			break;
1688 #endif
1689 #ifdef INET
1690 		case IPPROTO_IP:
1691 			switch (sopt->sopt_name) {
1692 			case IP_TOS:
1693 				inp->inp_ip_tos &= ~IPTOS_ECN_MASK;
1694 				break;
1695 			case IP_TTL:
1696 				/* Notify tcp stacks that care (e.g. RACK). */
1697 				break;
1698 			default:
1699 				return (error);
1700 			}
1701 			break;
1702 #endif
1703 		default:
1704 			return (error);
1705 		}
1706 		INP_WLOCK_RECHECK(inp);
1707 	} else if (sopt->sopt_name == TCP_FUNCTION_BLK) {
1708 		/*
1709 		 * Protect the TCP option TCP_FUNCTION_BLK so
1710 		 * that a sub-function can *never* overwrite this.
1711 		 */
1712 		struct tcp_function_set fsn;
1713 		struct tcp_function_block *blk;
1714 		void *ptr = NULL;
1715 
1716 		INP_WUNLOCK(inp);
1717 		error = sooptcopyin(sopt, &fsn, sizeof fsn, sizeof fsn);
1718 		if (error)
1719 			return (error);
1720 
1721 		INP_WLOCK_RECHECK(inp);
1722 
1723 		blk = find_and_ref_tcp_functions(&fsn);
1724 		if (blk == NULL) {
1725 			INP_WUNLOCK(inp);
1726 			return (ENOENT);
1727 		}
1728 		if (tp->t_fb == blk) {
1729 			/* You already have this */
1730 			refcount_release(&blk->tfb_refcnt);
1731 			INP_WUNLOCK(inp);
1732 			return (0);
1733 		}
1734 		if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1735 			refcount_release(&blk->tfb_refcnt);
1736 			INP_WUNLOCK(inp);
1737 			return (ENOENT);
1738 		}
1739 		error = (*blk->tfb_tcp_handoff_ok)(tp);
1740 		if (error) {
1741 			refcount_release(&blk->tfb_refcnt);
1742 			INP_WUNLOCK(inp);
1743 			return (error);
1744 		}
1745 		/*
1746 		 * Ensure the new stack takes ownership with a
1747 		 * clean slate on peak rate threshold.
1748 		 */
1749 		if (tp->t_fb->tfb_tcp_timer_stop_all != NULL)
1750 			tp->t_fb->tfb_tcp_timer_stop_all(tp);
1751 		if (blk->tfb_tcp_fb_init) {
1752 			error = (*blk->tfb_tcp_fb_init)(tp, &ptr);
1753 			if (error) {
1754 				/*
1755 				 * Release the ref count the lookup
1756 				 * acquired.
1757 				 */
1758 				refcount_release(&blk->tfb_refcnt);
1759 				/*
1760 				 * Now there is a chance that the
1761 				 * init() function mucked with some
1762 				 * things before it failed, such as
1763 				 * hpts or inp_flags2 or timer granularity.
1764 				 * It should not of, but lets give the old
1765 				 * stack a chance to reset to a known good state.
1766 				 */
1767 				if (tp->t_fb->tfb_switch_failed) {
1768 					(*tp->t_fb->tfb_switch_failed)(tp);
1769 				}
1770 			 	goto err_out;
1771 			}
1772 		}
1773 		if (tp->t_fb->tfb_tcp_fb_fini) {
1774 			struct epoch_tracker et;
1775 			/*
1776 			 * Tell the stack to cleanup with 0 i.e.
1777 			 * the tcb is not going away.
1778 			 */
1779 			NET_EPOCH_ENTER(et);
1780 			(*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
1781 			NET_EPOCH_EXIT(et);
1782 		}
1783 		/*
1784 		 * Release the old refcnt, the
1785 		 * lookup acquired a ref on the
1786 		 * new one already.
1787 		 */
1788 		refcount_release(&tp->t_fb->tfb_refcnt);
1789 		/*
1790 		 * Set in the new stack.
1791 		 */
1792 		tp->t_fb = blk;
1793 		tp->t_fb_ptr = ptr;
1794 #ifdef TCP_OFFLOAD
1795 		if (tp->t_flags & TF_TOE) {
1796 			tcp_offload_ctloutput(tp, sopt->sopt_dir,
1797 			     sopt->sopt_name);
1798 		}
1799 #endif
1800 err_out:
1801 		INP_WUNLOCK(inp);
1802 		return (error);
1803 
1804 	}
1805 
1806 	/* Pass in the INP locked, callee must unlock it. */
1807 	return (tp->t_fb->tfb_tcp_ctloutput(tp, sopt));
1808 }
1809 
1810 static int
1811 tcp_ctloutput_get(struct inpcb *inp, struct sockopt *sopt)
1812 {
1813 	struct socket *so = inp->inp_socket;
1814 	struct tcpcb *tp = intotcpcb(inp);
1815 	int error = 0;
1816 
1817 	MPASS(sopt->sopt_dir == SOPT_GET);
1818 	INP_WLOCK_ASSERT(inp);
1819 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1820 	    ("inp_flags == %x", inp->inp_flags));
1821 	KASSERT(so != NULL, ("inp_socket == NULL"));
1822 
1823 	if (sopt->sopt_level != IPPROTO_TCP) {
1824 		INP_WUNLOCK(inp);
1825 #ifdef INET6
1826 		if (inp->inp_vflag & INP_IPV6PROTO)
1827 			error = ip6_ctloutput(so, sopt);
1828 #endif /* INET6 */
1829 #if defined(INET6) && defined(INET)
1830 		else
1831 #endif
1832 #ifdef INET
1833 			error = ip_ctloutput(so, sopt);
1834 #endif
1835 		return (error);
1836 	}
1837 	if (((sopt->sopt_name == TCP_FUNCTION_BLK) ||
1838 	     (sopt->sopt_name == TCP_FUNCTION_ALIAS))) {
1839 		struct tcp_function_set fsn;
1840 
1841 		if (sopt->sopt_name == TCP_FUNCTION_ALIAS) {
1842 			memset(&fsn, 0, sizeof(fsn));
1843 			find_tcp_function_alias(tp->t_fb, &fsn);
1844 		} else {
1845 			strncpy(fsn.function_set_name,
1846 			    tp->t_fb->tfb_tcp_block_name,
1847 			    TCP_FUNCTION_NAME_LEN_MAX);
1848 			fsn.function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
1849 		}
1850 		fsn.pcbcnt = tp->t_fb->tfb_refcnt;
1851 		INP_WUNLOCK(inp);
1852 		error = sooptcopyout(sopt, &fsn, sizeof fsn);
1853 		return (error);
1854 	}
1855 
1856 	/* Pass in the INP locked, callee must unlock it. */
1857 	return (tp->t_fb->tfb_tcp_ctloutput(tp, sopt));
1858 }
1859 
1860 int
1861 tcp_ctloutput(struct socket *so, struct sockopt *sopt)
1862 {
1863 	struct	inpcb *inp;
1864 
1865 	inp = sotoinpcb(so);
1866 	KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL"));
1867 
1868 	INP_WLOCK(inp);
1869 	if (inp->inp_flags & INP_DROPPED) {
1870 		INP_WUNLOCK(inp);
1871 		return (ECONNRESET);
1872 	}
1873 	if (sopt->sopt_dir == SOPT_SET)
1874 		return (tcp_ctloutput_set(inp, sopt));
1875 	else if (sopt->sopt_dir == SOPT_GET)
1876 		return (tcp_ctloutput_get(inp, sopt));
1877 	else
1878 		panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
1879 }
1880 
1881 /*
1882  * If this assert becomes untrue, we need to change the size of the buf
1883  * variable in tcp_default_ctloutput().
1884  */
1885 #ifdef CTASSERT
1886 CTASSERT(TCP_CA_NAME_MAX <= TCP_LOG_ID_LEN);
1887 CTASSERT(TCP_LOG_REASON_LEN <= TCP_LOG_ID_LEN);
1888 #endif
1889 
1890 extern struct cc_algo newreno_cc_algo;
1891 
1892 static int
1893 tcp_set_cc_mod(struct inpcb *inp, struct sockopt *sopt)
1894 {
1895 	struct cc_algo *algo;
1896 	void *ptr = NULL;
1897 	struct tcpcb *tp;
1898 	struct cc_var cc_mem;
1899 	char	buf[TCP_CA_NAME_MAX];
1900 	size_t mem_sz;
1901 	int error;
1902 
1903 	INP_WUNLOCK(inp);
1904 	error = sooptcopyin(sopt, buf, TCP_CA_NAME_MAX - 1, 1);
1905 	if (error)
1906 		return(error);
1907 	buf[sopt->sopt_valsize] = '\0';
1908 	CC_LIST_RLOCK();
1909 	STAILQ_FOREACH(algo, &cc_list, entries) {
1910 		if (strncmp(buf, algo->name,
1911 			    TCP_CA_NAME_MAX) == 0) {
1912 			if (algo->flags & CC_MODULE_BEING_REMOVED) {
1913 				/* We can't "see" modules being unloaded */
1914 				continue;
1915 			}
1916 			break;
1917 		}
1918 	}
1919 	if (algo == NULL) {
1920 		CC_LIST_RUNLOCK();
1921 		return(ESRCH);
1922 	}
1923 	/*
1924 	 * With a reference the algorithm cannot be removed
1925 	 * so we hold a reference through the change process.
1926 	 */
1927 	cc_refer(algo);
1928 	CC_LIST_RUNLOCK();
1929 	if (algo->cb_init != NULL) {
1930 		/* We can now pre-get the memory for the CC */
1931 		mem_sz = (*algo->cc_data_sz)();
1932 		if (mem_sz == 0) {
1933 			goto no_mem_needed;
1934 		}
1935 		ptr = malloc(mem_sz, M_CC_MEM, M_WAITOK);
1936 	} else {
1937 no_mem_needed:
1938 		mem_sz = 0;
1939 		ptr = NULL;
1940 	}
1941 	/*
1942 	 * Make sure its all clean and zero and also get
1943 	 * back the inplock.
1944 	 */
1945 	memset(&cc_mem, 0, sizeof(cc_mem));
1946 	INP_WLOCK(inp);
1947 	if (inp->inp_flags & INP_DROPPED) {
1948 		INP_WUNLOCK(inp);
1949 		if (ptr)
1950 			free(ptr, M_CC_MEM);
1951 		/* Release our temp reference */
1952 		CC_LIST_RLOCK();
1953 		cc_release(algo);
1954 		CC_LIST_RUNLOCK();
1955 		return (ECONNRESET);
1956 	}
1957 	tp = intotcpcb(inp);
1958 	if (ptr != NULL)
1959 		memset(ptr, 0, mem_sz);
1960 	cc_mem.ccvc.tcp = tp;
1961 	/*
1962 	 * We once again hold a write lock over the tcb so it's
1963 	 * safe to do these things without ordering concerns.
1964 	 * Note here we init into stack memory.
1965 	 */
1966 	if (algo->cb_init != NULL)
1967 		error = algo->cb_init(&cc_mem, ptr);
1968 	else
1969 		error = 0;
1970 	/*
1971 	 * The CC algorithms, when given their memory
1972 	 * should not fail we could in theory have a
1973 	 * KASSERT here.
1974 	 */
1975 	if (error == 0) {
1976 		/*
1977 		 * Touchdown, lets go ahead and move the
1978 		 * connection to the new CC module by
1979 		 * copying in the cc_mem after we call
1980 		 * the old ones cleanup (if any).
1981 		 */
1982 		if (CC_ALGO(tp)->cb_destroy != NULL)
1983 			CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
1984 		/* Detach the old CC from the tcpcb  */
1985 		cc_detach(tp);
1986 		/* Copy in our temp memory that was inited */
1987 		memcpy(&tp->t_ccv, &cc_mem, sizeof(struct cc_var));
1988 		/* Now attach the new, which takes a reference */
1989 		cc_attach(tp, algo);
1990 		/* Ok now are we where we have gotten past any conn_init? */
1991 		if (TCPS_HAVEESTABLISHED(tp->t_state) && (CC_ALGO(tp)->conn_init != NULL)) {
1992 			/* Yep run the connection init for the new CC */
1993 			CC_ALGO(tp)->conn_init(&tp->t_ccv);
1994 		}
1995 	} else if (ptr)
1996 		free(ptr, M_CC_MEM);
1997 	INP_WUNLOCK(inp);
1998 	/* Now lets release our temp reference */
1999 	CC_LIST_RLOCK();
2000 	cc_release(algo);
2001 	CC_LIST_RUNLOCK();
2002 	return (error);
2003 }
2004 
2005 int
2006 tcp_default_ctloutput(struct tcpcb *tp, struct sockopt *sopt)
2007 {
2008 	struct inpcb *inp = tptoinpcb(tp);
2009 	int	error, opt, optval;
2010 	u_int	ui;
2011 	struct	tcp_info ti;
2012 #ifdef KERN_TLS
2013 	struct tls_enable tls;
2014 	struct socket *so = inp->inp_socket;
2015 #endif
2016 	char	*pbuf, buf[TCP_LOG_ID_LEN];
2017 #ifdef STATS
2018 	struct statsblob *sbp;
2019 #endif
2020 	size_t	len;
2021 
2022 	INP_WLOCK_ASSERT(inp);
2023 	KASSERT((inp->inp_flags & INP_DROPPED) == 0,
2024 	    ("inp_flags == %x", inp->inp_flags));
2025 	KASSERT(inp->inp_socket != NULL, ("inp_socket == NULL"));
2026 
2027 	switch (sopt->sopt_level) {
2028 #ifdef INET6
2029 	case IPPROTO_IPV6:
2030 		MPASS(inp->inp_vflag & INP_IPV6PROTO);
2031 		switch (sopt->sopt_name) {
2032 		case IPV6_USE_MIN_MTU:
2033 			tcp6_use_min_mtu(tp);
2034 			/* FALLTHROUGH */
2035 		}
2036 		INP_WUNLOCK(inp);
2037 		return (0);
2038 #endif
2039 #ifdef INET
2040 	case IPPROTO_IP:
2041 		INP_WUNLOCK(inp);
2042 		return (0);
2043 #endif
2044 	}
2045 
2046 	/*
2047 	 * For TCP_CCALGOOPT forward the control to CC module, for both
2048 	 * SOPT_SET and SOPT_GET.
2049 	 */
2050 	switch (sopt->sopt_name) {
2051 	case TCP_CCALGOOPT:
2052 		INP_WUNLOCK(inp);
2053 		if (sopt->sopt_valsize > CC_ALGOOPT_LIMIT)
2054 			return (EINVAL);
2055 		pbuf = malloc(sopt->sopt_valsize, M_TEMP, M_WAITOK | M_ZERO);
2056 		error = sooptcopyin(sopt, pbuf, sopt->sopt_valsize,
2057 		    sopt->sopt_valsize);
2058 		if (error) {
2059 			free(pbuf, M_TEMP);
2060 			return (error);
2061 		}
2062 		INP_WLOCK_RECHECK_CLEANUP(inp, free(pbuf, M_TEMP));
2063 		if (CC_ALGO(tp)->ctl_output != NULL)
2064 			error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, sopt, pbuf);
2065 		else
2066 			error = ENOENT;
2067 		INP_WUNLOCK(inp);
2068 		if (error == 0 && sopt->sopt_dir == SOPT_GET)
2069 			error = sooptcopyout(sopt, pbuf, sopt->sopt_valsize);
2070 		free(pbuf, M_TEMP);
2071 		return (error);
2072 	}
2073 
2074 	switch (sopt->sopt_dir) {
2075 	case SOPT_SET:
2076 		switch (sopt->sopt_name) {
2077 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2078 		case TCP_MD5SIG:
2079 			INP_WUNLOCK(inp);
2080 			if (!TCPMD5_ENABLED())
2081 				return (ENOPROTOOPT);
2082 			error = TCPMD5_PCBCTL(inp, sopt);
2083 			if (error)
2084 				return (error);
2085 			INP_WLOCK_RECHECK(inp);
2086 			goto unlock_and_done;
2087 #endif /* IPSEC */
2088 
2089 		case TCP_NODELAY:
2090 		case TCP_NOOPT:
2091 			INP_WUNLOCK(inp);
2092 			error = sooptcopyin(sopt, &optval, sizeof optval,
2093 			    sizeof optval);
2094 			if (error)
2095 				return (error);
2096 
2097 			INP_WLOCK_RECHECK(inp);
2098 			switch (sopt->sopt_name) {
2099 			case TCP_NODELAY:
2100 				opt = TF_NODELAY;
2101 				break;
2102 			case TCP_NOOPT:
2103 				opt = TF_NOOPT;
2104 				break;
2105 			default:
2106 				opt = 0; /* dead code to fool gcc */
2107 				break;
2108 			}
2109 
2110 			if (optval)
2111 				tp->t_flags |= opt;
2112 			else
2113 				tp->t_flags &= ~opt;
2114 unlock_and_done:
2115 #ifdef TCP_OFFLOAD
2116 			if (tp->t_flags & TF_TOE) {
2117 				tcp_offload_ctloutput(tp, sopt->sopt_dir,
2118 				    sopt->sopt_name);
2119 			}
2120 #endif
2121 			INP_WUNLOCK(inp);
2122 			break;
2123 
2124 		case TCP_NOPUSH:
2125 			INP_WUNLOCK(inp);
2126 			error = sooptcopyin(sopt, &optval, sizeof optval,
2127 			    sizeof optval);
2128 			if (error)
2129 				return (error);
2130 
2131 			INP_WLOCK_RECHECK(inp);
2132 			if (optval)
2133 				tp->t_flags |= TF_NOPUSH;
2134 			else if (tp->t_flags & TF_NOPUSH) {
2135 				tp->t_flags &= ~TF_NOPUSH;
2136 				if (TCPS_HAVEESTABLISHED(tp->t_state)) {
2137 					struct epoch_tracker et;
2138 
2139 					NET_EPOCH_ENTER(et);
2140 					error = tcp_output_nodrop(tp);
2141 					NET_EPOCH_EXIT(et);
2142 				}
2143 			}
2144 			goto unlock_and_done;
2145 
2146 		case TCP_REMOTE_UDP_ENCAPS_PORT:
2147 			INP_WUNLOCK(inp);
2148 			error = sooptcopyin(sopt, &optval, sizeof optval,
2149 			    sizeof optval);
2150 			if (error)
2151 				return (error);
2152 			if ((optval < TCP_TUNNELING_PORT_MIN) ||
2153 			    (optval > TCP_TUNNELING_PORT_MAX)) {
2154 				/* Its got to be in range */
2155 				return (EINVAL);
2156 			}
2157 			if ((V_tcp_udp_tunneling_port == 0) && (optval != 0)) {
2158 				/* You have to have enabled a UDP tunneling port first */
2159 				return (EINVAL);
2160 			}
2161 			INP_WLOCK_RECHECK(inp);
2162 			if (tp->t_state != TCPS_CLOSED) {
2163 				/* You can't change after you are connected */
2164 				error = EINVAL;
2165 			} else {
2166 				/* Ok we are all good set the port */
2167 				tp->t_port = htons(optval);
2168 			}
2169 			goto unlock_and_done;
2170 
2171 		case TCP_MAXSEG:
2172 			INP_WUNLOCK(inp);
2173 			error = sooptcopyin(sopt, &optval, sizeof optval,
2174 			    sizeof optval);
2175 			if (error)
2176 				return (error);
2177 
2178 			INP_WLOCK_RECHECK(inp);
2179 			if (optval > 0 && optval <= tp->t_maxseg &&
2180 			    optval + 40 >= V_tcp_minmss) {
2181 				tp->t_maxseg = optval;
2182 				if (tp->t_maxseg < V_tcp_mssdflt) {
2183 					/*
2184 					 * The MSS is so small we should not process incoming
2185 					 * SACK's since we are subject to attack in such a
2186 					 * case.
2187 					 */
2188 					tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
2189 				} else {
2190 					tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
2191 				}
2192 			} else
2193 				error = EINVAL;
2194 			goto unlock_and_done;
2195 
2196 		case TCP_INFO:
2197 			INP_WUNLOCK(inp);
2198 			error = EINVAL;
2199 			break;
2200 
2201 		case TCP_STATS:
2202 			INP_WUNLOCK(inp);
2203 #ifdef STATS
2204 			error = sooptcopyin(sopt, &optval, sizeof optval,
2205 			    sizeof optval);
2206 			if (error)
2207 				return (error);
2208 
2209 			if (optval > 0)
2210 				sbp = stats_blob_alloc(
2211 				    V_tcp_perconn_stats_dflt_tpl, 0);
2212 			else
2213 				sbp = NULL;
2214 
2215 			INP_WLOCK_RECHECK(inp);
2216 			if ((tp->t_stats != NULL && sbp == NULL) ||
2217 			    (tp->t_stats == NULL && sbp != NULL)) {
2218 				struct statsblob *t = tp->t_stats;
2219 				tp->t_stats = sbp;
2220 				sbp = t;
2221 			}
2222 			INP_WUNLOCK(inp);
2223 
2224 			stats_blob_destroy(sbp);
2225 #else
2226 			return (EOPNOTSUPP);
2227 #endif /* !STATS */
2228 			break;
2229 
2230 		case TCP_CONGESTION:
2231 			error = tcp_set_cc_mod(inp, sopt);
2232 			break;
2233 
2234 		case TCP_REUSPORT_LB_NUMA:
2235 			INP_WUNLOCK(inp);
2236 			error = sooptcopyin(sopt, &optval, sizeof(optval),
2237 			    sizeof(optval));
2238 			INP_WLOCK_RECHECK(inp);
2239 			if (!error)
2240 				error = in_pcblbgroup_numa(inp, optval);
2241 			INP_WUNLOCK(inp);
2242 			break;
2243 
2244 #ifdef KERN_TLS
2245 		case TCP_TXTLS_ENABLE:
2246 			INP_WUNLOCK(inp);
2247 			error = ktls_copyin_tls_enable(sopt, &tls);
2248 			if (error != 0)
2249 				break;
2250 			error = ktls_enable_tx(so, &tls);
2251 			ktls_cleanup_tls_enable(&tls);
2252 			break;
2253 		case TCP_TXTLS_MODE:
2254 			INP_WUNLOCK(inp);
2255 			error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2256 			if (error != 0)
2257 				return (error);
2258 
2259 			INP_WLOCK_RECHECK(inp);
2260 			error = ktls_set_tx_mode(so, ui);
2261 			INP_WUNLOCK(inp);
2262 			break;
2263 		case TCP_RXTLS_ENABLE:
2264 			INP_WUNLOCK(inp);
2265 			error = ktls_copyin_tls_enable(sopt, &tls);
2266 			if (error != 0)
2267 				break;
2268 			error = ktls_enable_rx(so, &tls);
2269 			ktls_cleanup_tls_enable(&tls);
2270 			break;
2271 #endif
2272 		case TCP_MAXUNACKTIME:
2273 		case TCP_KEEPIDLE:
2274 		case TCP_KEEPINTVL:
2275 		case TCP_KEEPINIT:
2276 			INP_WUNLOCK(inp);
2277 			error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2278 			if (error)
2279 				return (error);
2280 
2281 			if (ui > (UINT_MAX / hz)) {
2282 				error = EINVAL;
2283 				break;
2284 			}
2285 			ui *= hz;
2286 
2287 			INP_WLOCK_RECHECK(inp);
2288 			switch (sopt->sopt_name) {
2289 			case TCP_MAXUNACKTIME:
2290 				tp->t_maxunacktime = ui;
2291 				break;
2292 
2293 			case TCP_KEEPIDLE:
2294 				tp->t_keepidle = ui;
2295 				/*
2296 				 * XXX: better check current remaining
2297 				 * timeout and "merge" it with new value.
2298 				 */
2299 				if ((tp->t_state > TCPS_LISTEN) &&
2300 				    (tp->t_state <= TCPS_CLOSING))
2301 					tcp_timer_activate(tp, TT_KEEP,
2302 					    TP_KEEPIDLE(tp));
2303 				break;
2304 			case TCP_KEEPINTVL:
2305 				tp->t_keepintvl = ui;
2306 				if ((tp->t_state == TCPS_FIN_WAIT_2) &&
2307 				    (TP_MAXIDLE(tp) > 0))
2308 					tcp_timer_activate(tp, TT_2MSL,
2309 					    TP_MAXIDLE(tp));
2310 				break;
2311 			case TCP_KEEPINIT:
2312 				tp->t_keepinit = ui;
2313 				if (tp->t_state == TCPS_SYN_RECEIVED ||
2314 				    tp->t_state == TCPS_SYN_SENT)
2315 					tcp_timer_activate(tp, TT_KEEP,
2316 					    TP_KEEPINIT(tp));
2317 				break;
2318 			}
2319 			goto unlock_and_done;
2320 
2321 		case TCP_KEEPCNT:
2322 			INP_WUNLOCK(inp);
2323 			error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2324 			if (error)
2325 				return (error);
2326 
2327 			INP_WLOCK_RECHECK(inp);
2328 			tp->t_keepcnt = ui;
2329 			if ((tp->t_state == TCPS_FIN_WAIT_2) &&
2330 			    (TP_MAXIDLE(tp) > 0))
2331 				tcp_timer_activate(tp, TT_2MSL,
2332 				    TP_MAXIDLE(tp));
2333 			goto unlock_and_done;
2334 
2335 #ifdef TCPPCAP
2336 		case TCP_PCAP_OUT:
2337 		case TCP_PCAP_IN:
2338 			INP_WUNLOCK(inp);
2339 			error = sooptcopyin(sopt, &optval, sizeof optval,
2340 			    sizeof optval);
2341 			if (error)
2342 				return (error);
2343 
2344 			INP_WLOCK_RECHECK(inp);
2345 			if (optval >= 0)
2346 				tcp_pcap_set_sock_max(
2347 					(sopt->sopt_name == TCP_PCAP_OUT) ?
2348 					&(tp->t_outpkts) : &(tp->t_inpkts),
2349 					optval);
2350 			else
2351 				error = EINVAL;
2352 			goto unlock_and_done;
2353 #endif
2354 
2355 		case TCP_FASTOPEN: {
2356 			struct tcp_fastopen tfo_optval;
2357 
2358 			INP_WUNLOCK(inp);
2359 			if (!V_tcp_fastopen_client_enable &&
2360 			    !V_tcp_fastopen_server_enable)
2361 				return (EPERM);
2362 
2363 			error = sooptcopyin(sopt, &tfo_optval,
2364 				    sizeof(tfo_optval), sizeof(int));
2365 			if (error)
2366 				return (error);
2367 
2368 			INP_WLOCK_RECHECK(inp);
2369 			if ((tp->t_state != TCPS_CLOSED) &&
2370 			    (tp->t_state != TCPS_LISTEN)) {
2371 				error = EINVAL;
2372 				goto unlock_and_done;
2373 			}
2374 			if (tfo_optval.enable) {
2375 				if (tp->t_state == TCPS_LISTEN) {
2376 					if (!V_tcp_fastopen_server_enable) {
2377 						error = EPERM;
2378 						goto unlock_and_done;
2379 					}
2380 
2381 					if (tp->t_tfo_pending == NULL)
2382 						tp->t_tfo_pending =
2383 						    tcp_fastopen_alloc_counter();
2384 				} else {
2385 					/*
2386 					 * If a pre-shared key was provided,
2387 					 * stash it in the client cookie
2388 					 * field of the tcpcb for use during
2389 					 * connect.
2390 					 */
2391 					if (sopt->sopt_valsize ==
2392 					    sizeof(tfo_optval)) {
2393 						memcpy(tp->t_tfo_cookie.client,
2394 						       tfo_optval.psk,
2395 						       TCP_FASTOPEN_PSK_LEN);
2396 						tp->t_tfo_client_cookie_len =
2397 						    TCP_FASTOPEN_PSK_LEN;
2398 					}
2399 				}
2400 				tp->t_flags |= TF_FASTOPEN;
2401 			} else
2402 				tp->t_flags &= ~TF_FASTOPEN;
2403 			goto unlock_and_done;
2404 		}
2405 
2406 #ifdef TCP_BLACKBOX
2407 		case TCP_LOG:
2408 			INP_WUNLOCK(inp);
2409 			error = sooptcopyin(sopt, &optval, sizeof optval,
2410 			    sizeof optval);
2411 			if (error)
2412 				return (error);
2413 
2414 			INP_WLOCK_RECHECK(inp);
2415 			error = tcp_log_state_change(tp, optval);
2416 			goto unlock_and_done;
2417 
2418 		case TCP_LOGBUF:
2419 			INP_WUNLOCK(inp);
2420 			error = EINVAL;
2421 			break;
2422 
2423 		case TCP_LOGID:
2424 			INP_WUNLOCK(inp);
2425 			error = sooptcopyin(sopt, buf, TCP_LOG_ID_LEN - 1, 0);
2426 			if (error)
2427 				break;
2428 			buf[sopt->sopt_valsize] = '\0';
2429 			INP_WLOCK_RECHECK(inp);
2430 			error = tcp_log_set_id(tp, buf);
2431 			/* tcp_log_set_id() unlocks the INP. */
2432 			break;
2433 
2434 		case TCP_LOGDUMP:
2435 		case TCP_LOGDUMPID:
2436 			INP_WUNLOCK(inp);
2437 			error =
2438 			    sooptcopyin(sopt, buf, TCP_LOG_REASON_LEN - 1, 0);
2439 			if (error)
2440 				break;
2441 			buf[sopt->sopt_valsize] = '\0';
2442 			INP_WLOCK_RECHECK(inp);
2443 			if (sopt->sopt_name == TCP_LOGDUMP) {
2444 				error = tcp_log_dump_tp_logbuf(tp, buf,
2445 				    M_WAITOK, true);
2446 				INP_WUNLOCK(inp);
2447 			} else {
2448 				tcp_log_dump_tp_bucket_logbufs(tp, buf);
2449 				/*
2450 				 * tcp_log_dump_tp_bucket_logbufs() drops the
2451 				 * INP lock.
2452 				 */
2453 			}
2454 			break;
2455 #endif
2456 
2457 		default:
2458 			INP_WUNLOCK(inp);
2459 			error = ENOPROTOOPT;
2460 			break;
2461 		}
2462 		break;
2463 
2464 	case SOPT_GET:
2465 		tp = intotcpcb(inp);
2466 		switch (sopt->sopt_name) {
2467 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2468 		case TCP_MD5SIG:
2469 			INP_WUNLOCK(inp);
2470 			if (!TCPMD5_ENABLED())
2471 				return (ENOPROTOOPT);
2472 			error = TCPMD5_PCBCTL(inp, sopt);
2473 			break;
2474 #endif
2475 
2476 		case TCP_NODELAY:
2477 			optval = tp->t_flags & TF_NODELAY;
2478 			INP_WUNLOCK(inp);
2479 			error = sooptcopyout(sopt, &optval, sizeof optval);
2480 			break;
2481 		case TCP_MAXSEG:
2482 			optval = tp->t_maxseg;
2483 			INP_WUNLOCK(inp);
2484 			error = sooptcopyout(sopt, &optval, sizeof optval);
2485 			break;
2486 		case TCP_REMOTE_UDP_ENCAPS_PORT:
2487 			optval = ntohs(tp->t_port);
2488 			INP_WUNLOCK(inp);
2489 			error = sooptcopyout(sopt, &optval, sizeof optval);
2490 			break;
2491 		case TCP_NOOPT:
2492 			optval = tp->t_flags & TF_NOOPT;
2493 			INP_WUNLOCK(inp);
2494 			error = sooptcopyout(sopt, &optval, sizeof optval);
2495 			break;
2496 		case TCP_NOPUSH:
2497 			optval = tp->t_flags & TF_NOPUSH;
2498 			INP_WUNLOCK(inp);
2499 			error = sooptcopyout(sopt, &optval, sizeof optval);
2500 			break;
2501 		case TCP_INFO:
2502 			tcp_fill_info(tp, &ti);
2503 			INP_WUNLOCK(inp);
2504 			error = sooptcopyout(sopt, &ti, sizeof ti);
2505 			break;
2506 		case TCP_STATS:
2507 			{
2508 #ifdef STATS
2509 			int nheld;
2510 			TYPEOF_MEMBER(struct statsblob, flags) sbflags = 0;
2511 
2512 			error = 0;
2513 			socklen_t outsbsz = sopt->sopt_valsize;
2514 			if (tp->t_stats == NULL)
2515 				error = ENOENT;
2516 			else if (outsbsz >= tp->t_stats->cursz)
2517 				outsbsz = tp->t_stats->cursz;
2518 			else if (outsbsz >= sizeof(struct statsblob))
2519 				outsbsz = sizeof(struct statsblob);
2520 			else
2521 				error = EINVAL;
2522 			INP_WUNLOCK(inp);
2523 			if (error)
2524 				break;
2525 
2526 			sbp = sopt->sopt_val;
2527 			nheld = atop(round_page(((vm_offset_t)sbp) +
2528 			    (vm_size_t)outsbsz) - trunc_page((vm_offset_t)sbp));
2529 			vm_page_t ma[nheld];
2530 			if (vm_fault_quick_hold_pages(
2531 			    &curproc->p_vmspace->vm_map, (vm_offset_t)sbp,
2532 			    outsbsz, VM_PROT_READ | VM_PROT_WRITE, ma,
2533 			    nheld) < 0) {
2534 				error = EFAULT;
2535 				break;
2536 			}
2537 
2538 			if ((error = copyin_nofault(&(sbp->flags), &sbflags,
2539 			    SIZEOF_MEMBER(struct statsblob, flags))))
2540 				goto unhold;
2541 
2542 			INP_WLOCK_RECHECK(inp);
2543 			error = stats_blob_snapshot(&sbp, outsbsz, tp->t_stats,
2544 			    sbflags | SB_CLONE_USRDSTNOFAULT);
2545 			INP_WUNLOCK(inp);
2546 			sopt->sopt_valsize = outsbsz;
2547 unhold:
2548 			vm_page_unhold_pages(ma, nheld);
2549 #else
2550 			INP_WUNLOCK(inp);
2551 			error = EOPNOTSUPP;
2552 #endif /* !STATS */
2553 			break;
2554 			}
2555 		case TCP_CONGESTION:
2556 			len = strlcpy(buf, CC_ALGO(tp)->name, TCP_CA_NAME_MAX);
2557 			INP_WUNLOCK(inp);
2558 			error = sooptcopyout(sopt, buf, len + 1);
2559 			break;
2560 		case TCP_MAXUNACKTIME:
2561 		case TCP_KEEPIDLE:
2562 		case TCP_KEEPINTVL:
2563 		case TCP_KEEPINIT:
2564 		case TCP_KEEPCNT:
2565 			switch (sopt->sopt_name) {
2566 			case TCP_MAXUNACKTIME:
2567 				ui = TP_MAXUNACKTIME(tp) / hz;
2568 				break;
2569 			case TCP_KEEPIDLE:
2570 				ui = TP_KEEPIDLE(tp) / hz;
2571 				break;
2572 			case TCP_KEEPINTVL:
2573 				ui = TP_KEEPINTVL(tp) / hz;
2574 				break;
2575 			case TCP_KEEPINIT:
2576 				ui = TP_KEEPINIT(tp) / hz;
2577 				break;
2578 			case TCP_KEEPCNT:
2579 				ui = TP_KEEPCNT(tp);
2580 				break;
2581 			}
2582 			INP_WUNLOCK(inp);
2583 			error = sooptcopyout(sopt, &ui, sizeof(ui));
2584 			break;
2585 #ifdef TCPPCAP
2586 		case TCP_PCAP_OUT:
2587 		case TCP_PCAP_IN:
2588 			optval = tcp_pcap_get_sock_max(
2589 					(sopt->sopt_name == TCP_PCAP_OUT) ?
2590 					&(tp->t_outpkts) : &(tp->t_inpkts));
2591 			INP_WUNLOCK(inp);
2592 			error = sooptcopyout(sopt, &optval, sizeof optval);
2593 			break;
2594 #endif
2595 		case TCP_FASTOPEN:
2596 			optval = tp->t_flags & TF_FASTOPEN;
2597 			INP_WUNLOCK(inp);
2598 			error = sooptcopyout(sopt, &optval, sizeof optval);
2599 			break;
2600 #ifdef TCP_BLACKBOX
2601 		case TCP_LOG:
2602 			optval = tcp_get_bblog_state(tp);
2603 			INP_WUNLOCK(inp);
2604 			error = sooptcopyout(sopt, &optval, sizeof(optval));
2605 			break;
2606 		case TCP_LOGBUF:
2607 			/* tcp_log_getlogbuf() does INP_WUNLOCK(inp) */
2608 			error = tcp_log_getlogbuf(sopt, tp);
2609 			break;
2610 		case TCP_LOGID:
2611 			len = tcp_log_get_id(tp, buf);
2612 			INP_WUNLOCK(inp);
2613 			error = sooptcopyout(sopt, buf, len + 1);
2614 			break;
2615 		case TCP_LOGDUMP:
2616 		case TCP_LOGDUMPID:
2617 			INP_WUNLOCK(inp);
2618 			error = EINVAL;
2619 			break;
2620 #endif
2621 #ifdef KERN_TLS
2622 		case TCP_TXTLS_MODE:
2623 			error = ktls_get_tx_mode(so, &optval);
2624 			INP_WUNLOCK(inp);
2625 			if (error == 0)
2626 				error = sooptcopyout(sopt, &optval,
2627 				    sizeof(optval));
2628 			break;
2629 		case TCP_RXTLS_MODE:
2630 			error = ktls_get_rx_mode(so, &optval);
2631 			INP_WUNLOCK(inp);
2632 			if (error == 0)
2633 				error = sooptcopyout(sopt, &optval,
2634 				    sizeof(optval));
2635 			break;
2636 #endif
2637 		default:
2638 			INP_WUNLOCK(inp);
2639 			error = ENOPROTOOPT;
2640 			break;
2641 		}
2642 		break;
2643 	}
2644 	return (error);
2645 }
2646 #undef INP_WLOCK_RECHECK
2647 #undef INP_WLOCK_RECHECK_CLEANUP
2648 
2649 /*
2650  * Initiate (or continue) disconnect.
2651  * If embryonic state, just send reset (once).
2652  * If in ``let data drain'' option and linger null, just drop.
2653  * Otherwise (hard), mark socket disconnecting and drop
2654  * current input data; switch states based on user close, and
2655  * send segment to peer (with FIN).
2656  */
2657 static void
2658 tcp_disconnect(struct tcpcb *tp)
2659 {
2660 	struct inpcb *inp = tptoinpcb(tp);
2661 	struct socket *so = tptosocket(tp);
2662 
2663 	NET_EPOCH_ASSERT();
2664 	INP_WLOCK_ASSERT(inp);
2665 
2666 	/*
2667 	 * Neither tcp_close() nor tcp_drop() should return NULL, as the
2668 	 * socket is still open.
2669 	 */
2670 	if (tp->t_state < TCPS_ESTABLISHED &&
2671 	    !(tp->t_state > TCPS_LISTEN && (tp->t_flags & TF_FASTOPEN))) {
2672 		tp = tcp_close(tp);
2673 		KASSERT(tp != NULL,
2674 		    ("tcp_disconnect: tcp_close() returned NULL"));
2675 	} else if ((so->so_options & SO_LINGER) && so->so_linger == 0) {
2676 		tp = tcp_drop(tp, 0);
2677 		KASSERT(tp != NULL,
2678 		    ("tcp_disconnect: tcp_drop() returned NULL"));
2679 	} else {
2680 		soisdisconnecting(so);
2681 		sbflush(&so->so_rcv);
2682 		tcp_usrclosed(tp);
2683 		if (!(inp->inp_flags & INP_DROPPED))
2684 			/* Ignore stack's drop request, we already at it. */
2685 			(void)tcp_output_nodrop(tp);
2686 	}
2687 }
2688 
2689 /*
2690  * User issued close, and wish to trail through shutdown states:
2691  * if never received SYN, just forget it.  If got a SYN from peer,
2692  * but haven't sent FIN, then go to FIN_WAIT_1 state to send peer a FIN.
2693  * If already got a FIN from peer, then almost done; go to LAST_ACK
2694  * state.  In all other cases, have already sent FIN to peer (e.g.
2695  * after PRU_SHUTDOWN), and just have to play tedious game waiting
2696  * for peer to send FIN or not respond to keep-alives, etc.
2697  * We can let the user exit from the close as soon as the FIN is acked.
2698  */
2699 static void
2700 tcp_usrclosed(struct tcpcb *tp)
2701 {
2702 
2703 	NET_EPOCH_ASSERT();
2704 	INP_WLOCK_ASSERT(tptoinpcb(tp));
2705 
2706 	switch (tp->t_state) {
2707 	case TCPS_LISTEN:
2708 #ifdef TCP_OFFLOAD
2709 		tcp_offload_listen_stop(tp);
2710 #endif
2711 		tcp_state_change(tp, TCPS_CLOSED);
2712 		/* FALLTHROUGH */
2713 	case TCPS_CLOSED:
2714 		tp = tcp_close(tp);
2715 		/*
2716 		 * tcp_close() should never return NULL here as the socket is
2717 		 * still open.
2718 		 */
2719 		KASSERT(tp != NULL,
2720 		    ("tcp_usrclosed: tcp_close() returned NULL"));
2721 		break;
2722 
2723 	case TCPS_SYN_SENT:
2724 	case TCPS_SYN_RECEIVED:
2725 		tp->t_flags |= TF_NEEDFIN;
2726 		break;
2727 
2728 	case TCPS_ESTABLISHED:
2729 		tcp_state_change(tp, TCPS_FIN_WAIT_1);
2730 		break;
2731 
2732 	case TCPS_CLOSE_WAIT:
2733 		tcp_state_change(tp, TCPS_LAST_ACK);
2734 		break;
2735 	}
2736 	if (tp->t_acktime == 0)
2737 		tp->t_acktime = ticks;
2738 	if (tp->t_state >= TCPS_FIN_WAIT_2) {
2739 		tcp_free_sackholes(tp);
2740 		soisdisconnected(tptosocket(tp));
2741 		/* Prevent the connection hanging in FIN_WAIT_2 forever. */
2742 		if (tp->t_state == TCPS_FIN_WAIT_2) {
2743 			int timeout;
2744 
2745 			timeout = (tcp_fast_finwait2_recycle) ?
2746 			    tcp_finwait2_timeout : TP_MAXIDLE(tp);
2747 			tcp_timer_activate(tp, TT_2MSL, timeout);
2748 		}
2749 	}
2750 }
2751 
2752 #ifdef DDB
2753 static void
2754 db_print_indent(int indent)
2755 {
2756 	int i;
2757 
2758 	for (i = 0; i < indent; i++)
2759 		db_printf(" ");
2760 }
2761 
2762 static void
2763 db_print_tstate(int t_state)
2764 {
2765 
2766 	switch (t_state) {
2767 	case TCPS_CLOSED:
2768 		db_printf("TCPS_CLOSED");
2769 		return;
2770 
2771 	case TCPS_LISTEN:
2772 		db_printf("TCPS_LISTEN");
2773 		return;
2774 
2775 	case TCPS_SYN_SENT:
2776 		db_printf("TCPS_SYN_SENT");
2777 		return;
2778 
2779 	case TCPS_SYN_RECEIVED:
2780 		db_printf("TCPS_SYN_RECEIVED");
2781 		return;
2782 
2783 	case TCPS_ESTABLISHED:
2784 		db_printf("TCPS_ESTABLISHED");
2785 		return;
2786 
2787 	case TCPS_CLOSE_WAIT:
2788 		db_printf("TCPS_CLOSE_WAIT");
2789 		return;
2790 
2791 	case TCPS_FIN_WAIT_1:
2792 		db_printf("TCPS_FIN_WAIT_1");
2793 		return;
2794 
2795 	case TCPS_CLOSING:
2796 		db_printf("TCPS_CLOSING");
2797 		return;
2798 
2799 	case TCPS_LAST_ACK:
2800 		db_printf("TCPS_LAST_ACK");
2801 		return;
2802 
2803 	case TCPS_FIN_WAIT_2:
2804 		db_printf("TCPS_FIN_WAIT_2");
2805 		return;
2806 
2807 	case TCPS_TIME_WAIT:
2808 		db_printf("TCPS_TIME_WAIT");
2809 		return;
2810 
2811 	default:
2812 		db_printf("unknown");
2813 		return;
2814 	}
2815 }
2816 
2817 static void
2818 db_print_tflags(u_int t_flags)
2819 {
2820 	int comma;
2821 
2822 	comma = 0;
2823 	if (t_flags & TF_ACKNOW) {
2824 		db_printf("%sTF_ACKNOW", comma ? ", " : "");
2825 		comma = 1;
2826 	}
2827 	if (t_flags & TF_DELACK) {
2828 		db_printf("%sTF_DELACK", comma ? ", " : "");
2829 		comma = 1;
2830 	}
2831 	if (t_flags & TF_NODELAY) {
2832 		db_printf("%sTF_NODELAY", comma ? ", " : "");
2833 		comma = 1;
2834 	}
2835 	if (t_flags & TF_NOOPT) {
2836 		db_printf("%sTF_NOOPT", comma ? ", " : "");
2837 		comma = 1;
2838 	}
2839 	if (t_flags & TF_SENTFIN) {
2840 		db_printf("%sTF_SENTFIN", comma ? ", " : "");
2841 		comma = 1;
2842 	}
2843 	if (t_flags & TF_REQ_SCALE) {
2844 		db_printf("%sTF_REQ_SCALE", comma ? ", " : "");
2845 		comma = 1;
2846 	}
2847 	if (t_flags & TF_RCVD_SCALE) {
2848 		db_printf("%sTF_RECVD_SCALE", comma ? ", " : "");
2849 		comma = 1;
2850 	}
2851 	if (t_flags & TF_REQ_TSTMP) {
2852 		db_printf("%sTF_REQ_TSTMP", comma ? ", " : "");
2853 		comma = 1;
2854 	}
2855 	if (t_flags & TF_RCVD_TSTMP) {
2856 		db_printf("%sTF_RCVD_TSTMP", comma ? ", " : "");
2857 		comma = 1;
2858 	}
2859 	if (t_flags & TF_SACK_PERMIT) {
2860 		db_printf("%sTF_SACK_PERMIT", comma ? ", " : "");
2861 		comma = 1;
2862 	}
2863 	if (t_flags & TF_NEEDSYN) {
2864 		db_printf("%sTF_NEEDSYN", comma ? ", " : "");
2865 		comma = 1;
2866 	}
2867 	if (t_flags & TF_NEEDFIN) {
2868 		db_printf("%sTF_NEEDFIN", comma ? ", " : "");
2869 		comma = 1;
2870 	}
2871 	if (t_flags & TF_NOPUSH) {
2872 		db_printf("%sTF_NOPUSH", comma ? ", " : "");
2873 		comma = 1;
2874 	}
2875 	if (t_flags & TF_PREVVALID) {
2876 		db_printf("%sTF_PREVVALID", comma ? ", " : "");
2877 		comma = 1;
2878 	}
2879 	if (t_flags & TF_MORETOCOME) {
2880 		db_printf("%sTF_MORETOCOME", comma ? ", " : "");
2881 		comma = 1;
2882 	}
2883 	if (t_flags & TF_SONOTCONN) {
2884 		db_printf("%sTF_SONOTCONN", comma ? ", " : "");
2885 		comma = 1;
2886 	}
2887 	if (t_flags & TF_LASTIDLE) {
2888 		db_printf("%sTF_LASTIDLE", comma ? ", " : "");
2889 		comma = 1;
2890 	}
2891 	if (t_flags & TF_RXWIN0SENT) {
2892 		db_printf("%sTF_RXWIN0SENT", comma ? ", " : "");
2893 		comma = 1;
2894 	}
2895 	if (t_flags & TF_FASTRECOVERY) {
2896 		db_printf("%sTF_FASTRECOVERY", comma ? ", " : "");
2897 		comma = 1;
2898 	}
2899 	if (t_flags & TF_CONGRECOVERY) {
2900 		db_printf("%sTF_CONGRECOVERY", comma ? ", " : "");
2901 		comma = 1;
2902 	}
2903 	if (t_flags & TF_WASFRECOVERY) {
2904 		db_printf("%sTF_WASFRECOVERY", comma ? ", " : "");
2905 		comma = 1;
2906 	}
2907 	if (t_flags & TF_WASCRECOVERY) {
2908 		db_printf("%sTF_WASCRECOVERY", comma ? ", " : "");
2909 		comma = 1;
2910 	}
2911 	if (t_flags & TF_SIGNATURE) {
2912 		db_printf("%sTF_SIGNATURE", comma ? ", " : "");
2913 		comma = 1;
2914 	}
2915 	if (t_flags & TF_FORCEDATA) {
2916 		db_printf("%sTF_FORCEDATA", comma ? ", " : "");
2917 		comma = 1;
2918 	}
2919 	if (t_flags & TF_TSO) {
2920 		db_printf("%sTF_TSO", comma ? ", " : "");
2921 		comma = 1;
2922 	}
2923 	if (t_flags & TF_FASTOPEN) {
2924 		db_printf("%sTF_FASTOPEN", comma ? ", " : "");
2925 		comma = 1;
2926 	}
2927 }
2928 
2929 static void
2930 db_print_tflags2(u_int t_flags2)
2931 {
2932 	int comma;
2933 
2934 	comma = 0;
2935 	if (t_flags2 & TF2_PLPMTU_BLACKHOLE) {
2936 		db_printf("%sTF2_PLPMTU_BLACKHOLE", comma ? ", " : "");
2937 		comma = 1;
2938 	}
2939 	if (t_flags2 & TF2_PLPMTU_PMTUD) {
2940 		db_printf("%sTF2_PLPMTU_PMTUD", comma ? ", " : "");
2941 		comma = 1;
2942 	}
2943 	if (t_flags2 & TF2_PLPMTU_MAXSEGSNT) {
2944 		db_printf("%sTF2_PLPMTU_MAXSEGSNT", comma ? ", " : "");
2945 		comma = 1;
2946 	}
2947 	if (t_flags2 & TF2_LOG_AUTO) {
2948 		db_printf("%sTF2_LOG_AUTO", comma ? ", " : "");
2949 		comma = 1;
2950 	}
2951 	if (t_flags2 & TF2_DROP_AF_DATA) {
2952 		db_printf("%sTF2_DROP_AF_DATA", comma ? ", " : "");
2953 		comma = 1;
2954 	}
2955 	if (t_flags2 & TF2_ECN_PERMIT) {
2956 		db_printf("%sTF2_ECN_PERMIT", comma ? ", " : "");
2957 		comma = 1;
2958 	}
2959 	if (t_flags2 & TF2_ECN_SND_CWR) {
2960 		db_printf("%sTF2_ECN_SND_CWR", comma ? ", " : "");
2961 		comma = 1;
2962 	}
2963 	if (t_flags2 & TF2_ECN_SND_ECE) {
2964 		db_printf("%sTF2_ECN_SND_ECE", comma ? ", " : "");
2965 		comma = 1;
2966 	}
2967 	if (t_flags2 & TF2_ACE_PERMIT) {
2968 		db_printf("%sTF2_ACE_PERMIT", comma ? ", " : "");
2969 		comma = 1;
2970 	}
2971 	if (t_flags2 & TF2_FBYTES_COMPLETE) {
2972 		db_printf("%sTF2_FBYTES_COMPLETE", comma ? ", " : "");
2973 		comma = 1;
2974 	}
2975 }
2976 
2977 static void
2978 db_print_toobflags(char t_oobflags)
2979 {
2980 	int comma;
2981 
2982 	comma = 0;
2983 	if (t_oobflags & TCPOOB_HAVEDATA) {
2984 		db_printf("%sTCPOOB_HAVEDATA", comma ? ", " : "");
2985 		comma = 1;
2986 	}
2987 	if (t_oobflags & TCPOOB_HADDATA) {
2988 		db_printf("%sTCPOOB_HADDATA", comma ? ", " : "");
2989 		comma = 1;
2990 	}
2991 }
2992 
2993 static void
2994 db_print_tcpcb(struct tcpcb *tp, const char *name, int indent)
2995 {
2996 
2997 	db_print_indent(indent);
2998 	db_printf("%s at %p\n", name, tp);
2999 
3000 	indent += 2;
3001 
3002 	db_print_indent(indent);
3003 	db_printf("t_segq first: %p   t_segqlen: %d   t_dupacks: %d\n",
3004 	   TAILQ_FIRST(&tp->t_segq), tp->t_segqlen, tp->t_dupacks);
3005 
3006 	db_print_indent(indent);
3007 	db_printf("t_callout: %p   t_timers: %p\n",
3008 	    &tp->t_callout, &tp->t_timers);
3009 
3010 	db_print_indent(indent);
3011 	db_printf("t_state: %d (", tp->t_state);
3012 	db_print_tstate(tp->t_state);
3013 	db_printf(")\n");
3014 
3015 	db_print_indent(indent);
3016 	db_printf("t_flags: 0x%x (", tp->t_flags);
3017 	db_print_tflags(tp->t_flags);
3018 	db_printf(")\n");
3019 
3020 	db_print_indent(indent);
3021 	db_printf("t_flags2: 0x%x (", tp->t_flags2);
3022 	db_print_tflags2(tp->t_flags2);
3023 	db_printf(")\n");
3024 
3025 	db_print_indent(indent);
3026 	db_printf("snd_una: 0x%08x   snd_max: 0x%08x   snd_nxt: 0x%08x\n",
3027 	    tp->snd_una, tp->snd_max, tp->snd_nxt);
3028 
3029 	db_print_indent(indent);
3030 	db_printf("snd_up: 0x%08x   snd_wl1: 0x%08x   snd_wl2: 0x%08x\n",
3031 	   tp->snd_up, tp->snd_wl1, tp->snd_wl2);
3032 
3033 	db_print_indent(indent);
3034 	db_printf("iss: 0x%08x   irs: 0x%08x   rcv_nxt: 0x%08x\n",
3035 	    tp->iss, tp->irs, tp->rcv_nxt);
3036 
3037 	db_print_indent(indent);
3038 	db_printf("rcv_adv: 0x%08x   rcv_wnd: %u   rcv_up: 0x%08x\n",
3039 	    tp->rcv_adv, tp->rcv_wnd, tp->rcv_up);
3040 
3041 	db_print_indent(indent);
3042 	db_printf("snd_wnd: %u   snd_cwnd: %u\n",
3043 	   tp->snd_wnd, tp->snd_cwnd);
3044 
3045 	db_print_indent(indent);
3046 	db_printf("snd_ssthresh: %u   snd_recover: "
3047 	    "0x%08x\n", tp->snd_ssthresh, tp->snd_recover);
3048 
3049 	db_print_indent(indent);
3050 	db_printf("t_rcvtime: %u   t_startime: %u\n",
3051 	    tp->t_rcvtime, tp->t_starttime);
3052 
3053 	db_print_indent(indent);
3054 	db_printf("t_rttime: %u   t_rtsq: 0x%08x\n",
3055 	    tp->t_rtttime, tp->t_rtseq);
3056 
3057 	db_print_indent(indent);
3058 	db_printf("t_rxtcur: %d   t_maxseg: %u   t_srtt: %d\n",
3059 	    tp->t_rxtcur, tp->t_maxseg, tp->t_srtt);
3060 
3061 	db_print_indent(indent);
3062 	db_printf("t_rttvar: %d   t_rxtshift: %d   t_rttmin: %u\n",
3063 	    tp->t_rttvar, tp->t_rxtshift, tp->t_rttmin);
3064 
3065 	db_print_indent(indent);
3066 	db_printf("t_rttupdated: %u   max_sndwnd: %u   t_softerror: %d\n",
3067 	    tp->t_rttupdated, tp->max_sndwnd, tp->t_softerror);
3068 
3069 	db_print_indent(indent);
3070 	db_printf("t_oobflags: 0x%x (", tp->t_oobflags);
3071 	db_print_toobflags(tp->t_oobflags);
3072 	db_printf(")   t_iobc: 0x%02x\n", tp->t_iobc);
3073 
3074 	db_print_indent(indent);
3075 	db_printf("snd_scale: %u   rcv_scale: %u   request_r_scale: %u\n",
3076 	    tp->snd_scale, tp->rcv_scale, tp->request_r_scale);
3077 
3078 	db_print_indent(indent);
3079 	db_printf("ts_recent: %u   ts_recent_age: %u\n",
3080 	    tp->ts_recent, tp->ts_recent_age);
3081 
3082 	db_print_indent(indent);
3083 	db_printf("ts_offset: %u   last_ack_sent: 0x%08x   snd_cwnd_prev: "
3084 	    "%u\n", tp->ts_offset, tp->last_ack_sent, tp->snd_cwnd_prev);
3085 
3086 	db_print_indent(indent);
3087 	db_printf("snd_ssthresh_prev: %u   snd_recover_prev: 0x%08x   "
3088 	    "t_badrxtwin: %u\n", tp->snd_ssthresh_prev,
3089 	    tp->snd_recover_prev, tp->t_badrxtwin);
3090 
3091 	db_print_indent(indent);
3092 	db_printf("snd_numholes: %d  snd_holes first: %p\n",
3093 	    tp->snd_numholes, TAILQ_FIRST(&tp->snd_holes));
3094 
3095 	db_print_indent(indent);
3096 	db_printf("snd_fack: 0x%08x   rcv_numsacks: %d\n",
3097 	    tp->snd_fack, tp->rcv_numsacks);
3098 
3099 	/* Skip sackblks, sackhint. */
3100 
3101 	db_print_indent(indent);
3102 	db_printf("t_rttlow: %d   rfbuf_ts: %u   rfbuf_cnt: %d\n",
3103 	    tp->t_rttlow, tp->rfbuf_ts, tp->rfbuf_cnt);
3104 }
3105 
3106 DB_SHOW_COMMAND(tcpcb, db_show_tcpcb)
3107 {
3108 	struct tcpcb *tp;
3109 
3110 	if (!have_addr) {
3111 		db_printf("usage: show tcpcb <addr>\n");
3112 		return;
3113 	}
3114 	tp = (struct tcpcb *)addr;
3115 
3116 	db_print_tcpcb(tp, "tcpcb", 0);
3117 }
3118 #endif
3119