1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1993
5 * The Regents of the University of California.
6 * Copyright (c) 2006-2007 Robert N. M. Watson
7 * Copyright (c) 2010-2011 Juniper Networks, Inc.
8 * All rights reserved.
9 *
10 * Portions of this software were developed by Robert N. M. Watson under
11 * contract to Juniper Networks, Inc.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 */
37
38 #include "opt_ddb.h"
39 #include "opt_inet.h"
40 #include "opt_inet6.h"
41 #include "opt_ipsec.h"
42 #include "opt_kern_tls.h"
43
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/arb.h>
47 #include <sys/limits.h>
48 #include <sys/malloc.h>
49 #include <sys/refcount.h>
50 #include <sys/kernel.h>
51 #include <sys/ktls.h>
52 #include <sys/qmath.h>
53 #include <sys/sysctl.h>
54 #include <sys/mbuf.h>
55 #ifdef INET6
56 #include <sys/domain.h>
57 #endif /* INET6 */
58 #include <sys/socket.h>
59 #include <sys/socketvar.h>
60 #include <sys/protosw.h>
61 #include <sys/proc.h>
62 #include <sys/jail.h>
63 #include <sys/stats.h>
64
65 #ifdef DDB
66 #include <ddb/ddb.h>
67 #endif
68
69 #include <net/if.h>
70 #include <net/if_var.h>
71 #include <net/route.h>
72 #include <net/vnet.h>
73
74 #include <netinet/in.h>
75 #include <netinet/in_kdtrace.h>
76 #include <netinet/in_pcb.h>
77 #include <netinet/in_rss.h>
78 #include <netinet/in_systm.h>
79 #include <netinet/in_var.h>
80 #include <netinet/ip.h>
81 #include <netinet/ip_var.h>
82 #ifdef INET6
83 #include <netinet/ip6.h>
84 #include <netinet6/in6_pcb.h>
85 #include <netinet6/in6_rss.h>
86 #include <netinet6/ip6_var.h>
87 #include <netinet6/scope6_var.h>
88 #endif
89 #include <netinet/tcp.h>
90 #include <netinet/tcp_fsm.h>
91 #include <netinet/tcp_seq.h>
92 #include <netinet/tcp_timer.h>
93 #include <netinet/tcp_var.h>
94 #include <netinet/tcp_log_buf.h>
95 #include <netinet/tcpip.h>
96 #include <netinet/cc/cc.h>
97 #include <netinet/tcp_fastopen.h>
98 #include <netinet/tcp_hpts.h>
99 #ifdef TCP_OFFLOAD
100 #include <netinet/tcp_offload.h>
101 #endif
102 #include <netipsec/ipsec_support.h>
103
104 #include <vm/vm.h>
105 #include <vm/vm_param.h>
106 #include <vm/pmap.h>
107 #include <vm/vm_extern.h>
108 #include <vm/vm_map.h>
109 #include <vm/vm_page.h>
110
111 /*
112 * TCP protocol interface to socket abstraction.
113 */
114 #ifdef INET
115 static int tcp_connect(struct tcpcb *, struct sockaddr_in *,
116 struct thread *td);
117 #endif /* INET */
118 #ifdef INET6
119 static int tcp6_connect(struct tcpcb *, struct sockaddr_in6 *,
120 struct thread *td);
121 #endif /* INET6 */
122 static void tcp_disconnect(struct tcpcb *);
123 static void tcp_usrclosed(struct tcpcb *);
124 static void tcp_fill_info(const struct tcpcb *, struct tcp_info *);
125
126 static int tcp_pru_options_support(struct tcpcb *tp, int flags);
127
128 static void
tcp_bblog_pru(struct tcpcb * tp,uint32_t pru,int error)129 tcp_bblog_pru(struct tcpcb *tp, uint32_t pru, int error)
130 {
131 struct tcp_log_buffer *lgb;
132
133 KASSERT(tp != NULL, ("tcp_bblog_pru: tp == NULL"));
134 INP_WLOCK_ASSERT(tptoinpcb(tp));
135 if (tcp_bblogging_on(tp)) {
136 lgb = tcp_log_event(tp, NULL, NULL, NULL, TCP_LOG_PRU, error,
137 0, NULL, false, NULL, NULL, 0, NULL);
138 } else {
139 lgb = NULL;
140 }
141 if (lgb != NULL) {
142 if (error >= 0) {
143 lgb->tlb_errno = (uint32_t)error;
144 }
145 lgb->tlb_flex1 = pru;
146 }
147 }
148
149 /*
150 * TCP attaches to socket via pr_attach(), reserving space,
151 * and an internet control block.
152 */
153 static int
tcp_usr_attach(struct socket * so,int proto,struct thread * td)154 tcp_usr_attach(struct socket *so, int proto, struct thread *td)
155 {
156 struct inpcb *inp;
157 struct tcpcb *tp = NULL;
158 int error;
159
160 inp = sotoinpcb(so);
161 KASSERT(inp == NULL, ("tcp_usr_attach: inp != NULL"));
162
163 error = soreserve(so, V_tcp_sendspace, V_tcp_recvspace);
164 if (error)
165 goto out;
166
167 so->so_rcv.sb_flags |= SB_AUTOSIZE;
168 so->so_snd.sb_flags |= (SB_AUTOLOWAT | SB_AUTOSIZE);
169 error = in_pcballoc(so, &V_tcbinfo);
170 if (error)
171 goto out;
172 inp = sotoinpcb(so);
173 tp = tcp_newtcpcb(inp, NULL);
174 if (tp == NULL) {
175 error = ENOBUFS;
176 in_pcbfree(inp);
177 goto out;
178 }
179 tp->t_state = TCPS_CLOSED;
180 tcp_bblog_pru(tp, PRU_ATTACH, error);
181 INP_WUNLOCK(inp);
182 TCPSTATES_INC(TCPS_CLOSED);
183 out:
184 TCP_PROBE2(debug__user, tp, PRU_ATTACH);
185 return (error);
186 }
187
188 /*
189 * tcp_usr_detach is called when the socket layer loses its final reference
190 * to the socket, be it a file descriptor reference, a reference from TCP,
191 * etc. At this point, there is only one case in which we will keep around
192 * inpcb state: time wait.
193 */
194 static void
tcp_usr_detach(struct socket * so)195 tcp_usr_detach(struct socket *so)
196 {
197 struct inpcb *inp;
198 struct tcpcb *tp;
199
200 inp = sotoinpcb(so);
201 KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
202 INP_WLOCK(inp);
203 KASSERT(so->so_pcb == inp && inp->inp_socket == so,
204 ("%s: socket %p inp %p mismatch", __func__, so, inp));
205
206 tp = intotcpcb(inp);
207
208 KASSERT(tp->t_flags & TF_DISCONNECTED ||
209 tp->t_state < TCPS_SYN_SENT,
210 ("%s: inp %p not disconnected or embryonic", __func__, inp));
211
212 TCPSTATES_DEC(tp->t_state);
213 tcp_discardcb(tp);
214 in_pcbfree(inp);
215 }
216
217 #ifdef INET
218 /*
219 * Give the socket an address.
220 */
221 static int
tcp_usr_bind(struct socket * so,struct sockaddr * nam,struct thread * td)222 tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
223 {
224 struct inpcb *inp = sotoinpcb(so);
225 struct tcpcb *tp = intotcpcb(inp);
226 struct sockaddr_in *sinp;
227 int error = 0;
228
229 INP_WLOCK(inp);
230 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
231 INP_WUNLOCK(inp);
232 return (EINVAL);
233 }
234
235 sinp = (struct sockaddr_in *)nam;
236 if (nam->sa_family != AF_INET) {
237 /*
238 * Preserve compatibility with old programs.
239 */
240 if (nam->sa_family != AF_UNSPEC ||
241 nam->sa_len < offsetof(struct sockaddr_in, sin_zero) ||
242 sinp->sin_addr.s_addr != INADDR_ANY) {
243 error = EAFNOSUPPORT;
244 goto out;
245 }
246 nam->sa_family = AF_INET;
247 }
248 if (nam->sa_len != sizeof(*sinp)) {
249 error = EINVAL;
250 goto out;
251 }
252 /*
253 * Must check for multicast addresses and disallow binding
254 * to them.
255 */
256 if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
257 error = EAFNOSUPPORT;
258 goto out;
259 }
260 error = in_pcbbind(inp, sinp, V_tcp_bind_all_fibs ? 0 : INPBIND_FIB,
261 td->td_ucred);
262 out:
263 tcp_bblog_pru(tp, PRU_BIND, error);
264 TCP_PROBE2(debug__user, tp, PRU_BIND);
265 INP_WUNLOCK(inp);
266
267 return (error);
268 }
269 #endif /* INET */
270
271 #ifdef INET6
272 static int
tcp6_usr_bind(struct socket * so,struct sockaddr * nam,struct thread * td)273 tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
274 {
275 struct inpcb *inp = sotoinpcb(so);
276 struct tcpcb *tp = intotcpcb(inp);
277 struct sockaddr_in6 *sin6;
278 int error = 0;
279 u_char vflagsav;
280
281 INP_WLOCK(inp);
282 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
283 INP_WUNLOCK(inp);
284 return (EINVAL);
285 }
286
287 vflagsav = inp->inp_vflag;
288
289 sin6 = (struct sockaddr_in6 *)nam;
290 if (nam->sa_family != AF_INET6) {
291 error = EAFNOSUPPORT;
292 goto out;
293 }
294 if (nam->sa_len != sizeof(*sin6)) {
295 error = EINVAL;
296 goto out;
297 }
298 /*
299 * Must check for multicast addresses and disallow binding
300 * to them.
301 */
302 if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
303 error = EAFNOSUPPORT;
304 goto out;
305 }
306
307 inp->inp_vflag &= ~INP_IPV4;
308 inp->inp_vflag |= INP_IPV6;
309 #ifdef INET
310 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0) {
311 if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr))
312 inp->inp_vflag |= INP_IPV4;
313 else if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
314 struct sockaddr_in sin;
315
316 in6_sin6_2_sin(&sin, sin6);
317 if (IN_MULTICAST(ntohl(sin.sin_addr.s_addr))) {
318 error = EAFNOSUPPORT;
319 goto out;
320 }
321 inp->inp_vflag |= INP_IPV4;
322 inp->inp_vflag &= ~INP_IPV6;
323 error = in_pcbbind(inp, &sin, 0, td->td_ucred);
324 goto out;
325 }
326 }
327 #endif
328 error = in6_pcbbind(inp, sin6, V_tcp_bind_all_fibs ? 0 : INPBIND_FIB,
329 td->td_ucred);
330 out:
331 if (error != 0)
332 inp->inp_vflag = vflagsav;
333 tcp_bblog_pru(tp, PRU_BIND, error);
334 TCP_PROBE2(debug__user, tp, PRU_BIND);
335 INP_WUNLOCK(inp);
336 return (error);
337 }
338 #endif /* INET6 */
339
340 #ifdef INET
341 /*
342 * Prepare to accept connections.
343 */
344 static int
tcp_usr_listen(struct socket * so,int backlog,struct thread * td)345 tcp_usr_listen(struct socket *so, int backlog, struct thread *td)
346 {
347 struct inpcb *inp = sotoinpcb(so);
348 struct tcpcb *tp = intotcpcb(inp);
349 int error = 0;
350 bool already_listening;
351
352 INP_WLOCK(inp);
353 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
354 INP_WUNLOCK(inp);
355 return (EINVAL);
356 }
357
358 SOCK_LOCK(so);
359 already_listening = SOLISTENING(so);
360 error = solisten_proto_check(so);
361 if (error != 0) {
362 SOCK_UNLOCK(so);
363 goto out;
364 }
365 if (inp->inp_lport == 0) {
366 error = in_pcbbind(inp, NULL,
367 V_tcp_bind_all_fibs ? 0 : INPBIND_FIB, td->td_ucred);
368 }
369 if (error == 0) {
370 tcp_state_change(tp, TCPS_LISTEN);
371 solisten_proto(so, backlog);
372 #ifdef TCP_OFFLOAD
373 if ((so->so_options & SO_NO_OFFLOAD) == 0)
374 tcp_offload_listen_start(tp);
375 #endif
376 } else {
377 solisten_proto_abort(so);
378 }
379 SOCK_UNLOCK(so);
380 if (already_listening)
381 goto out;
382
383 if (error == 0) {
384 in_pcblisten(inp);
385 if (tp->t_flags & TF_FASTOPEN)
386 tp->t_tfo_pending = tcp_fastopen_alloc_counter();
387 }
388
389 out:
390 tcp_bblog_pru(tp, PRU_LISTEN, error);
391 TCP_PROBE2(debug__user, tp, PRU_LISTEN);
392 INP_WUNLOCK(inp);
393 return (error);
394 }
395 #endif /* INET */
396
397 #ifdef INET6
398 static int
tcp6_usr_listen(struct socket * so,int backlog,struct thread * td)399 tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
400 {
401 struct inpcb *inp = sotoinpcb(so);
402 struct tcpcb *tp = intotcpcb(inp);
403 u_char vflagsav;
404 int error = 0;
405 bool already_listening;
406
407 INP_WLOCK(inp);
408 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
409 INP_WUNLOCK(inp);
410 return (EINVAL);
411 }
412
413 vflagsav = inp->inp_vflag;
414
415 SOCK_LOCK(so);
416 already_listening = SOLISTENING(so);
417 error = solisten_proto_check(so);
418 if (error != 0) {
419 SOCK_UNLOCK(so);
420 goto out;
421 }
422 if (inp->inp_lport == 0) {
423 inp->inp_vflag &= ~INP_IPV4;
424 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0)
425 inp->inp_vflag |= INP_IPV4;
426 error = in6_pcbbind(inp, NULL,
427 V_tcp_bind_all_fibs ? 0 : INPBIND_FIB, td->td_ucred);
428 }
429 if (error == 0) {
430 tcp_state_change(tp, TCPS_LISTEN);
431 solisten_proto(so, backlog);
432 #ifdef TCP_OFFLOAD
433 if ((so->so_options & SO_NO_OFFLOAD) == 0)
434 tcp_offload_listen_start(tp);
435 #endif
436 } else {
437 solisten_proto_abort(so);
438 }
439 SOCK_UNLOCK(so);
440 if (already_listening)
441 goto out;
442
443 if (error == 0) {
444 in_pcblisten(inp);
445 if (tp->t_flags & TF_FASTOPEN)
446 tp->t_tfo_pending = tcp_fastopen_alloc_counter();
447 } else
448 inp->inp_vflag = vflagsav;
449
450 out:
451 tcp_bblog_pru(tp, PRU_LISTEN, error);
452 TCP_PROBE2(debug__user, tp, PRU_LISTEN);
453 INP_WUNLOCK(inp);
454 return (error);
455 }
456 #endif /* INET6 */
457
458 #ifdef INET
459 /*
460 * Initiate connection to peer.
461 * Create a template for use in transmissions on this connection.
462 * Enter SYN_SENT state, and mark socket as connecting.
463 * Start keep-alive timer, and seed output sequence space.
464 * Send initial segment on connection.
465 */
466 static int
tcp_usr_connect(struct socket * so,struct sockaddr * nam,struct thread * td)467 tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
468 {
469 struct epoch_tracker et;
470 struct inpcb *inp = sotoinpcb(so);
471 struct tcpcb *tp = intotcpcb(inp);
472 struct sockaddr_in *sinp;
473 int error = 0;
474
475 INP_WLOCK(inp);
476 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
477 INP_WUNLOCK(inp);
478 return (ECONNREFUSED);
479 }
480
481 sinp = (struct sockaddr_in *)nam;
482 if (nam->sa_family != AF_INET) {
483 error = EAFNOSUPPORT;
484 goto out;
485 }
486 if (nam->sa_len != sizeof (*sinp)) {
487 error = EINVAL;
488 goto out;
489 }
490 /*
491 * Must disallow TCP ``connections'' to multicast addresses.
492 */
493 if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
494 error = EAFNOSUPPORT;
495 goto out;
496 }
497 if (ntohl(sinp->sin_addr.s_addr) == INADDR_BROADCAST) {
498 error = EACCES;
499 goto out;
500 }
501 if ((error = prison_remote_ip4(td->td_ucred, &sinp->sin_addr)) != 0)
502 goto out;
503 if (SOLISTENING(so)) {
504 error = EOPNOTSUPP;
505 goto out;
506 }
507 NET_EPOCH_ENTER(et);
508 if ((error = tcp_connect(tp, sinp, td)) != 0)
509 goto out_in_epoch;
510 #ifdef TCP_OFFLOAD
511 if (registered_toedevs > 0 &&
512 (so->so_options & SO_NO_OFFLOAD) == 0 &&
513 (error = tcp_offload_connect(so, nam)) == 0)
514 goto out_in_epoch;
515 #endif
516 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
517 error = tcp_output(tp);
518 KASSERT(error >= 0, ("TCP stack %s requested tcp_drop(%p) at connect()"
519 ", error code %d", tp->t_fb->tfb_tcp_block_name, tp, -error));
520 out_in_epoch:
521 NET_EPOCH_EXIT(et);
522 out:
523 tcp_bblog_pru(tp, PRU_CONNECT, error);
524 TCP_PROBE2(debug__user, tp, PRU_CONNECT);
525 INP_WUNLOCK(inp);
526 return (error);
527 }
528 #endif /* INET */
529
530 #ifdef INET6
531 static int
tcp6_usr_connect(struct socket * so,struct sockaddr * nam,struct thread * td)532 tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
533 {
534 struct epoch_tracker et;
535 struct inpcb *inp = sotoinpcb(so);
536 struct tcpcb *tp = intotcpcb(inp);
537 struct sockaddr_in6 *sin6;
538 int error = 0;
539 u_int8_t incflagsav;
540 u_char vflagsav;
541
542 INP_WLOCK(inp);
543 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
544 INP_WUNLOCK(inp);
545 return (ECONNREFUSED);
546 }
547
548 vflagsav = inp->inp_vflag;
549 incflagsav = inp->inp_inc.inc_flags;
550
551 sin6 = (struct sockaddr_in6 *)nam;
552 if (nam->sa_family != AF_INET6) {
553 error = EAFNOSUPPORT;
554 goto out;
555 }
556 if (nam->sa_len != sizeof (*sin6)) {
557 error = EINVAL;
558 goto out;
559 }
560 /*
561 * Must disallow TCP ``connections'' to multicast addresses.
562 */
563 if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
564 error = EAFNOSUPPORT;
565 goto out;
566 }
567 if (SOLISTENING(so)) {
568 error = EOPNOTSUPP;
569 goto out;
570 }
571 #ifdef INET
572 /*
573 * XXXRW: Some confusion: V4/V6 flags relate to binding, and
574 * therefore probably require the hash lock, which isn't held here.
575 * Is this a significant problem?
576 */
577 if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
578 struct sockaddr_in sin;
579
580 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0) {
581 error = EINVAL;
582 goto out;
583 }
584 if ((inp->inp_vflag & INP_IPV4) == 0) {
585 error = EAFNOSUPPORT;
586 goto out;
587 }
588
589 in6_sin6_2_sin(&sin, sin6);
590 if (IN_MULTICAST(ntohl(sin.sin_addr.s_addr))) {
591 error = EAFNOSUPPORT;
592 goto out;
593 }
594 if (ntohl(sin.sin_addr.s_addr) == INADDR_BROADCAST) {
595 error = EACCES;
596 goto out;
597 }
598 if ((error = prison_remote_ip4(td->td_ucred,
599 &sin.sin_addr)) != 0)
600 goto out;
601 inp->inp_vflag |= INP_IPV4;
602 inp->inp_vflag &= ~INP_IPV6;
603 NET_EPOCH_ENTER(et);
604 if ((error = tcp_connect(tp, &sin, td)) != 0)
605 goto out_in_epoch;
606 #ifdef TCP_OFFLOAD
607 if (registered_toedevs > 0 &&
608 (so->so_options & SO_NO_OFFLOAD) == 0 &&
609 (error = tcp_offload_connect(so, nam)) == 0)
610 goto out_in_epoch;
611 #endif
612 error = tcp_output(tp);
613 goto out_in_epoch;
614 } else {
615 if ((inp->inp_vflag & INP_IPV6) == 0) {
616 error = EAFNOSUPPORT;
617 goto out;
618 }
619 }
620 #endif
621 if ((error = prison_remote_ip6(td->td_ucred, &sin6->sin6_addr)) != 0)
622 goto out;
623 inp->inp_vflag &= ~INP_IPV4;
624 inp->inp_vflag |= INP_IPV6;
625 inp->inp_inc.inc_flags |= INC_ISIPV6;
626 NET_EPOCH_ENTER(et);
627 if ((error = tcp6_connect(tp, sin6, td)) != 0)
628 goto out_in_epoch;
629 #ifdef TCP_OFFLOAD
630 if (registered_toedevs > 0 &&
631 (so->so_options & SO_NO_OFFLOAD) == 0 &&
632 (error = tcp_offload_connect(so, nam)) == 0)
633 goto out_in_epoch;
634 #endif
635 tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
636 error = tcp_output(tp);
637 out_in_epoch:
638 NET_EPOCH_EXIT(et);
639 out:
640 KASSERT(error >= 0, ("TCP stack %s requested tcp_drop(%p) at connect()"
641 ", error code %d", tp->t_fb->tfb_tcp_block_name, tp, -error));
642 /*
643 * If the implicit bind in the connect call fails, restore
644 * the flags we modified.
645 */
646 if (error != 0 && inp->inp_lport == 0) {
647 inp->inp_vflag = vflagsav;
648 inp->inp_inc.inc_flags = incflagsav;
649 }
650
651 tcp_bblog_pru(tp, PRU_CONNECT, error);
652 TCP_PROBE2(debug__user, tp, PRU_CONNECT);
653 INP_WUNLOCK(inp);
654 return (error);
655 }
656 #endif /* INET6 */
657
658 /*
659 * Initiate disconnect from peer.
660 * If connection never passed embryonic stage, just drop;
661 * else if don't need to let data drain, then can just drop anyways,
662 * else have to begin TCP shutdown process: mark socket disconnecting,
663 * drain unread data, state switch to reflect user close, and
664 * send segment (e.g. FIN) to peer. Socket will be really disconnected
665 * when peer sends FIN and acks ours.
666 *
667 * SHOULD IMPLEMENT LATER PRU_CONNECT VIA REALLOC TCPCB.
668 */
669 static int
tcp_usr_disconnect(struct socket * so)670 tcp_usr_disconnect(struct socket *so)
671 {
672 struct inpcb *inp;
673 struct tcpcb *tp = NULL;
674 struct epoch_tracker et;
675
676 NET_EPOCH_ENTER(et);
677 inp = sotoinpcb(so);
678 KASSERT(inp != NULL, ("tcp_usr_disconnect: inp == NULL"));
679 INP_WLOCK(inp);
680 tp = intotcpcb(inp);
681
682 if (tp->t_state == TCPS_TIME_WAIT)
683 goto out;
684 tcp_disconnect(tp);
685 out:
686 tcp_bblog_pru(tp, PRU_DISCONNECT, 0);
687 TCP_PROBE2(debug__user, tp, PRU_DISCONNECT);
688 INP_WUNLOCK(inp);
689 NET_EPOCH_EXIT(et);
690 return (0);
691 }
692
693 #ifdef INET
694 /*
695 * Accept a connection. Essentially all the work is done at higher levels;
696 * just return the address of the peer, storing through addr.
697 */
698 static int
tcp_usr_accept(struct socket * so,struct sockaddr * sa)699 tcp_usr_accept(struct socket *so, struct sockaddr *sa)
700 {
701 struct inpcb *inp = sotoinpcb(so);
702 struct tcpcb *tp = intotcpcb(inp);
703 int error = 0;
704
705 INP_WLOCK(inp);
706 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
707 INP_WUNLOCK(inp);
708 return (ECONNABORTED);
709 }
710
711 if (so->so_state & SS_ISDISCONNECTED)
712 error = ECONNABORTED;
713 else
714 *(struct sockaddr_in *)sa = (struct sockaddr_in ){
715 .sin_family = AF_INET,
716 .sin_len = sizeof(struct sockaddr_in),
717 .sin_port = inp->inp_fport,
718 .sin_addr = inp->inp_faddr,
719 };
720 tcp_bblog_pru(tp, PRU_ACCEPT, error);
721 TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
722 INP_WUNLOCK(inp);
723
724 return (error);
725 }
726 #endif /* INET */
727
728 #ifdef INET6
729 static int
tcp6_usr_accept(struct socket * so,struct sockaddr * sa)730 tcp6_usr_accept(struct socket *so, struct sockaddr *sa)
731 {
732 struct inpcb *inp = sotoinpcb(so);
733 struct tcpcb *tp = intotcpcb(inp);
734 int error = 0;
735
736 INP_WLOCK(inp);
737 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
738 INP_WUNLOCK(inp);
739 return (ECONNABORTED);
740 }
741
742 if (so->so_state & SS_ISDISCONNECTED) {
743 error = ECONNABORTED;
744 } else {
745 if (inp->inp_vflag & INP_IPV4) {
746 struct sockaddr_in sin = {
747 .sin_family = AF_INET,
748 .sin_len = sizeof(struct sockaddr_in),
749 .sin_port = inp->inp_fport,
750 .sin_addr = inp->inp_faddr,
751 };
752 in6_sin_2_v4mapsin6(&sin, (struct sockaddr_in6 *)sa);
753 } else {
754 *(struct sockaddr_in6 *)sa = (struct sockaddr_in6 ){
755 .sin6_family = AF_INET6,
756 .sin6_len = sizeof(struct sockaddr_in6),
757 .sin6_port = inp->inp_fport,
758 .sin6_addr = inp->in6p_faddr,
759 };
760 /* XXX: should catch errors */
761 (void)sa6_recoverscope((struct sockaddr_in6 *)sa);
762 }
763 }
764
765 tcp_bblog_pru(tp, PRU_ACCEPT, error);
766 TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
767 INP_WUNLOCK(inp);
768
769 return (error);
770 }
771 #endif /* INET6 */
772
773 /*
774 * Mark the connection as being incapable of further output.
775 */
776 static int
tcp_usr_shutdown(struct socket * so,enum shutdown_how how)777 tcp_usr_shutdown(struct socket *so, enum shutdown_how how)
778 {
779 struct epoch_tracker et;
780 struct inpcb *inp = sotoinpcb(so);
781 struct tcpcb *tp = intotcpcb(inp);
782 int error = 0;
783
784 SOCK_LOCK(so);
785 if (SOLISTENING(so)) {
786 if (how != SHUT_WR) {
787 so->so_error = ECONNABORTED;
788 solisten_wakeup(so); /* unlocks so */
789 } else
790 SOCK_UNLOCK(so);
791 return (ENOTCONN);
792 } else if ((so->so_state &
793 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) {
794 SOCK_UNLOCK(so);
795 return (ENOTCONN);
796 }
797 SOCK_UNLOCK(so);
798
799 switch (how) {
800 case SHUT_RD:
801 sorflush(so);
802 break;
803 case SHUT_RDWR:
804 sorflush(so);
805 /* FALLTHROUGH */
806 case SHUT_WR:
807 /*
808 * XXXGL: mimicing old soshutdown() here. But shouldn't we
809 * return ECONNRESEST for SHUT_RD as well?
810 */
811 INP_WLOCK(inp);
812 if (tp->t_flags & TF_DISCONNECTED) {
813 INP_WUNLOCK(inp);
814 return (ECONNRESET);
815 }
816
817 socantsendmore(so);
818 NET_EPOCH_ENTER(et);
819 tcp_usrclosed(tp);
820 error = tcp_output_nodrop(tp);
821 tcp_bblog_pru(tp, PRU_SHUTDOWN, error);
822 TCP_PROBE2(debug__user, tp, PRU_SHUTDOWN);
823 error = tcp_unlock_or_drop(tp, error);
824 NET_EPOCH_EXIT(et);
825 }
826 wakeup(&so->so_timeo);
827
828 return (error);
829 }
830
831 /*
832 * After a receive, possibly send window update to peer.
833 */
834 static int
tcp_usr_rcvd(struct socket * so,int flags)835 tcp_usr_rcvd(struct socket *so, int flags)
836 {
837 struct epoch_tracker et;
838 struct inpcb *inp = sotoinpcb(so);
839 struct tcpcb *tp = intotcpcb(inp);
840 int outrv = 0, error = 0;
841
842 INP_WLOCK(inp);
843 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
844 /* XXXGL: how could this happen?! */
845 INP_WUNLOCK(inp);
846 return (ECONNRESET);
847 }
848
849 NET_EPOCH_ENTER(et);
850 /*
851 * For passively-created TFO connections, don't attempt a window
852 * update while still in SYN_RECEIVED as this may trigger an early
853 * SYN|ACK. It is preferable to have the SYN|ACK be sent along with
854 * application response data, or failing that, when the DELACK timer
855 * expires.
856 */
857 if ((tp->t_flags & TF_FASTOPEN) && (tp->t_state == TCPS_SYN_RECEIVED))
858 goto out;
859 #ifdef TCP_OFFLOAD
860 if (tp->t_flags & TF_TOE)
861 tcp_offload_rcvd(tp);
862 else
863 #endif
864 outrv = tcp_output_nodrop(tp);
865 out:
866 tcp_bblog_pru(tp, PRU_RCVD, error);
867 TCP_PROBE2(debug__user, tp, PRU_RCVD);
868 (void) tcp_unlock_or_drop(tp, outrv);
869 NET_EPOCH_EXIT(et);
870 return (error);
871 }
872
873 /*
874 * Do a send by putting data in output queue and updating urgent
875 * marker if URG set. Possibly send more data. Unlike the other
876 * pr_*() routines, the mbuf chains are our responsibility. We
877 * must either enqueue them or free them. The other pr_*() routines
878 * generally are caller-frees.
879 */
880 static int
tcp_usr_send(struct socket * so,int flags,struct mbuf * m,struct sockaddr * nam,struct mbuf * control,struct thread * td)881 tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
882 struct sockaddr *nam, struct mbuf *control, struct thread *td)
883 {
884 struct epoch_tracker et;
885 struct inpcb *inp = sotoinpcb(so);
886 struct tcpcb *tp = intotcpcb(inp);
887 #ifdef INET
888 #ifdef INET6
889 struct sockaddr_in sin;
890 #endif
891 struct sockaddr_in *sinp;
892 #endif
893 #ifdef INET6
894 struct sockaddr_in6 *sin6;
895 int isipv6;
896 #endif
897 int error = 0;
898 u_int8_t incflagsav;
899 u_char vflagsav;
900 bool restoreflags;
901
902 INP_WLOCK(inp);
903 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
904 if (m != NULL && (flags & PRUS_NOTREADY) == 0)
905 m_freem(m);
906 INP_WUNLOCK(inp);
907 return (ECONNRESET);
908 }
909
910 vflagsav = inp->inp_vflag;
911 incflagsav = inp->inp_inc.inc_flags;
912 restoreflags = false;
913
914 NET_EPOCH_ENTER(et);
915 if (control != NULL) {
916 /* TCP doesn't do control messages (rights, creds, etc) */
917 if (control->m_len > 0) {
918 m_freem(control);
919 error = EINVAL;
920 goto out;
921 }
922 m_freem(control); /* empty control, just free it */
923 }
924
925 if ((flags & PRUS_OOB) != 0 &&
926 (error = tcp_pru_options_support(tp, PRUS_OOB)) != 0)
927 goto out;
928
929 if (nam != NULL && tp->t_state < TCPS_SYN_SENT) {
930 if (tp->t_state == TCPS_LISTEN) {
931 error = EINVAL;
932 goto out;
933 }
934 switch (nam->sa_family) {
935 #ifdef INET
936 case AF_INET:
937 sinp = (struct sockaddr_in *)nam;
938 if (sinp->sin_len != sizeof(struct sockaddr_in)) {
939 error = EINVAL;
940 goto out;
941 }
942 if ((inp->inp_vflag & INP_IPV6) != 0) {
943 error = EAFNOSUPPORT;
944 goto out;
945 }
946 if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
947 error = EAFNOSUPPORT;
948 goto out;
949 }
950 if (ntohl(sinp->sin_addr.s_addr) == INADDR_BROADCAST) {
951 error = EACCES;
952 goto out;
953 }
954 if ((error = prison_remote_ip4(td->td_ucred,
955 &sinp->sin_addr)))
956 goto out;
957 #ifdef INET6
958 isipv6 = 0;
959 #endif
960 break;
961 #endif /* INET */
962 #ifdef INET6
963 case AF_INET6:
964 sin6 = (struct sockaddr_in6 *)nam;
965 if (sin6->sin6_len != sizeof(*sin6)) {
966 error = EINVAL;
967 goto out;
968 }
969 if ((inp->inp_vflag & INP_IPV6PROTO) == 0) {
970 error = EAFNOSUPPORT;
971 goto out;
972 }
973 if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
974 error = EAFNOSUPPORT;
975 goto out;
976 }
977 if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
978 #ifdef INET
979 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0) {
980 error = EINVAL;
981 goto out;
982 }
983 if ((inp->inp_vflag & INP_IPV4) == 0) {
984 error = EAFNOSUPPORT;
985 goto out;
986 }
987 restoreflags = true;
988 inp->inp_vflag &= ~INP_IPV6;
989 sinp = &sin;
990 in6_sin6_2_sin(sinp, sin6);
991 if (IN_MULTICAST(
992 ntohl(sinp->sin_addr.s_addr))) {
993 error = EAFNOSUPPORT;
994 goto out;
995 }
996 if ((error = prison_remote_ip4(td->td_ucred,
997 &sinp->sin_addr)))
998 goto out;
999 isipv6 = 0;
1000 #else /* !INET */
1001 error = EAFNOSUPPORT;
1002 goto out;
1003 #endif /* INET */
1004 } else {
1005 if ((inp->inp_vflag & INP_IPV6) == 0) {
1006 error = EAFNOSUPPORT;
1007 goto out;
1008 }
1009 restoreflags = true;
1010 inp->inp_vflag &= ~INP_IPV4;
1011 inp->inp_inc.inc_flags |= INC_ISIPV6;
1012 if ((error = prison_remote_ip6(td->td_ucred,
1013 &sin6->sin6_addr)))
1014 goto out;
1015 isipv6 = 1;
1016 }
1017 break;
1018 #endif /* INET6 */
1019 default:
1020 error = EAFNOSUPPORT;
1021 goto out;
1022 }
1023 }
1024 if (!(flags & PRUS_OOB)) {
1025 if (tp->t_acktime == 0)
1026 tp->t_acktime = ticks;
1027 sbappendstream(&so->so_snd, m, flags);
1028 m = NULL;
1029 if (nam && tp->t_state < TCPS_SYN_SENT) {
1030 KASSERT(tp->t_state == TCPS_CLOSED,
1031 ("%s: tp %p is listening", __func__, tp));
1032
1033 /*
1034 * Do implied connect if not yet connected,
1035 * initialize window to default value, and
1036 * initialize maxseg using peer's cached MSS.
1037 */
1038 #ifdef INET6
1039 if (isipv6)
1040 error = tcp6_connect(tp, sin6, td);
1041 #endif /* INET6 */
1042 #if defined(INET6) && defined(INET)
1043 else
1044 #endif
1045 #ifdef INET
1046 error = tcp_connect(tp, sinp, td);
1047 #endif
1048 /*
1049 * The bind operation in tcp_connect succeeded. We
1050 * no longer want to restore the flags if later
1051 * operations fail.
1052 */
1053 if (error == 0 || inp->inp_lport != 0)
1054 restoreflags = false;
1055
1056 if (error) {
1057 /* m is freed if PRUS_NOTREADY is unset. */
1058 sbflush(&so->so_snd);
1059 goto out;
1060 }
1061 if (tp->t_flags & TF_FASTOPEN)
1062 tcp_fastopen_connect(tp);
1063 else {
1064 tp->snd_wnd = TTCP_CLIENT_SND_WND;
1065 tcp_mss(tp, -1);
1066 }
1067 }
1068 if (flags & PRUS_EOF) {
1069 /*
1070 * Close the send side of the connection after
1071 * the data is sent.
1072 */
1073 socantsendmore(so);
1074 tcp_usrclosed(tp);
1075 }
1076 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1077 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
1078 (tp->t_fbyte_out == 0) &&
1079 (so->so_snd.sb_ccc > 0)) {
1080 tp->t_fbyte_out = ticks;
1081 if (tp->t_fbyte_out == 0)
1082 tp->t_fbyte_out = 1;
1083 if (tp->t_fbyte_out && tp->t_fbyte_in)
1084 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
1085 }
1086 if (!(flags & PRUS_NOTREADY)) {
1087 if (flags & PRUS_MORETOCOME)
1088 tp->t_flags |= TF_MORETOCOME;
1089 error = tcp_output_nodrop(tp);
1090 if (flags & PRUS_MORETOCOME)
1091 tp->t_flags &= ~TF_MORETOCOME;
1092 }
1093 } else {
1094 /*
1095 * XXXRW: PRUS_EOF not implemented with PRUS_OOB?
1096 */
1097 SOCK_SENDBUF_LOCK(so);
1098 if (sbspace(&so->so_snd) < -512) {
1099 SOCK_SENDBUF_UNLOCK(so);
1100 error = ENOBUFS;
1101 goto out;
1102 }
1103 /*
1104 * According to RFC961 (Assigned Protocols),
1105 * the urgent pointer points to the last octet
1106 * of urgent data. We continue, however,
1107 * to consider it to indicate the first octet
1108 * of data past the urgent section.
1109 * Otherwise, snd_up should be one lower.
1110 */
1111 if (tp->t_acktime == 0)
1112 tp->t_acktime = ticks;
1113 sbappendstream_locked(&so->so_snd, m, flags);
1114 SOCK_SENDBUF_UNLOCK(so);
1115 m = NULL;
1116 if (nam && tp->t_state < TCPS_SYN_SENT) {
1117 /*
1118 * Do implied connect if not yet connected,
1119 * initialize window to default value, and
1120 * initialize maxseg using peer's cached MSS.
1121 */
1122
1123 /*
1124 * Not going to contemplate SYN|URG
1125 */
1126 if (tp->t_flags & TF_FASTOPEN)
1127 tp->t_flags &= ~TF_FASTOPEN;
1128 #ifdef INET6
1129 if (isipv6)
1130 error = tcp6_connect(tp, sin6, td);
1131 #endif /* INET6 */
1132 #if defined(INET6) && defined(INET)
1133 else
1134 #endif
1135 #ifdef INET
1136 error = tcp_connect(tp, sinp, td);
1137 #endif
1138 /*
1139 * The bind operation in tcp_connect succeeded. We
1140 * no longer want to restore the flags if later
1141 * operations fail.
1142 */
1143 if (error == 0 || inp->inp_lport != 0)
1144 restoreflags = false;
1145
1146 if (error != 0) {
1147 /* m is freed if PRUS_NOTREADY is unset. */
1148 sbflush(&so->so_snd);
1149 goto out;
1150 }
1151 tp->snd_wnd = TTCP_CLIENT_SND_WND;
1152 tcp_mss(tp, -1);
1153 }
1154 tp->snd_up = tp->snd_una + sbavail(&so->so_snd);
1155 if ((flags & PRUS_NOTREADY) == 0) {
1156 tp->t_flags |= TF_FORCEDATA;
1157 error = tcp_output_nodrop(tp);
1158 tp->t_flags &= ~TF_FORCEDATA;
1159 }
1160 }
1161 TCP_LOG_EVENT(tp, NULL,
1162 &inp->inp_socket->so_rcv,
1163 &inp->inp_socket->so_snd,
1164 TCP_LOG_USERSEND, error,
1165 0, NULL, false);
1166
1167 out:
1168 /*
1169 * In case of PRUS_NOTREADY, the caller or tcp_usr_ready() is
1170 * responsible for freeing memory.
1171 */
1172 if (m != NULL && (flags & PRUS_NOTREADY) == 0)
1173 m_freem(m);
1174
1175 /*
1176 * If the request was unsuccessful and we changed flags,
1177 * restore the original flags.
1178 */
1179 if (error != 0 && restoreflags) {
1180 inp->inp_vflag = vflagsav;
1181 inp->inp_inc.inc_flags = incflagsav;
1182 }
1183 tcp_bblog_pru(tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
1184 ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND), error);
1185 TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
1186 ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
1187 error = tcp_unlock_or_drop(tp, error);
1188 NET_EPOCH_EXIT(et);
1189 return (error);
1190 }
1191
1192 static int
tcp_usr_ready(struct socket * so,struct mbuf * m,int count)1193 tcp_usr_ready(struct socket *so, struct mbuf *m, int count)
1194 {
1195 struct epoch_tracker et;
1196 struct inpcb *inp = sotoinpcb(so);
1197 struct tcpcb *tp = intotcpcb(inp);
1198 int error;
1199
1200 INP_WLOCK(inp);
1201 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
1202 INP_WUNLOCK(inp);
1203 mb_free_notready(m, count);
1204 return (ECONNRESET);
1205 }
1206
1207 SOCK_SENDBUF_LOCK(so);
1208 error = sbready(&so->so_snd, m, count);
1209 SOCK_SENDBUF_UNLOCK(so);
1210 if (error) {
1211 INP_WUNLOCK(inp);
1212 return (error);
1213 }
1214 NET_EPOCH_ENTER(et);
1215 error = tcp_output_unlock(tp);
1216 NET_EPOCH_EXIT(et);
1217
1218 return (error);
1219 }
1220
1221 /*
1222 * Abort the TCP. Drop the connection abruptly.
1223 */
1224 static void
tcp_usr_abort(struct socket * so)1225 tcp_usr_abort(struct socket *so)
1226 {
1227 struct epoch_tracker et;
1228 struct inpcb *inp = sotoinpcb(so);
1229 struct tcpcb *tp = intotcpcb(inp);
1230
1231 /*
1232 * If we still have full TCP state, and we're not dropped, drop.
1233 */
1234 NET_EPOCH_ENTER(et);
1235 INP_WLOCK(inp);
1236 if (!(tp->t_flags & TF_DISCONNECTED)) {
1237 tp = tcp_drop(tp, ECONNABORTED);
1238 if (tp == NULL)
1239 goto dropped;
1240 tcp_bblog_pru(tp, PRU_ABORT, 0);
1241 TCP_PROBE2(debug__user, tp, PRU_ABORT);
1242 }
1243 if (!(tp->t_flags & TF_DISCONNECTED)) {
1244 soref(so);
1245 inp->inp_flags |= INP_SOCKREF;
1246 }
1247 INP_WUNLOCK(inp);
1248 dropped:
1249 NET_EPOCH_EXIT(et);
1250 }
1251
1252 /*
1253 * TCP socket is closed. Start friendly disconnect.
1254 */
1255 static void
tcp_usr_close(struct socket * so)1256 tcp_usr_close(struct socket *so)
1257 {
1258 struct epoch_tracker et;
1259 struct inpcb *inp = sotoinpcb(so);
1260 struct tcpcb *tp = intotcpcb(inp);
1261
1262 /*
1263 * If we are still connected and we're not dropped, initiate
1264 * a disconnect.
1265 */
1266 NET_EPOCH_ENTER(et);
1267 INP_WLOCK(inp);
1268 if (!(tp->t_flags & TF_DISCONNECTED)) {
1269 if (tp->t_state != TCPS_TIME_WAIT) {
1270 tp->t_flags |= TF_CLOSED;
1271 tcp_disconnect(tp);
1272 tcp_bblog_pru(tp, PRU_CLOSE, 0);
1273 TCP_PROBE2(debug__user, tp, PRU_CLOSE);
1274 }
1275 }
1276 if (!(tp->t_flags & TF_DISCONNECTED)) {
1277 soref(so);
1278 inp->inp_flags |= INP_SOCKREF;
1279 }
1280 INP_WUNLOCK(inp);
1281 NET_EPOCH_EXIT(et);
1282 }
1283
1284 static int
tcp_pru_options_support(struct tcpcb * tp,int flags)1285 tcp_pru_options_support(struct tcpcb *tp, int flags)
1286 {
1287 /*
1288 * If the specific TCP stack has a pru_options
1289 * specified then it does not always support
1290 * all the PRU_XX options and we must ask it.
1291 * If the function is not specified then all
1292 * of the PRU_XX options are supported.
1293 */
1294 int ret = 0;
1295
1296 if (tp->t_fb->tfb_pru_options) {
1297 ret = (*tp->t_fb->tfb_pru_options)(tp, flags);
1298 }
1299 return (ret);
1300 }
1301
1302 /*
1303 * Receive out-of-band data.
1304 */
1305 static int
tcp_usr_rcvoob(struct socket * so,struct mbuf * m,int flags)1306 tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
1307 {
1308 struct inpcb *inp = sotoinpcb(so);
1309 struct tcpcb *tp = intotcpcb(inp);
1310 int error = 0;
1311
1312 INP_WLOCK(inp);
1313 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
1314 /* XXXGL: how could this happen?! */
1315 INP_WUNLOCK(inp);
1316 return (ECONNRESET);
1317 }
1318
1319 error = tcp_pru_options_support(tp, PRUS_OOB);
1320 if (error) {
1321 goto out;
1322 }
1323 if ((so->so_oobmark == 0 &&
1324 (so->so_rcv.sb_state & SBS_RCVATMARK) == 0) ||
1325 so->so_options & SO_OOBINLINE ||
1326 tp->t_oobflags & TCPOOB_HADDATA) {
1327 error = EINVAL;
1328 goto out;
1329 }
1330 if ((tp->t_oobflags & TCPOOB_HAVEDATA) == 0) {
1331 error = EWOULDBLOCK;
1332 goto out;
1333 }
1334 m->m_len = 1;
1335 *mtod(m, caddr_t) = tp->t_iobc;
1336 if ((flags & MSG_PEEK) == 0)
1337 tp->t_oobflags ^= (TCPOOB_HAVEDATA | TCPOOB_HADDATA);
1338
1339 out:
1340 tcp_bblog_pru(tp, PRU_RCVOOB, error);
1341 TCP_PROBE2(debug__user, tp, PRU_RCVOOB);
1342 INP_WUNLOCK(inp);
1343 return (error);
1344 }
1345
1346 #ifdef INET
1347 struct protosw tcp_protosw = {
1348 .pr_type = SOCK_STREAM,
1349 .pr_protocol = IPPROTO_TCP,
1350 .pr_flags = PR_CONNREQUIRED | PR_IMPLOPCL | PR_WANTRCVD |
1351 PR_CAPATTACH,
1352 .pr_ctloutput = tcp_ctloutput,
1353 .pr_abort = tcp_usr_abort,
1354 .pr_accept = tcp_usr_accept,
1355 .pr_attach = tcp_usr_attach,
1356 .pr_bind = tcp_usr_bind,
1357 .pr_connect = tcp_usr_connect,
1358 .pr_control = in_control,
1359 .pr_detach = tcp_usr_detach,
1360 .pr_disconnect = tcp_usr_disconnect,
1361 .pr_listen = tcp_usr_listen,
1362 .pr_peeraddr = in_getpeeraddr,
1363 .pr_rcvd = tcp_usr_rcvd,
1364 .pr_rcvoob = tcp_usr_rcvoob,
1365 .pr_send = tcp_usr_send,
1366 .pr_sendfile_wait = sendfile_wait_generic,
1367 .pr_ready = tcp_usr_ready,
1368 .pr_shutdown = tcp_usr_shutdown,
1369 .pr_sockaddr = in_getsockaddr,
1370 .pr_sosetlabel = in_pcbsosetlabel,
1371 .pr_close = tcp_usr_close,
1372 };
1373 #endif /* INET */
1374
1375 #ifdef INET6
1376 struct protosw tcp6_protosw = {
1377 .pr_type = SOCK_STREAM,
1378 .pr_protocol = IPPROTO_TCP,
1379 .pr_flags = PR_CONNREQUIRED | PR_IMPLOPCL |PR_WANTRCVD |
1380 PR_CAPATTACH,
1381 .pr_ctloutput = tcp_ctloutput,
1382 .pr_abort = tcp_usr_abort,
1383 .pr_accept = tcp6_usr_accept,
1384 .pr_attach = tcp_usr_attach,
1385 .pr_bind = tcp6_usr_bind,
1386 .pr_connect = tcp6_usr_connect,
1387 .pr_control = in6_control,
1388 .pr_detach = tcp_usr_detach,
1389 .pr_disconnect = tcp_usr_disconnect,
1390 .pr_listen = tcp6_usr_listen,
1391 .pr_peeraddr = in6_mapped_peeraddr,
1392 .pr_rcvd = tcp_usr_rcvd,
1393 .pr_rcvoob = tcp_usr_rcvoob,
1394 .pr_send = tcp_usr_send,
1395 .pr_sendfile_wait = sendfile_wait_generic,
1396 .pr_ready = tcp_usr_ready,
1397 .pr_shutdown = tcp_usr_shutdown,
1398 .pr_sockaddr = in6_mapped_sockaddr,
1399 .pr_sosetlabel = in_pcbsosetlabel,
1400 .pr_close = tcp_usr_close,
1401 };
1402 #endif /* INET6 */
1403
1404 #ifdef INET
1405 /*
1406 * Common subroutine to open a TCP connection to remote host specified
1407 * by struct sockaddr_in. Call in_pcbconnect() to choose local host address
1408 * and assign a local port number and install the inpcb into the hash.
1409 * Initialize connection parameters and enter SYN-SENT state.
1410 */
1411 static int
tcp_connect(struct tcpcb * tp,struct sockaddr_in * sin,struct thread * td)1412 tcp_connect(struct tcpcb *tp, struct sockaddr_in *sin, struct thread *td)
1413 {
1414 struct inpcb *inp = tptoinpcb(tp);
1415 struct socket *so = tptosocket(tp);
1416 int error;
1417
1418 NET_EPOCH_ASSERT();
1419 INP_WLOCK_ASSERT(inp);
1420
1421 if (__predict_false((so->so_state &
1422 (SS_ISCONNECTING | SS_ISCONNECTED | SS_ISDISCONNECTING |
1423 SS_ISDISCONNECTED)) != 0))
1424 return (EISCONN);
1425 if (__predict_false((so->so_options & SO_REUSEPORT_LB) != 0))
1426 return (EOPNOTSUPP);
1427
1428 error = in_pcbconnect(inp, sin, td->td_ucred);
1429 if (error != 0)
1430 return (error);
1431
1432 /* set the hash on the connection */
1433 rss_proto_software_hash_v4(inp->inp_faddr, inp->inp_laddr,
1434 inp->inp_fport, inp->inp_lport, IPPROTO_TCP,
1435 &inp->inp_flowid, &inp->inp_flowtype);
1436 /*
1437 * Compute window scaling to request:
1438 * Scale to fit into sweet spot. See tcp_syncache.c.
1439 * XXX: This should move to tcp_output().
1440 */
1441 while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1442 (TCP_MAXWIN << tp->request_r_scale) < sb_max)
1443 tp->request_r_scale++;
1444
1445 soisconnecting(so);
1446 TCPSTAT_INC(tcps_connattempt);
1447 tcp_state_change(tp, TCPS_SYN_SENT);
1448 tp->iss = tcp_new_isn(&inp->inp_inc);
1449 if (tp->t_flags & TF_REQ_TSTMP)
1450 tp->ts_offset = tcp_new_ts_offset(&inp->inp_inc);
1451 tcp_sendseqinit(tp);
1452
1453 return (0);
1454 }
1455 #endif /* INET */
1456
1457 #ifdef INET6
1458 static int
tcp6_connect(struct tcpcb * tp,struct sockaddr_in6 * sin6,struct thread * td)1459 tcp6_connect(struct tcpcb *tp, struct sockaddr_in6 *sin6, struct thread *td)
1460 {
1461 struct inpcb *inp = tptoinpcb(tp);
1462 struct socket *so = tptosocket(tp);
1463 int error;
1464
1465 NET_EPOCH_ASSERT();
1466 INP_WLOCK_ASSERT(inp);
1467
1468 if (__predict_false((so->so_state &
1469 (SS_ISCONNECTING | SS_ISCONNECTED | SS_ISDISCONNECTING |
1470 SS_ISDISCONNECTED)) != 0))
1471 return (EISCONN);
1472 if (__predict_false((so->so_options & SO_REUSEPORT_LB) != 0))
1473 return (EOPNOTSUPP);
1474
1475 error = in6_pcbconnect(inp, sin6, td->td_ucred, true);
1476 if (error != 0)
1477 return (error);
1478
1479 /* set the hash on the connection */
1480 rss_proto_software_hash_v6(&inp->in6p_faddr,
1481 &inp->in6p_laddr, inp->inp_fport, inp->inp_lport, IPPROTO_TCP,
1482 &inp->inp_flowid, &inp->inp_flowtype);
1483 /* Compute window scaling to request. */
1484 while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1485 (TCP_MAXWIN << tp->request_r_scale) < sb_max)
1486 tp->request_r_scale++;
1487
1488 soisconnecting(so);
1489 TCPSTAT_INC(tcps_connattempt);
1490 tcp_state_change(tp, TCPS_SYN_SENT);
1491 tp->iss = tcp_new_isn(&inp->inp_inc);
1492 if (tp->t_flags & TF_REQ_TSTMP)
1493 tp->ts_offset = tcp_new_ts_offset(&inp->inp_inc);
1494 tcp_sendseqinit(tp);
1495
1496 return (0);
1497 }
1498 #endif /* INET6 */
1499
1500 /*
1501 * Export TCP internal state information via a struct tcp_info, based on the
1502 * Linux 2.6 API. Not ABI compatible as our constants are mapped differently
1503 * (TCP state machine, etc). We export all information using FreeBSD-native
1504 * constants -- for example, the numeric values for tcpi_state will differ
1505 * from Linux.
1506 */
1507 void
tcp_fill_info(const struct tcpcb * tp,struct tcp_info * ti)1508 tcp_fill_info(const struct tcpcb *tp, struct tcp_info *ti)
1509 {
1510
1511 INP_LOCK_ASSERT(tptoinpcb(tp));
1512 bzero(ti, sizeof(*ti));
1513
1514 ti->tcpi_state = tp->t_state;
1515 if ((tp->t_flags & TF_REQ_TSTMP) && (tp->t_flags & TF_RCVD_TSTMP))
1516 ti->tcpi_options |= TCPI_OPT_TIMESTAMPS;
1517 if (tp->t_flags & TF_SACK_PERMIT)
1518 ti->tcpi_options |= TCPI_OPT_SACK;
1519 if ((tp->t_flags & TF_REQ_SCALE) && (tp->t_flags & TF_RCVD_SCALE)) {
1520 ti->tcpi_options |= TCPI_OPT_WSCALE;
1521 ti->tcpi_snd_wscale = tp->snd_scale;
1522 ti->tcpi_rcv_wscale = tp->rcv_scale;
1523 }
1524 switch (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT)) {
1525 case TF2_ECN_PERMIT:
1526 ti->tcpi_options |= TCPI_OPT_ECN;
1527 break;
1528 case TF2_ACE_PERMIT:
1529 /* FALLTHROUGH */
1530 case TF2_ECN_PERMIT | TF2_ACE_PERMIT:
1531 ti->tcpi_options |= TCPI_OPT_ACE;
1532 break;
1533 default:
1534 break;
1535 }
1536 if (tp->t_flags & TF_FASTOPEN)
1537 ti->tcpi_options |= TCPI_OPT_TFO;
1538
1539 ti->tcpi_rto = tp->t_rxtcur * tick;
1540 ti->tcpi_last_data_recv = ((uint32_t)ticks - tp->t_rcvtime) * tick;
1541 ti->tcpi_rtt = ((u_int64_t)tp->t_srtt * tick) >> TCP_RTT_SHIFT;
1542 ti->tcpi_rttvar = ((u_int64_t)tp->t_rttvar * tick) >> TCP_RTTVAR_SHIFT;
1543
1544 ti->tcpi_snd_ssthresh = tp->snd_ssthresh;
1545 ti->tcpi_snd_cwnd = tp->snd_cwnd;
1546
1547 /*
1548 * FreeBSD-specific extension fields for tcp_info.
1549 */
1550 ti->tcpi_rcv_space = tp->rcv_wnd;
1551 ti->tcpi_rcv_nxt = tp->rcv_nxt;
1552 ti->tcpi_snd_wnd = tp->snd_wnd;
1553 ti->tcpi_snd_bwnd = 0; /* Unused, kept for compat. */
1554 ti->tcpi_snd_nxt = tp->snd_nxt;
1555 ti->tcpi_snd_mss = tp->t_maxseg;
1556 ti->tcpi_rcv_mss = tp->t_maxseg;
1557 ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack;
1558 ti->tcpi_rcv_ooopack = tp->t_rcvoopack;
1559 ti->tcpi_snd_zerowin = tp->t_sndzerowin;
1560 ti->tcpi_snd_una = tp->snd_una;
1561 ti->tcpi_snd_max = tp->snd_max;
1562 ti->tcpi_rcv_numsacks = tp->rcv_numsacks;
1563 ti->tcpi_rcv_adv = tp->rcv_adv;
1564 ti->tcpi_dupacks = tp->t_dupacks;
1565 ti->tcpi_rttmin = tp->t_rttlow;
1566 #ifdef TCP_OFFLOAD
1567 if (tp->t_flags & TF_TOE) {
1568 ti->tcpi_options |= TCPI_OPT_TOE;
1569 tcp_offload_tcp_info(tp, ti);
1570 }
1571 #endif
1572 /*
1573 * AccECN related counters.
1574 */
1575 if ((tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT)) ==
1576 (TF2_ECN_PERMIT | TF2_ACE_PERMIT))
1577 /*
1578 * Internal counter starts at 5 for AccECN
1579 * but 0 for RFC3168 ECN.
1580 */
1581 ti->tcpi_delivered_ce = tp->t_scep - 5;
1582 else
1583 ti->tcpi_delivered_ce = tp->t_scep;
1584 ti->tcpi_received_ce = tp->t_rcep;
1585 }
1586
1587 /*
1588 * tcp_ctloutput() must drop the inpcb lock before performing copyin on
1589 * socket option arguments. When it re-acquires the lock after the copy, it
1590 * has to revalidate that the connection is still valid for the socket
1591 * option.
1592 * XXXGL: review if this is really needed
1593 */
1594 #define INP_WLOCK_RECHECK_CLEANUP(inp, cleanup) do { \
1595 INP_WLOCK(inp); \
1596 if (tp->t_flags & TF_DISCONNECTED) { \
1597 INP_WUNLOCK(inp); \
1598 cleanup; \
1599 return (ECONNRESET); \
1600 } \
1601 } while(0)
1602 #define INP_WLOCK_RECHECK(inp) INP_WLOCK_RECHECK_CLEANUP((inp), /* noop */)
1603
1604 int
tcp_ctloutput_set(struct inpcb * inp,struct sockopt * sopt)1605 tcp_ctloutput_set(struct inpcb *inp, struct sockopt *sopt)
1606 {
1607 struct socket *so = inp->inp_socket;
1608 struct tcpcb *tp = intotcpcb(inp);
1609 int error = 0;
1610
1611 MPASS(sopt->sopt_dir == SOPT_SET);
1612 INP_WLOCK_ASSERT(inp);
1613 KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
1614 ("tp_flags == %x", tp->t_flags));
1615 KASSERT(so != NULL, ("inp_socket == NULL"));
1616
1617 if (sopt->sopt_level != IPPROTO_TCP) {
1618 INP_WUNLOCK(inp);
1619 #ifdef INET6
1620 if (inp->inp_vflag & INP_IPV6PROTO)
1621 error = ip6_ctloutput(so, sopt);
1622 #endif
1623 #if defined(INET6) && defined(INET)
1624 else
1625 #endif
1626 #ifdef INET
1627 error = ip_ctloutput(so, sopt);
1628 #endif
1629 /*
1630 * When an IP-level socket option affects TCP, pass control
1631 * down to stack tfb_tcp_ctloutput, otherwise return what
1632 * IP level returned.
1633 */
1634 switch (sopt->sopt_level) {
1635 #ifdef INET6
1636 case IPPROTO_IPV6:
1637 if ((inp->inp_vflag & INP_IPV6PROTO) == 0)
1638 return (error);
1639 switch (sopt->sopt_name) {
1640 case IPV6_TCLASS:
1641 /* Notify tcp stacks that care (e.g. RACK). */
1642 break;
1643 case IPV6_USE_MIN_MTU:
1644 /* Update t_maxseg accordingly. */
1645 break;
1646 default:
1647 return (error);
1648 }
1649 break;
1650 #endif
1651 #ifdef INET
1652 case IPPROTO_IP:
1653 switch (sopt->sopt_name) {
1654 case IP_TOS:
1655 inp->inp_ip_tos &= ~IPTOS_ECN_MASK;
1656 break;
1657 case IP_TTL:
1658 /* Notify tcp stacks that care (e.g. RACK). */
1659 break;
1660 default:
1661 return (error);
1662 }
1663 break;
1664 #endif
1665 default:
1666 return (error);
1667 }
1668 INP_WLOCK_RECHECK(inp);
1669 } else if (sopt->sopt_name == TCP_FUNCTION_BLK) {
1670 /*
1671 * Protect the TCP option TCP_FUNCTION_BLK so
1672 * that a sub-function can *never* overwrite this.
1673 */
1674 struct tcp_function_set fsn;
1675 struct tcp_function_block *blk;
1676 void *ptr = NULL;
1677
1678 INP_WUNLOCK(inp);
1679 error = sooptcopyin(sopt, &fsn, sizeof fsn, sizeof fsn);
1680 if (error)
1681 return (error);
1682
1683 INP_WLOCK_RECHECK(inp);
1684
1685 blk = find_and_ref_tcp_functions(&fsn);
1686 if (blk == NULL) {
1687 INP_WUNLOCK(inp);
1688 return (ENOENT);
1689 }
1690 if (tp->t_fb == blk) {
1691 /* You already have this */
1692 refcount_release(&blk->tfb_refcnt);
1693 INP_WUNLOCK(inp);
1694 return (0);
1695 }
1696 if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1697 refcount_release(&blk->tfb_refcnt);
1698 INP_WUNLOCK(inp);
1699 return (ENOENT);
1700 }
1701 error = (*blk->tfb_tcp_handoff_ok)(tp);
1702 if (error) {
1703 refcount_release(&blk->tfb_refcnt);
1704 INP_WUNLOCK(inp);
1705 return (error);
1706 }
1707 /*
1708 * Ensure the new stack takes ownership with a
1709 * clean slate on peak rate threshold.
1710 */
1711 if (tp->t_fb->tfb_tcp_timer_stop_all != NULL)
1712 tp->t_fb->tfb_tcp_timer_stop_all(tp);
1713 if (blk->tfb_tcp_fb_init) {
1714 error = (*blk->tfb_tcp_fb_init)(tp, &ptr);
1715 if (error) {
1716 /*
1717 * Release the ref count the lookup
1718 * acquired.
1719 */
1720 refcount_release(&blk->tfb_refcnt);
1721 /*
1722 * Now there is a chance that the
1723 * init() function mucked with some
1724 * things before it failed, such as
1725 * hpts or inp_flags2 or timer granularity.
1726 * It should not of, but lets give the old
1727 * stack a chance to reset to a known good state.
1728 */
1729 if (tp->t_fb->tfb_switch_failed) {
1730 (*tp->t_fb->tfb_switch_failed)(tp);
1731 }
1732 goto err_out;
1733 }
1734 }
1735 if (tp->t_fb->tfb_tcp_fb_fini) {
1736 struct epoch_tracker et;
1737 /*
1738 * Tell the stack to cleanup with 0 i.e.
1739 * the tcb is not going away.
1740 */
1741 NET_EPOCH_ENTER(et);
1742 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
1743 NET_EPOCH_EXIT(et);
1744 }
1745 /*
1746 * Release the old refcnt, the
1747 * lookup acquired a ref on the
1748 * new one already.
1749 */
1750 refcount_release(&tp->t_fb->tfb_refcnt);
1751 /*
1752 * Set in the new stack.
1753 */
1754 tp->t_fb = blk;
1755 tp->t_fb_ptr = ptr;
1756 #ifdef TCP_OFFLOAD
1757 if (tp->t_flags & TF_TOE) {
1758 tcp_offload_ctloutput(tp, sopt->sopt_dir,
1759 sopt->sopt_name);
1760 }
1761 #endif
1762 err_out:
1763 INP_WUNLOCK(inp);
1764 return (error);
1765
1766 }
1767
1768 /* Pass in the INP locked, callee must unlock it. */
1769 return (tp->t_fb->tfb_tcp_ctloutput(tp, sopt));
1770 }
1771
1772 static int
tcp_ctloutput_get(struct inpcb * inp,struct sockopt * sopt)1773 tcp_ctloutput_get(struct inpcb *inp, struct sockopt *sopt)
1774 {
1775 struct socket *so = inp->inp_socket;
1776 struct tcpcb *tp = intotcpcb(inp);
1777 int error = 0;
1778
1779 MPASS(sopt->sopt_dir == SOPT_GET);
1780 INP_WLOCK_ASSERT(inp);
1781 KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
1782 ("tp_flags == %x", tp->t_flags));
1783 KASSERT(so != NULL, ("inp_socket == NULL"));
1784
1785 if (sopt->sopt_level != IPPROTO_TCP) {
1786 INP_WUNLOCK(inp);
1787 #ifdef INET6
1788 if (inp->inp_vflag & INP_IPV6PROTO)
1789 error = ip6_ctloutput(so, sopt);
1790 #endif /* INET6 */
1791 #if defined(INET6) && defined(INET)
1792 else
1793 #endif
1794 #ifdef INET
1795 error = ip_ctloutput(so, sopt);
1796 #endif
1797 return (error);
1798 }
1799 if (((sopt->sopt_name == TCP_FUNCTION_BLK) ||
1800 (sopt->sopt_name == TCP_FUNCTION_ALIAS))) {
1801 struct tcp_function_set fsn;
1802
1803 if (sopt->sopt_name == TCP_FUNCTION_ALIAS) {
1804 memset(&fsn, 0, sizeof(fsn));
1805 find_tcp_function_alias(tp->t_fb, &fsn);
1806 } else {
1807 strncpy(fsn.function_set_name,
1808 tp->t_fb->tfb_tcp_block_name,
1809 TCP_FUNCTION_NAME_LEN_MAX);
1810 fsn.function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
1811 }
1812 fsn.pcbcnt = tp->t_fb->tfb_refcnt;
1813 INP_WUNLOCK(inp);
1814 error = sooptcopyout(sopt, &fsn, sizeof fsn);
1815 return (error);
1816 }
1817
1818 /* Pass in the INP locked, callee must unlock it. */
1819 return (tp->t_fb->tfb_tcp_ctloutput(tp, sopt));
1820 }
1821
1822 int
tcp_ctloutput(struct socket * so,struct sockopt * sopt)1823 tcp_ctloutput(struct socket *so, struct sockopt *sopt)
1824 {
1825 struct inpcb *inp = sotoinpcb(so);
1826 struct tcpcb *tp = intotcpcb(inp);
1827
1828 INP_WLOCK(inp);
1829 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
1830 INP_WUNLOCK(inp);
1831 return (ECONNRESET);
1832 }
1833 if (sopt->sopt_dir == SOPT_SET)
1834 return (tcp_ctloutput_set(inp, sopt));
1835 else if (sopt->sopt_dir == SOPT_GET)
1836 return (tcp_ctloutput_get(inp, sopt));
1837 else
1838 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
1839 }
1840
1841 /*
1842 * If this assert becomes untrue, we need to change the size of the buf
1843 * variable in tcp_default_ctloutput().
1844 */
1845 #ifdef CTASSERT
1846 CTASSERT(TCP_CA_NAME_MAX <= TCP_LOG_ID_LEN);
1847 CTASSERT(TCP_LOG_REASON_LEN <= TCP_LOG_ID_LEN);
1848 #endif
1849
1850 extern struct cc_algo newreno_cc_algo;
1851
1852 static int
tcp_set_cc_mod(struct inpcb * inp,struct sockopt * sopt)1853 tcp_set_cc_mod(struct inpcb *inp, struct sockopt *sopt)
1854 {
1855 struct cc_algo *algo;
1856 void *ptr = NULL;
1857 struct tcpcb *tp = intotcpcb(inp);
1858 struct cc_var cc_mem;
1859 char buf[TCP_CA_NAME_MAX];
1860 size_t mem_sz;
1861 int error;
1862
1863 INP_WUNLOCK(inp);
1864 error = sooptcopyin(sopt, buf, TCP_CA_NAME_MAX - 1, 1);
1865 if (error)
1866 return(error);
1867 buf[sopt->sopt_valsize] = '\0';
1868 CC_LIST_RLOCK();
1869 STAILQ_FOREACH(algo, &cc_list, entries) {
1870 if (strncmp(buf, algo->name,
1871 TCP_CA_NAME_MAX) == 0) {
1872 if (algo->flags & CC_MODULE_BEING_REMOVED) {
1873 /* We can't "see" modules being unloaded */
1874 continue;
1875 }
1876 break;
1877 }
1878 }
1879 if (algo == NULL) {
1880 CC_LIST_RUNLOCK();
1881 return(ESRCH);
1882 }
1883 /*
1884 * With a reference the algorithm cannot be removed
1885 * so we hold a reference through the change process.
1886 */
1887 cc_refer(algo);
1888 CC_LIST_RUNLOCK();
1889 if (algo->cb_init != NULL) {
1890 /* We can now pre-get the memory for the CC */
1891 mem_sz = (*algo->cc_data_sz)();
1892 if (mem_sz == 0) {
1893 goto no_mem_needed;
1894 }
1895 ptr = malloc(mem_sz, M_CC_MEM, M_WAITOK);
1896 } else {
1897 no_mem_needed:
1898 mem_sz = 0;
1899 ptr = NULL;
1900 }
1901 /*
1902 * Make sure its all clean and zero and also get
1903 * back the inplock.
1904 */
1905 memset(&cc_mem, 0, sizeof(cc_mem));
1906 INP_WLOCK(inp);
1907 if (__predict_false(tp->t_flags & TF_DISCONNECTED)) {
1908 INP_WUNLOCK(inp);
1909 if (ptr)
1910 free(ptr, M_CC_MEM);
1911 /* Release our temp reference */
1912 CC_LIST_RLOCK();
1913 cc_release(algo);
1914 CC_LIST_RUNLOCK();
1915 return (ECONNRESET);
1916 }
1917 if (ptr != NULL)
1918 memset(ptr, 0, mem_sz);
1919 cc_mem.tp = tp;
1920 /*
1921 * We once again hold a write lock over the tcb so it's
1922 * safe to do these things without ordering concerns.
1923 * Note here we init into stack memory.
1924 */
1925 if (algo->cb_init != NULL)
1926 error = algo->cb_init(&cc_mem, ptr);
1927 else
1928 error = 0;
1929 /*
1930 * The CC algorithms, when given their memory
1931 * should not fail we could in theory have a
1932 * KASSERT here.
1933 */
1934 if (error == 0) {
1935 /*
1936 * Touchdown, lets go ahead and move the
1937 * connection to the new CC module by
1938 * copying in the cc_mem after we call
1939 * the old ones cleanup (if any).
1940 */
1941 if (CC_ALGO(tp)->cb_destroy != NULL)
1942 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
1943 /* Detach the old CC from the tcpcb */
1944 cc_detach(tp);
1945 /* Copy in our temp memory that was inited */
1946 memcpy(&tp->t_ccv, &cc_mem, sizeof(struct cc_var));
1947 /* Now attach the new, which takes a reference */
1948 cc_attach(tp, algo);
1949 /* Ok now are we where we have gotten past any conn_init? */
1950 if (TCPS_HAVEESTABLISHED(tp->t_state) && (CC_ALGO(tp)->conn_init != NULL)) {
1951 /* Yep run the connection init for the new CC */
1952 CC_ALGO(tp)->conn_init(&tp->t_ccv);
1953 }
1954 } else if (ptr)
1955 free(ptr, M_CC_MEM);
1956 INP_WUNLOCK(inp);
1957 /* Now lets release our temp reference */
1958 CC_LIST_RLOCK();
1959 cc_release(algo);
1960 CC_LIST_RUNLOCK();
1961 return (error);
1962 }
1963
1964 int
tcp_default_ctloutput(struct tcpcb * tp,struct sockopt * sopt)1965 tcp_default_ctloutput(struct tcpcb *tp, struct sockopt *sopt)
1966 {
1967 struct inpcb *inp = tptoinpcb(tp);
1968 int error, opt, optval;
1969 u_int ui;
1970 struct tcp_info ti;
1971 #ifdef KERN_TLS
1972 struct tls_enable tls;
1973 struct socket *so = inp->inp_socket;
1974 #endif
1975 char *pbuf, buf[TCP_LOG_ID_LEN];
1976 #ifdef STATS
1977 struct statsblob *sbp;
1978 #endif
1979 size_t len;
1980
1981 INP_WLOCK_ASSERT(inp);
1982 KASSERT((tp->t_flags & TF_DISCONNECTED) == 0,
1983 ("tp_flags == %x", tp->t_flags));
1984 KASSERT(inp->inp_socket != NULL, ("inp_socket == NULL"));
1985
1986 switch (sopt->sopt_level) {
1987 #ifdef INET6
1988 case IPPROTO_IPV6:
1989 MPASS(inp->inp_vflag & INP_IPV6PROTO);
1990 switch (sopt->sopt_name) {
1991 case IPV6_USE_MIN_MTU:
1992 tcp6_use_min_mtu(tp);
1993 /* FALLTHROUGH */
1994 }
1995 INP_WUNLOCK(inp);
1996 return (0);
1997 #endif
1998 #ifdef INET
1999 case IPPROTO_IP:
2000 INP_WUNLOCK(inp);
2001 return (0);
2002 #endif
2003 }
2004
2005 /*
2006 * For TCP_CCALGOOPT forward the control to CC module, for both
2007 * SOPT_SET and SOPT_GET.
2008 */
2009 switch (sopt->sopt_name) {
2010 case TCP_CCALGOOPT:
2011 INP_WUNLOCK(inp);
2012 if (sopt->sopt_valsize > CC_ALGOOPT_LIMIT)
2013 return (EINVAL);
2014 pbuf = malloc(sopt->sopt_valsize, M_TEMP, M_WAITOK | M_ZERO);
2015 error = sooptcopyin(sopt, pbuf, sopt->sopt_valsize,
2016 sopt->sopt_valsize);
2017 if (error) {
2018 free(pbuf, M_TEMP);
2019 return (error);
2020 }
2021 INP_WLOCK_RECHECK_CLEANUP(inp, free(pbuf, M_TEMP));
2022 if (CC_ALGO(tp)->ctl_output != NULL)
2023 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, sopt, pbuf);
2024 else
2025 error = ENOENT;
2026 INP_WUNLOCK(inp);
2027 if (error == 0 && sopt->sopt_dir == SOPT_GET)
2028 error = sooptcopyout(sopt, pbuf, sopt->sopt_valsize);
2029 free(pbuf, M_TEMP);
2030 return (error);
2031 }
2032
2033 switch (sopt->sopt_dir) {
2034 case SOPT_SET:
2035 switch (sopt->sopt_name) {
2036 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2037 case TCP_MD5SIG:
2038 INP_WUNLOCK(inp);
2039 if (!TCPMD5_ENABLED())
2040 return (ENOPROTOOPT);
2041 error = TCPMD5_PCBCTL(inp, sopt);
2042 if (error)
2043 return (error);
2044 INP_WLOCK_RECHECK(inp);
2045 goto unlock_and_done;
2046 #endif /* IPSEC */
2047
2048 case TCP_NODELAY:
2049 case TCP_NOOPT:
2050 INP_WUNLOCK(inp);
2051 error = sooptcopyin(sopt, &optval, sizeof optval,
2052 sizeof optval);
2053 if (error)
2054 return (error);
2055
2056 INP_WLOCK_RECHECK(inp);
2057 switch (sopt->sopt_name) {
2058 case TCP_NODELAY:
2059 opt = TF_NODELAY;
2060 break;
2061 case TCP_NOOPT:
2062 opt = TF_NOOPT;
2063 break;
2064 default:
2065 opt = 0; /* dead code to fool gcc */
2066 break;
2067 }
2068
2069 if (optval)
2070 tp->t_flags |= opt;
2071 else
2072 tp->t_flags &= ~opt;
2073 unlock_and_done:
2074 #ifdef TCP_OFFLOAD
2075 if (tp->t_flags & TF_TOE) {
2076 tcp_offload_ctloutput(tp, sopt->sopt_dir,
2077 sopt->sopt_name);
2078 }
2079 #endif
2080 INP_WUNLOCK(inp);
2081 break;
2082
2083 case TCP_NOPUSH:
2084 INP_WUNLOCK(inp);
2085 error = sooptcopyin(sopt, &optval, sizeof optval,
2086 sizeof optval);
2087 if (error)
2088 return (error);
2089
2090 INP_WLOCK_RECHECK(inp);
2091 if (optval)
2092 tp->t_flags |= TF_NOPUSH;
2093 else if (tp->t_flags & TF_NOPUSH) {
2094 tp->t_flags &= ~TF_NOPUSH;
2095 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
2096 struct epoch_tracker et;
2097
2098 NET_EPOCH_ENTER(et);
2099 error = tcp_output_nodrop(tp);
2100 NET_EPOCH_EXIT(et);
2101 }
2102 }
2103 goto unlock_and_done;
2104
2105 case TCP_REMOTE_UDP_ENCAPS_PORT:
2106 INP_WUNLOCK(inp);
2107 error = sooptcopyin(sopt, &optval, sizeof optval,
2108 sizeof optval);
2109 if (error)
2110 return (error);
2111 if ((optval < TCP_TUNNELING_PORT_MIN) ||
2112 (optval > TCP_TUNNELING_PORT_MAX)) {
2113 /* Its got to be in range */
2114 return (EINVAL);
2115 }
2116 if ((V_tcp_udp_tunneling_port == 0) && (optval != 0)) {
2117 /* You have to have enabled a UDP tunneling port first */
2118 return (EINVAL);
2119 }
2120 INP_WLOCK_RECHECK(inp);
2121 if (tp->t_state != TCPS_CLOSED) {
2122 /* You can't change after you are connected */
2123 error = EINVAL;
2124 } else {
2125 /* Ok we are all good set the port */
2126 tp->t_port = htons(optval);
2127 }
2128 goto unlock_and_done;
2129
2130 case TCP_MAXSEG:
2131 INP_WUNLOCK(inp);
2132 error = sooptcopyin(sopt, &optval, sizeof optval,
2133 sizeof optval);
2134 if (error)
2135 return (error);
2136
2137 INP_WLOCK_RECHECK(inp);
2138 if (optval > 0 && optval <= tp->t_maxseg &&
2139 optval + 40 >= V_tcp_minmss) {
2140 tp->t_maxseg = optval;
2141 if (tp->t_maxseg < V_tcp_mssdflt) {
2142 /*
2143 * The MSS is so small we should not process incoming
2144 * SACK's since we are subject to attack in such a
2145 * case.
2146 */
2147 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
2148 } else {
2149 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
2150 }
2151 } else
2152 error = EINVAL;
2153 goto unlock_and_done;
2154
2155 case TCP_INFO:
2156 INP_WUNLOCK(inp);
2157 error = EINVAL;
2158 break;
2159
2160 case TCP_STATS:
2161 INP_WUNLOCK(inp);
2162 #ifdef STATS
2163 error = sooptcopyin(sopt, &optval, sizeof optval,
2164 sizeof optval);
2165 if (error)
2166 return (error);
2167
2168 if (optval > 0)
2169 sbp = stats_blob_alloc(
2170 V_tcp_perconn_stats_dflt_tpl, 0);
2171 else
2172 sbp = NULL;
2173
2174 INP_WLOCK_RECHECK(inp);
2175 if ((tp->t_stats != NULL && sbp == NULL) ||
2176 (tp->t_stats == NULL && sbp != NULL)) {
2177 struct statsblob *t = tp->t_stats;
2178 tp->t_stats = sbp;
2179 sbp = t;
2180 }
2181 INP_WUNLOCK(inp);
2182
2183 stats_blob_destroy(sbp);
2184 #else
2185 return (EOPNOTSUPP);
2186 #endif /* !STATS */
2187 break;
2188
2189 case TCP_CONGESTION:
2190 error = tcp_set_cc_mod(inp, sopt);
2191 break;
2192
2193 case TCP_REUSPORT_LB_NUMA:
2194 INP_WUNLOCK(inp);
2195 error = sooptcopyin(sopt, &optval, sizeof(optval),
2196 sizeof(optval));
2197 INP_WLOCK_RECHECK(inp);
2198 if (!error)
2199 error = in_pcblbgroup_numa(inp, optval);
2200 INP_WUNLOCK(inp);
2201 break;
2202
2203 #ifdef KERN_TLS
2204 case TCP_TXTLS_ENABLE:
2205 INP_WUNLOCK(inp);
2206 error = ktls_copyin_tls_enable(sopt, &tls);
2207 if (error != 0)
2208 break;
2209 error = ktls_enable_tx(so, &tls);
2210 ktls_cleanup_tls_enable(&tls);
2211 break;
2212 case TCP_TXTLS_MODE:
2213 INP_WUNLOCK(inp);
2214 error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2215 if (error != 0)
2216 return (error);
2217
2218 INP_WLOCK_RECHECK(inp);
2219 error = ktls_set_tx_mode(so, ui);
2220 INP_WUNLOCK(inp);
2221 break;
2222 case TCP_RXTLS_ENABLE:
2223 INP_WUNLOCK(inp);
2224 error = ktls_copyin_tls_enable(sopt, &tls);
2225 if (error != 0)
2226 break;
2227 error = ktls_enable_rx(so, &tls);
2228 ktls_cleanup_tls_enable(&tls);
2229 break;
2230 #endif
2231 case TCP_MAXUNACKTIME:
2232 case TCP_KEEPIDLE:
2233 case TCP_KEEPINTVL:
2234 case TCP_KEEPINIT:
2235 INP_WUNLOCK(inp);
2236 error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2237 if (error)
2238 return (error);
2239
2240 if (ui > (UINT_MAX / hz)) {
2241 error = EINVAL;
2242 break;
2243 }
2244 ui *= hz;
2245
2246 INP_WLOCK_RECHECK(inp);
2247 switch (sopt->sopt_name) {
2248 case TCP_MAXUNACKTIME:
2249 tp->t_maxunacktime = ui;
2250 break;
2251
2252 case TCP_KEEPIDLE:
2253 tp->t_keepidle = ui;
2254 /*
2255 * XXX: better check current remaining
2256 * timeout and "merge" it with new value.
2257 */
2258 if ((tp->t_state > TCPS_LISTEN) &&
2259 (tp->t_state <= TCPS_CLOSING))
2260 tcp_timer_activate(tp, TT_KEEP,
2261 TP_KEEPIDLE(tp));
2262 break;
2263 case TCP_KEEPINTVL:
2264 tp->t_keepintvl = ui;
2265 if ((tp->t_state == TCPS_FIN_WAIT_2) &&
2266 (TP_MAXIDLE(tp) > 0))
2267 tcp_timer_activate(tp, TT_2MSL,
2268 TP_MAXIDLE(tp));
2269 break;
2270 case TCP_KEEPINIT:
2271 tp->t_keepinit = ui;
2272 if (tp->t_state == TCPS_SYN_RECEIVED ||
2273 tp->t_state == TCPS_SYN_SENT)
2274 tcp_timer_activate(tp, TT_KEEP,
2275 TP_KEEPINIT(tp));
2276 break;
2277 }
2278 goto unlock_and_done;
2279
2280 case TCP_KEEPCNT:
2281 INP_WUNLOCK(inp);
2282 error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2283 if (error)
2284 return (error);
2285
2286 INP_WLOCK_RECHECK(inp);
2287 tp->t_keepcnt = ui;
2288 if ((tp->t_state == TCPS_FIN_WAIT_2) &&
2289 (TP_MAXIDLE(tp) > 0))
2290 tcp_timer_activate(tp, TT_2MSL,
2291 TP_MAXIDLE(tp));
2292 goto unlock_and_done;
2293
2294 case TCP_FASTOPEN: {
2295 struct tcp_fastopen tfo_optval;
2296
2297 INP_WUNLOCK(inp);
2298 if (!V_tcp_fastopen_client_enable &&
2299 !V_tcp_fastopen_server_enable)
2300 return (EPERM);
2301
2302 error = sooptcopyin(sopt, &tfo_optval,
2303 sizeof(tfo_optval), sizeof(int));
2304 if (error)
2305 return (error);
2306
2307 INP_WLOCK_RECHECK(inp);
2308 if ((tp->t_state != TCPS_CLOSED) &&
2309 (tp->t_state != TCPS_LISTEN)) {
2310 error = EINVAL;
2311 goto unlock_and_done;
2312 }
2313 if (tfo_optval.enable) {
2314 if (tp->t_state == TCPS_LISTEN) {
2315 if (!V_tcp_fastopen_server_enable) {
2316 error = EPERM;
2317 goto unlock_and_done;
2318 }
2319
2320 if (tp->t_tfo_pending == NULL)
2321 tp->t_tfo_pending =
2322 tcp_fastopen_alloc_counter();
2323 } else {
2324 /*
2325 * If a pre-shared key was provided,
2326 * stash it in the client cookie
2327 * field of the tcpcb for use during
2328 * connect.
2329 */
2330 if (sopt->sopt_valsize ==
2331 sizeof(tfo_optval)) {
2332 memcpy(tp->t_tfo_cookie.client,
2333 tfo_optval.psk,
2334 TCP_FASTOPEN_PSK_LEN);
2335 tp->t_tfo_client_cookie_len =
2336 TCP_FASTOPEN_PSK_LEN;
2337 }
2338 }
2339 tp->t_flags |= TF_FASTOPEN;
2340 } else
2341 tp->t_flags &= ~TF_FASTOPEN;
2342 goto unlock_and_done;
2343 }
2344
2345 #ifdef TCP_BLACKBOX
2346 case TCP_LOG:
2347 INP_WUNLOCK(inp);
2348 error = sooptcopyin(sopt, &optval, sizeof optval,
2349 sizeof optval);
2350 if (error)
2351 return (error);
2352
2353 INP_WLOCK_RECHECK(inp);
2354 error = tcp_log_state_change(tp, optval);
2355 goto unlock_and_done;
2356
2357 case TCP_LOGBUF:
2358 INP_WUNLOCK(inp);
2359 error = EINVAL;
2360 break;
2361
2362 case TCP_LOGID:
2363 INP_WUNLOCK(inp);
2364 error = sooptcopyin(sopt, buf, TCP_LOG_ID_LEN - 1, 0);
2365 if (error)
2366 break;
2367 buf[sopt->sopt_valsize] = '\0';
2368 INP_WLOCK_RECHECK(inp);
2369 error = tcp_log_set_id(tp, buf);
2370 /* tcp_log_set_id() unlocks the INP. */
2371 break;
2372
2373 case TCP_LOGDUMP:
2374 case TCP_LOGDUMPID:
2375 INP_WUNLOCK(inp);
2376 error =
2377 sooptcopyin(sopt, buf, TCP_LOG_REASON_LEN - 1, 0);
2378 if (error)
2379 break;
2380 buf[sopt->sopt_valsize] = '\0';
2381 INP_WLOCK_RECHECK(inp);
2382 if (sopt->sopt_name == TCP_LOGDUMP) {
2383 error = tcp_log_dump_tp_logbuf(tp, buf,
2384 M_WAITOK, true);
2385 INP_WUNLOCK(inp);
2386 } else {
2387 tcp_log_dump_tp_bucket_logbufs(tp, buf);
2388 /*
2389 * tcp_log_dump_tp_bucket_logbufs() drops the
2390 * INP lock.
2391 */
2392 }
2393 break;
2394 #endif
2395
2396 default:
2397 INP_WUNLOCK(inp);
2398 error = ENOPROTOOPT;
2399 break;
2400 }
2401 break;
2402
2403 case SOPT_GET:
2404 switch (sopt->sopt_name) {
2405 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2406 case TCP_MD5SIG:
2407 INP_WUNLOCK(inp);
2408 if (!TCPMD5_ENABLED())
2409 return (ENOPROTOOPT);
2410 error = TCPMD5_PCBCTL(inp, sopt);
2411 break;
2412 #endif
2413
2414 case TCP_NODELAY:
2415 optval = tp->t_flags & TF_NODELAY;
2416 INP_WUNLOCK(inp);
2417 error = sooptcopyout(sopt, &optval, sizeof optval);
2418 break;
2419 case TCP_MAXSEG:
2420 optval = tp->t_maxseg;
2421 INP_WUNLOCK(inp);
2422 error = sooptcopyout(sopt, &optval, sizeof optval);
2423 break;
2424 case TCP_REMOTE_UDP_ENCAPS_PORT:
2425 optval = ntohs(tp->t_port);
2426 INP_WUNLOCK(inp);
2427 error = sooptcopyout(sopt, &optval, sizeof optval);
2428 break;
2429 case TCP_NOOPT:
2430 optval = tp->t_flags & TF_NOOPT;
2431 INP_WUNLOCK(inp);
2432 error = sooptcopyout(sopt, &optval, sizeof optval);
2433 break;
2434 case TCP_NOPUSH:
2435 optval = tp->t_flags & TF_NOPUSH;
2436 INP_WUNLOCK(inp);
2437 error = sooptcopyout(sopt, &optval, sizeof optval);
2438 break;
2439 case TCP_INFO:
2440 tcp_fill_info(tp, &ti);
2441 INP_WUNLOCK(inp);
2442 error = sooptcopyout(sopt, &ti, sizeof ti);
2443 break;
2444 case TCP_STATS:
2445 {
2446 #ifdef STATS
2447 int nheld;
2448 TYPEOF_MEMBER(struct statsblob, flags) sbflags = 0;
2449
2450 error = 0;
2451 socklen_t outsbsz = sopt->sopt_valsize;
2452 if (tp->t_stats == NULL)
2453 error = ENOENT;
2454 else if (outsbsz >= tp->t_stats->cursz)
2455 outsbsz = tp->t_stats->cursz;
2456 else if (outsbsz >= sizeof(struct statsblob))
2457 outsbsz = sizeof(struct statsblob);
2458 else
2459 error = EINVAL;
2460 INP_WUNLOCK(inp);
2461 if (error)
2462 break;
2463
2464 sbp = sopt->sopt_val;
2465 nheld = atop(round_page(((vm_offset_t)sbp) +
2466 (vm_size_t)outsbsz) - trunc_page((vm_offset_t)sbp));
2467 vm_page_t ma[nheld];
2468 if (vm_fault_quick_hold_pages(
2469 &curproc->p_vmspace->vm_map, (vm_offset_t)sbp,
2470 outsbsz, VM_PROT_READ | VM_PROT_WRITE, ma,
2471 nheld) < 0) {
2472 error = EFAULT;
2473 break;
2474 }
2475
2476 if ((error = copyin_nofault(&(sbp->flags), &sbflags,
2477 SIZEOF_MEMBER(struct statsblob, flags))))
2478 goto unhold;
2479
2480 INP_WLOCK_RECHECK(inp);
2481 error = stats_blob_snapshot(&sbp, outsbsz, tp->t_stats,
2482 sbflags | SB_CLONE_USRDSTNOFAULT);
2483 INP_WUNLOCK(inp);
2484 sopt->sopt_valsize = outsbsz;
2485 unhold:
2486 vm_page_unhold_pages(ma, nheld);
2487 #else
2488 INP_WUNLOCK(inp);
2489 error = EOPNOTSUPP;
2490 #endif /* !STATS */
2491 break;
2492 }
2493 case TCP_CONGESTION:
2494 len = strlcpy(buf, CC_ALGO(tp)->name, TCP_CA_NAME_MAX);
2495 INP_WUNLOCK(inp);
2496 error = sooptcopyout(sopt, buf, len + 1);
2497 break;
2498 case TCP_MAXUNACKTIME:
2499 case TCP_KEEPIDLE:
2500 case TCP_KEEPINTVL:
2501 case TCP_KEEPINIT:
2502 case TCP_KEEPCNT:
2503 switch (sopt->sopt_name) {
2504 case TCP_MAXUNACKTIME:
2505 ui = TP_MAXUNACKTIME(tp) / hz;
2506 break;
2507 case TCP_KEEPIDLE:
2508 ui = TP_KEEPIDLE(tp) / hz;
2509 break;
2510 case TCP_KEEPINTVL:
2511 ui = TP_KEEPINTVL(tp) / hz;
2512 break;
2513 case TCP_KEEPINIT:
2514 ui = TP_KEEPINIT(tp) / hz;
2515 break;
2516 case TCP_KEEPCNT:
2517 ui = TP_KEEPCNT(tp);
2518 break;
2519 }
2520 INP_WUNLOCK(inp);
2521 error = sooptcopyout(sopt, &ui, sizeof(ui));
2522 break;
2523 case TCP_FASTOPEN:
2524 optval = tp->t_flags & TF_FASTOPEN;
2525 INP_WUNLOCK(inp);
2526 error = sooptcopyout(sopt, &optval, sizeof optval);
2527 break;
2528 #ifdef TCP_BLACKBOX
2529 case TCP_LOG:
2530 optval = tcp_get_bblog_state(tp);
2531 INP_WUNLOCK(inp);
2532 error = sooptcopyout(sopt, &optval, sizeof(optval));
2533 break;
2534 case TCP_LOGBUF:
2535 /* tcp_log_getlogbuf() does INP_WUNLOCK(inp) */
2536 error = tcp_log_getlogbuf(sopt, tp);
2537 break;
2538 case TCP_LOGID:
2539 len = tcp_log_get_id(tp, buf);
2540 INP_WUNLOCK(inp);
2541 error = sooptcopyout(sopt, buf, len + 1);
2542 break;
2543 case TCP_LOGDUMP:
2544 case TCP_LOGDUMPID:
2545 INP_WUNLOCK(inp);
2546 error = EINVAL;
2547 break;
2548 #endif
2549 #ifdef KERN_TLS
2550 case TCP_TXTLS_MODE:
2551 error = ktls_get_tx_mode(so, &optval);
2552 INP_WUNLOCK(inp);
2553 if (error == 0)
2554 error = sooptcopyout(sopt, &optval,
2555 sizeof(optval));
2556 break;
2557 case TCP_RXTLS_MODE:
2558 error = ktls_get_rx_mode(so, &optval);
2559 INP_WUNLOCK(inp);
2560 if (error == 0)
2561 error = sooptcopyout(sopt, &optval,
2562 sizeof(optval));
2563 break;
2564 #endif
2565 default:
2566 INP_WUNLOCK(inp);
2567 error = ENOPROTOOPT;
2568 break;
2569 }
2570 break;
2571 }
2572 return (error);
2573 }
2574 #undef INP_WLOCK_RECHECK
2575 #undef INP_WLOCK_RECHECK_CLEANUP
2576
2577 /*
2578 * Initiate (or continue) disconnect.
2579 * If embryonic state, just send reset (once).
2580 * If in ``let data drain'' option and linger null, just drop.
2581 * Otherwise (hard), mark socket disconnecting and drop
2582 * current input data; switch states based on user close, and
2583 * send segment to peer (with FIN).
2584 */
2585 static void
tcp_disconnect(struct tcpcb * tp)2586 tcp_disconnect(struct tcpcb *tp)
2587 {
2588 struct socket *so = tptosocket(tp);
2589
2590 NET_EPOCH_ASSERT();
2591 INP_WLOCK_ASSERT(tptoinpcb(tp));
2592
2593 /*
2594 * Neither tcp_close() nor tcp_drop() should return NULL, as the
2595 * socket is still open.
2596 */
2597 if (tp->t_state < TCPS_ESTABLISHED &&
2598 !(tp->t_state > TCPS_LISTEN && (tp->t_flags & TF_FASTOPEN))) {
2599 tp = tcp_close(tp);
2600 KASSERT(tp != NULL,
2601 ("tcp_disconnect: tcp_close() returned NULL"));
2602 } else if ((so->so_options & SO_LINGER) && so->so_linger == 0) {
2603 tp = tcp_drop(tp, 0);
2604 KASSERT(tp != NULL,
2605 ("tcp_disconnect: tcp_drop() returned NULL"));
2606 } else {
2607 soisdisconnecting(so);
2608 sbflush(&so->so_rcv);
2609 tcp_usrclosed(tp);
2610 if (!(tp->t_flags & TF_DISCONNECTED))
2611 /* Ignore stack's drop request, we already at it. */
2612 (void)tcp_output_nodrop(tp);
2613 }
2614 }
2615
2616 /*
2617 * User issued close, and wish to trail through shutdown states:
2618 * if never received SYN, just forget it. If got a SYN from peer,
2619 * but haven't sent FIN, then go to FIN_WAIT_1 state to send peer a FIN.
2620 * If already got a FIN from peer, then almost done; go to LAST_ACK
2621 * state. In all other cases, have already sent FIN to peer (e.g.
2622 * after PRU_SHUTDOWN), and just have to play tedious game waiting
2623 * for peer to send FIN or not respond to keep-alives, etc.
2624 * We can let the user exit from the close as soon as the FIN is acked.
2625 */
2626 static void
tcp_usrclosed(struct tcpcb * tp)2627 tcp_usrclosed(struct tcpcb *tp)
2628 {
2629
2630 NET_EPOCH_ASSERT();
2631 INP_WLOCK_ASSERT(tptoinpcb(tp));
2632
2633 switch (tp->t_state) {
2634 case TCPS_LISTEN:
2635 case TCPS_CLOSED:
2636 tp = tcp_close(tp);
2637 /*
2638 * tcp_close() should never return NULL here as the socket is
2639 * still open.
2640 */
2641 KASSERT(tp != NULL,
2642 ("tcp_usrclosed: tcp_close() returned NULL"));
2643 break;
2644
2645 case TCPS_SYN_SENT:
2646 case TCPS_SYN_RECEIVED:
2647 tp->t_flags |= TF_NEEDFIN;
2648 break;
2649
2650 case TCPS_ESTABLISHED:
2651 tcp_state_change(tp, TCPS_FIN_WAIT_1);
2652 break;
2653
2654 case TCPS_CLOSE_WAIT:
2655 tcp_state_change(tp, TCPS_LAST_ACK);
2656 break;
2657 }
2658 if (tp->t_acktime == 0)
2659 tp->t_acktime = ticks;
2660 if (tp->t_state >= TCPS_FIN_WAIT_2) {
2661 tcp_free_sackholes(tp);
2662 soisdisconnected(tptosocket(tp));
2663 /* Prevent the connection hanging in FIN_WAIT_2 forever. */
2664 if (tp->t_state == TCPS_FIN_WAIT_2) {
2665 int timeout;
2666
2667 timeout = (tcp_fast_finwait2_recycle) ?
2668 tcp_finwait2_timeout : TP_MAXIDLE(tp);
2669 tcp_timer_activate(tp, TT_2MSL, timeout);
2670 }
2671 }
2672 }
2673
2674 #ifdef DDB
2675 static void
db_print_indent(int indent)2676 db_print_indent(int indent)
2677 {
2678 int i;
2679
2680 for (i = 0; i < indent; i++)
2681 db_printf(" ");
2682 }
2683
2684 static void
db_print_tstate(int t_state)2685 db_print_tstate(int t_state)
2686 {
2687
2688 switch (t_state) {
2689 case TCPS_CLOSED:
2690 db_printf("TCPS_CLOSED");
2691 return;
2692
2693 case TCPS_LISTEN:
2694 db_printf("TCPS_LISTEN");
2695 return;
2696
2697 case TCPS_SYN_SENT:
2698 db_printf("TCPS_SYN_SENT");
2699 return;
2700
2701 case TCPS_SYN_RECEIVED:
2702 db_printf("TCPS_SYN_RECEIVED");
2703 return;
2704
2705 case TCPS_ESTABLISHED:
2706 db_printf("TCPS_ESTABLISHED");
2707 return;
2708
2709 case TCPS_CLOSE_WAIT:
2710 db_printf("TCPS_CLOSE_WAIT");
2711 return;
2712
2713 case TCPS_FIN_WAIT_1:
2714 db_printf("TCPS_FIN_WAIT_1");
2715 return;
2716
2717 case TCPS_CLOSING:
2718 db_printf("TCPS_CLOSING");
2719 return;
2720
2721 case TCPS_LAST_ACK:
2722 db_printf("TCPS_LAST_ACK");
2723 return;
2724
2725 case TCPS_FIN_WAIT_2:
2726 db_printf("TCPS_FIN_WAIT_2");
2727 return;
2728
2729 case TCPS_TIME_WAIT:
2730 db_printf("TCPS_TIME_WAIT");
2731 return;
2732
2733 default:
2734 db_printf("unknown");
2735 return;
2736 }
2737 }
2738
2739 static void
db_print_bblog_state(int state)2740 db_print_bblog_state(int state)
2741 {
2742 switch (state) {
2743 case TCP_LOG_STATE_RATIO_OFF:
2744 db_printf("TCP_LOG_STATE_RATIO_OFF");
2745 break;
2746 case TCP_LOG_STATE_CLEAR:
2747 db_printf("TCP_LOG_STATE_CLEAR");
2748 break;
2749 case TCP_LOG_STATE_OFF:
2750 db_printf("TCP_LOG_STATE_OFF");
2751 break;
2752 case TCP_LOG_STATE_TAIL:
2753 db_printf("TCP_LOG_STATE_TAIL");
2754 break;
2755 case TCP_LOG_STATE_HEAD:
2756 db_printf("TCP_LOG_STATE_HEAD");
2757 break;
2758 case TCP_LOG_STATE_HEAD_AUTO:
2759 db_printf("TCP_LOG_STATE_HEAD_AUTO");
2760 break;
2761 case TCP_LOG_STATE_CONTINUAL:
2762 db_printf("TCP_LOG_STATE_CONTINUAL");
2763 break;
2764 case TCP_LOG_STATE_TAIL_AUTO:
2765 db_printf("TCP_LOG_STATE_TAIL_AUTO");
2766 break;
2767 case TCP_LOG_VIA_BBPOINTS:
2768 db_printf("TCP_LOG_STATE_BBPOINTS");
2769 break;
2770 default:
2771 db_printf("UNKNOWN(%d)", state);
2772 break;
2773 }
2774 }
2775
2776 static void
db_print_tcpcb(struct tcpcb * tp,const char * name,int indent,bool show_bblog,bool show_inpcb,bool only_locked)2777 db_print_tcpcb(struct tcpcb *tp, const char *name, int indent, bool show_bblog,
2778 bool show_inpcb, bool only_locked)
2779 {
2780
2781 if (only_locked && tp->t_inpcb.inp_lock.rw_lock == RW_UNLOCKED)
2782 return;
2783
2784 db_print_indent(indent);
2785 db_printf("%s at %p\n", name, tp);
2786
2787 indent += 2;
2788
2789 if (show_inpcb)
2790 db_print_inpcb(tptoinpcb(tp), "t_inpcb", indent);
2791
2792 db_print_indent(indent);
2793 db_printf("t_segq first: %p t_segqlen: %d t_dupacks: %d\n",
2794 TAILQ_FIRST(&tp->t_segq), tp->t_segqlen, tp->t_dupacks);
2795
2796 db_print_indent(indent);
2797 db_printf("t_callout: %p t_timers: %p\n",
2798 &tp->t_callout, &tp->t_timers);
2799
2800 db_print_indent(indent);
2801 db_printf("t_state: %d (", tp->t_state);
2802 db_print_tstate(tp->t_state);
2803 db_printf(")\n");
2804
2805 db_print_indent(indent);
2806 db_printf("t_flags: 0x%b\n", tp->t_flags, TF_BITS);
2807
2808 db_print_indent(indent);
2809 db_printf("t_flags2: 0x%b\n", tp->t_flags2, TF2_BITS);
2810
2811 db_print_indent(indent);
2812 db_printf("snd_una: 0x%08x snd_max: 0x%08x snd_nxt: 0x%08x\n",
2813 tp->snd_una, tp->snd_max, tp->snd_nxt);
2814
2815 db_print_indent(indent);
2816 db_printf("snd_up: 0x%08x snd_wl1: 0x%08x snd_wl2: 0x%08x\n",
2817 tp->snd_up, tp->snd_wl1, tp->snd_wl2);
2818
2819 db_print_indent(indent);
2820 db_printf("iss: 0x%08x irs: 0x%08x rcv_nxt: 0x%08x\n",
2821 tp->iss, tp->irs, tp->rcv_nxt);
2822
2823 db_print_indent(indent);
2824 db_printf("rcv_adv: 0x%08x rcv_wnd: %u rcv_up: 0x%08x\n",
2825 tp->rcv_adv, tp->rcv_wnd, tp->rcv_up);
2826
2827 db_print_indent(indent);
2828 db_printf("snd_wnd: %u snd_cwnd: %u\n",
2829 tp->snd_wnd, tp->snd_cwnd);
2830
2831 db_print_indent(indent);
2832 db_printf("snd_ssthresh: %u snd_recover: "
2833 "0x%08x\n", tp->snd_ssthresh, tp->snd_recover);
2834
2835 db_print_indent(indent);
2836 db_printf("t_rcvtime: %u t_startime: %u\n",
2837 tp->t_rcvtime, tp->t_starttime);
2838
2839 db_print_indent(indent);
2840 db_printf("t_rttime: %u t_rtsq: 0x%08x\n",
2841 tp->t_rtttime, tp->t_rtseq);
2842
2843 db_print_indent(indent);
2844 db_printf("t_rxtcur: %d t_maxseg: %u t_srtt: %d\n",
2845 tp->t_rxtcur, tp->t_maxseg, tp->t_srtt);
2846
2847 db_print_indent(indent);
2848 db_printf("t_rttvar: %d t_rxtshift: %d t_rttmin: %u\n",
2849 tp->t_rttvar, tp->t_rxtshift, tp->t_rttmin);
2850
2851 db_print_indent(indent);
2852 db_printf("t_rttupdated: %u max_sndwnd: %u t_softerror: %d\n",
2853 tp->t_rttupdated, tp->max_sndwnd, tp->t_softerror);
2854
2855 db_print_indent(indent);
2856 db_printf("t_oobflags: 0x%b t_iobc: 0x%02x\n", tp->t_oobflags,
2857 TCPOOB_BITS, tp->t_iobc);
2858
2859 db_print_indent(indent);
2860 db_printf("snd_scale: %u rcv_scale: %u request_r_scale: %u\n",
2861 tp->snd_scale, tp->rcv_scale, tp->request_r_scale);
2862
2863 db_print_indent(indent);
2864 db_printf("ts_recent: %u ts_recent_age: %u\n",
2865 tp->ts_recent, tp->ts_recent_age);
2866
2867 db_print_indent(indent);
2868 db_printf("ts_offset: %u last_ack_sent: 0x%08x snd_cwnd_prev: "
2869 "%u\n", tp->ts_offset, tp->last_ack_sent, tp->snd_cwnd_prev);
2870
2871 db_print_indent(indent);
2872 db_printf("snd_ssthresh_prev: %u snd_recover_prev: 0x%08x "
2873 "t_badrxtwin: %u\n", tp->snd_ssthresh_prev,
2874 tp->snd_recover_prev, tp->t_badrxtwin);
2875
2876 db_print_indent(indent);
2877 db_printf("snd_numholes: %d snd_holes first: %p\n",
2878 tp->snd_numholes, TAILQ_FIRST(&tp->snd_holes));
2879
2880 db_print_indent(indent);
2881 db_printf("snd_fack: 0x%08x rcv_numsacks: %d\n",
2882 tp->snd_fack, tp->rcv_numsacks);
2883
2884 /* Skip sackblks, sackhint. */
2885
2886 db_print_indent(indent);
2887 db_printf("t_rttlow: %d rfbuf_ts: %u rfbuf_cnt: %d\n",
2888 tp->t_rttlow, tp->rfbuf_ts, tp->rfbuf_cnt);
2889
2890 db_print_indent(indent);
2891 db_printf("t_fb.tfb_tcp_block_name: %s\n", tp->t_fb->tfb_tcp_block_name);
2892
2893 db_print_indent(indent);
2894 db_printf("t_cc.name: %s\n", tp->t_cc->name);
2895
2896 db_print_indent(indent);
2897 db_printf("_t_logstate: %d (", tp->_t_logstate);
2898 db_print_bblog_state(tp->_t_logstate);
2899 db_printf(")\n");
2900
2901 db_print_indent(indent);
2902 db_printf("t_lognum: %d t_loglimit: %d t_logsn: %u\n",
2903 tp->t_lognum, tp->t_loglimit, tp->t_logsn);
2904
2905 if (show_bblog) {
2906 #ifdef TCP_BLACKBOX
2907 db_print_bblog_entries(&tp->t_logs, indent);
2908 #else
2909 db_print_indent(indent);
2910 db_printf("BBLog not supported\n");
2911 #endif
2912 }
2913 }
2914
DB_SHOW_COMMAND(tcpcb,db_show_tcpcb)2915 DB_SHOW_COMMAND(tcpcb, db_show_tcpcb)
2916 {
2917 struct tcpcb *tp;
2918 bool show_bblog, show_inpcb;
2919
2920 if (!have_addr) {
2921 db_printf("usage: show tcpcb[/bi] <addr>\n");
2922 return;
2923 }
2924 show_bblog = strchr(modif, 'b') != NULL;
2925 show_inpcb = strchr(modif, 'i') != NULL;
2926 tp = (struct tcpcb *)addr;
2927 db_print_tcpcb(tp, "tcpcb", 0, show_bblog, show_inpcb, false);
2928 }
2929
DB_SHOW_ALL_COMMAND(tcpcbs,db_show_all_tcpcbs)2930 DB_SHOW_ALL_COMMAND(tcpcbs, db_show_all_tcpcbs)
2931 {
2932 VNET_ITERATOR_DECL(vnet_iter);
2933 struct inpcb *inp;
2934 bool only_locked, show_bblog, show_inpcb;
2935
2936 only_locked = strchr(modif, 'l') != NULL;
2937 show_bblog = strchr(modif, 'b') != NULL;
2938 show_inpcb = strchr(modif, 'i') != NULL;
2939 VNET_FOREACH(vnet_iter) {
2940 CURVNET_SET(vnet_iter);
2941 for (u_int i = 0; i <= V_tcbinfo.ipi_hashmask; i++)
2942 CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_hash_exact[i].head,
2943 inp_hash_exact) {
2944 db_print_tcpcb(intotcpcb(inp), "tcpcb", 0,
2945 show_bblog, show_inpcb, only_locked);
2946 if (db_pager_quit)
2947 goto break_hash;
2948 }
2949 for (u_int i = 0; i <= V_tcbinfo.ipi_hashmask; i++)
2950 CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_hash_wild[i].head,
2951 inp_hash_wild) {
2952 db_print_tcpcb(intotcpcb(inp), "tcpcb", 0,
2953 show_bblog, show_inpcb, only_locked);
2954 if (db_pager_quit)
2955 goto break_hash;
2956 }
2957 break_hash:
2958 CK_LIST_FOREACH(inp, &V_tcbinfo.ipi_list_unconn.head,
2959 inp_unconn_list) {
2960 db_print_tcpcb(intotcpcb(inp), "tcpcb", 0,
2961 show_bblog, show_inpcb, only_locked);
2962 if (db_pager_quit)
2963 break;
2964 }
2965 CURVNET_RESTORE();
2966 }
2967 }
2968 #endif
2969