1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1988, 1990, 1993 5 * The Regents of the University of California. 6 * Copyright (c) 2004 The FreeBSD Foundation 7 * Copyright (c) 2004-2008 Robert N. M. Watson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pr_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pr_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pr_attach() has 50 * been successfully called. If pr_attach() returned an error, 51 * pr_detach() will not be called. Socket layer private. 52 * 53 * pr_abort() and pr_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pr_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. pr_fdclose() is called when userspace invokes close(2) on a socket 58 * file descriptor. 59 * 60 * socreate() creates a socket and attaches protocol state. This is a public 61 * interface that may be used by socket layer consumers to create new 62 * sockets. 63 * 64 * sonewconn() creates a socket and attaches protocol state. This is a 65 * public interface that may be used by protocols to create new sockets when 66 * a new connection is received and will be available for accept() on a 67 * listen socket. 68 * 69 * soclose() destroys a socket after possibly waiting for it to disconnect. 70 * This is a public interface that socket consumers should use to close and 71 * release a socket when done with it. 72 * 73 * soabort() destroys a socket without waiting for it to disconnect (used 74 * only for incoming connections that are already partially or fully 75 * connected). This is used internally by the socket layer when clearing 76 * listen socket queues (due to overflow or close on the listen socket), but 77 * is also a public interface protocols may use to abort connections in 78 * their incomplete listen queues should they no longer be required. Sockets 79 * placed in completed connection listen queues should not be aborted for 80 * reasons described in the comment above the soclose() implementation. This 81 * is not a general purpose close routine, and except in the specific 82 * circumstances described here, should not be used. 83 * 84 * sofree() will free a socket and its protocol state if all references on 85 * the socket have been released, and is the public interface to attempt to 86 * free a socket when a reference is removed. This is a socket layer private 87 * interface. 88 * 89 * NOTE: In addition to socreate() and soclose(), which provide a single 90 * socket reference to the consumer to be managed as required, there are two 91 * calls to explicitly manage socket references, soref(), and sorele(). 92 * Currently, these are generally required only when transitioning a socket 93 * from a listen queue to a file descriptor, in order to prevent garbage 94 * collection of the socket at an untimely moment. For a number of reasons, 95 * these interfaces are not preferred, and should be avoided. 96 * 97 * NOTE: With regard to VNETs the general rule is that callers do not set 98 * curvnet. Exceptions to this rule include soabort(), sodisconnect(), 99 * sofree(), sorele(), sonewconn() and sorflush(), which are usually called 100 * from a pre-set VNET context. sopoll_generic() currently does not need a 101 * VNET context to be set. 102 */ 103 104 #include <sys/cdefs.h> 105 #include "opt_inet.h" 106 #include "opt_inet6.h" 107 #include "opt_kern_tls.h" 108 #include "opt_ktrace.h" 109 #include "opt_sctp.h" 110 111 #include <sys/param.h> 112 #include <sys/systm.h> 113 #include <sys/capsicum.h> 114 #include <sys/fcntl.h> 115 #include <sys/limits.h> 116 #include <sys/lock.h> 117 #include <sys/mac.h> 118 #include <sys/malloc.h> 119 #include <sys/mbuf.h> 120 #include <sys/mutex.h> 121 #include <sys/domain.h> 122 #include <sys/file.h> /* for struct knote */ 123 #include <sys/hhook.h> 124 #include <sys/kernel.h> 125 #include <sys/khelp.h> 126 #include <sys/kthread.h> 127 #include <sys/ktls.h> 128 #include <sys/event.h> 129 #include <sys/eventhandler.h> 130 #include <sys/poll.h> 131 #include <sys/proc.h> 132 #include <sys/protosw.h> 133 #include <sys/sbuf.h> 134 #include <sys/socket.h> 135 #include <sys/socketvar.h> 136 #include <sys/resourcevar.h> 137 #include <net/route.h> 138 #include <sys/sched.h> 139 #include <sys/signalvar.h> 140 #include <sys/smp.h> 141 #include <sys/stat.h> 142 #include <sys/sx.h> 143 #include <sys/sysctl.h> 144 #include <sys/taskqueue.h> 145 #include <sys/uio.h> 146 #include <sys/un.h> 147 #include <sys/unpcb.h> 148 #include <sys/jail.h> 149 #include <sys/syslog.h> 150 #include <netinet/in.h> 151 #include <netinet/in_pcb.h> 152 #include <netinet/tcp.h> 153 154 #include <net/vnet.h> 155 156 #include <security/mac/mac_framework.h> 157 #include <security/mac/mac_internal.h> 158 159 #include <vm/uma.h> 160 161 #ifdef COMPAT_FREEBSD32 162 #include <sys/mount.h> 163 #include <sys/sysent.h> 164 #include <compat/freebsd32/freebsd32.h> 165 #endif 166 167 static int soreceive_generic_locked(struct socket *so, 168 struct sockaddr **psa, struct uio *uio, struct mbuf **mp, 169 struct mbuf **controlp, int *flagsp); 170 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 171 int flags); 172 static int soreceive_stream_locked(struct socket *so, struct sockbuf *sb, 173 struct sockaddr **psa, struct uio *uio, struct mbuf **mp, 174 struct mbuf **controlp, int flags); 175 static int sosend_generic_locked(struct socket *so, struct sockaddr *addr, 176 struct uio *uio, struct mbuf *top, struct mbuf *control, 177 int flags, struct thread *td); 178 static void so_rdknl_lock(void *); 179 static void so_rdknl_unlock(void *); 180 static void so_rdknl_assert_lock(void *, int); 181 static void so_wrknl_lock(void *); 182 static void so_wrknl_unlock(void *); 183 static void so_wrknl_assert_lock(void *, int); 184 185 static void filt_sordetach(struct knote *kn); 186 static int filt_soread(struct knote *kn, long hint); 187 static void filt_sowdetach(struct knote *kn); 188 static int filt_sowrite(struct knote *kn, long hint); 189 static int filt_soempty(struct knote *kn, long hint); 190 191 static const struct filterops soread_filtops = { 192 .f_isfd = 1, 193 .f_detach = filt_sordetach, 194 .f_event = filt_soread, 195 .f_copy = knote_triv_copy, 196 }; 197 static const struct filterops sowrite_filtops = { 198 .f_isfd = 1, 199 .f_detach = filt_sowdetach, 200 .f_event = filt_sowrite, 201 .f_copy = knote_triv_copy, 202 }; 203 static const struct filterops soempty_filtops = { 204 .f_isfd = 1, 205 .f_detach = filt_sowdetach, 206 .f_event = filt_soempty, 207 .f_copy = knote_triv_copy, 208 }; 209 210 so_gen_t so_gencnt; /* generation count for sockets */ 211 212 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 213 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 214 215 #define VNET_SO_ASSERT(so) \ 216 VNET_ASSERT(curvnet != NULL, \ 217 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so))); 218 219 #ifdef SOCKET_HHOOK 220 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]); 221 #define V_socket_hhh VNET(socket_hhh) 222 static inline int hhook_run_socket(struct socket *, void *, int32_t); 223 #endif 224 225 #ifdef COMPAT_FREEBSD32 226 #ifdef __amd64__ 227 /* off_t has 4-byte alignment on i386 but not on other 32-bit platforms. */ 228 #define __splice32_packed __packed 229 #else 230 #define __splice32_packed 231 #endif 232 struct splice32 { 233 int32_t sp_fd; 234 int64_t sp_max; 235 struct timeval32 sp_idle; 236 } __splice32_packed; 237 #undef __splice32_packed 238 #endif 239 240 /* 241 * Limit on the number of connections in the listen queue waiting 242 * for accept(2). 243 * NB: The original sysctl somaxconn is still available but hidden 244 * to prevent confusion about the actual purpose of this number. 245 */ 246 VNET_DEFINE_STATIC(u_int, somaxconn) = SOMAXCONN; 247 #define V_somaxconn VNET(somaxconn) 248 249 static int 250 sysctl_somaxconn(SYSCTL_HANDLER_ARGS) 251 { 252 int error; 253 u_int val; 254 255 val = V_somaxconn; 256 error = sysctl_handle_int(oidp, &val, 0, req); 257 if (error || !req->newptr ) 258 return (error); 259 260 /* 261 * The purpose of the UINT_MAX / 3 limit, is so that the formula 262 * 3 * sol_qlimit / 2 263 * below, will not overflow. 264 */ 265 266 if (val < 1 || val > UINT_MAX / 3) 267 return (EINVAL); 268 269 V_somaxconn = val; 270 return (0); 271 } 272 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue, 273 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, sizeof(u_int), 274 sysctl_somaxconn, "IU", 275 "Maximum listen socket pending connection accept queue size"); 276 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, 277 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, 278 sizeof(u_int), sysctl_somaxconn, "IU", 279 "Maximum listen socket pending connection accept queue size (compat)"); 280 281 static u_int numopensockets; 282 static int 283 sysctl_numopensockets(SYSCTL_HANDLER_ARGS) 284 { 285 u_int val; 286 287 #ifdef VIMAGE 288 if(!IS_DEFAULT_VNET(curvnet)) 289 val = curvnet->vnet_sockcnt; 290 else 291 #endif 292 val = numopensockets; 293 return (sysctl_handle_int(oidp, &val, 0, req)); 294 } 295 SYSCTL_PROC(_kern_ipc, OID_AUTO, numopensockets, 296 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, sizeof(u_int), 297 sysctl_numopensockets, "IU", "Number of open sockets"); 298 299 /* 300 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 301 * so_gencnt field. 302 */ 303 static struct mtx so_global_mtx; 304 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 305 306 /* 307 * General IPC sysctl name space, used by sockets and a variety of other IPC 308 * types. 309 */ 310 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 311 "IPC"); 312 313 /* 314 * Initialize the socket subsystem and set up the socket 315 * memory allocator. 316 */ 317 static uma_zone_t socket_zone; 318 int maxsockets; 319 320 static void 321 socket_zone_change(void *tag) 322 { 323 324 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 325 } 326 327 static int splice_init_state; 328 static struct sx splice_init_lock; 329 SX_SYSINIT(splice_init_lock, &splice_init_lock, "splice_init"); 330 331 static SYSCTL_NODE(_kern_ipc, OID_AUTO, splice, CTLFLAG_RW, 0, 332 "Settings relating to the SO_SPLICE socket option"); 333 334 static bool splice_receive_stream = true; 335 SYSCTL_BOOL(_kern_ipc_splice, OID_AUTO, receive_stream, CTLFLAG_RWTUN, 336 &splice_receive_stream, 0, 337 "Use soreceive_stream() for stream splices"); 338 339 static int splice_num_wq = -1; 340 static int 341 sysctl_splice_num_wq(SYSCTL_HANDLER_ARGS) 342 { 343 int error, new; 344 345 new = splice_num_wq; 346 error = sysctl_handle_int(oidp, &new, 0, req); 347 if (error == 0 && req->newptr && new != splice_num_wq) { 348 if (!cold) 349 sx_xlock(&splice_init_lock); 350 if (new < -1 || new > mp_ncpus || 351 (new <= 0 && splice_init_state != 0)) { 352 error = EINVAL; 353 } else { 354 splice_num_wq = new; 355 } 356 if (!cold) 357 sx_xunlock(&splice_init_lock); 358 } 359 return (error); 360 } 361 SYSCTL_PROC(_kern_ipc_splice, OID_AUTO, num_wq, 362 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 363 &splice_num_wq, 0, sysctl_splice_num_wq, "IU", 364 "Number of splice worker queues"); 365 366 static uma_zone_t splice_zone; 367 static struct proc *splice_proc; 368 struct splice_wq { 369 struct mtx mtx; 370 STAILQ_HEAD(, so_splice) head; 371 bool running; 372 } __aligned(CACHE_LINE_SIZE); 373 static struct splice_wq *splice_wq; 374 static uint32_t splice_index = 0; 375 376 static void so_splice_timeout(void *arg, int pending); 377 static void so_splice_xfer(struct so_splice *s); 378 static int so_unsplice(struct socket *so, bool timeout); 379 380 static void 381 splice_work_thread(void *ctx) 382 { 383 struct splice_wq *wq = ctx; 384 struct so_splice *s, *s_temp; 385 STAILQ_HEAD(, so_splice) local_head; 386 int cpu; 387 388 cpu = wq - splice_wq; 389 if (bootverbose) 390 printf("starting so_splice worker thread for CPU %d\n", cpu); 391 392 for (;;) { 393 mtx_lock(&wq->mtx); 394 while (STAILQ_EMPTY(&wq->head)) { 395 wq->running = false; 396 mtx_sleep(wq, &wq->mtx, 0, "-", 0); 397 wq->running = true; 398 } 399 STAILQ_INIT(&local_head); 400 STAILQ_CONCAT(&local_head, &wq->head); 401 STAILQ_INIT(&wq->head); 402 mtx_unlock(&wq->mtx); 403 STAILQ_FOREACH_SAFE(s, &local_head, next, s_temp) { 404 mtx_lock(&s->mtx); 405 CURVNET_SET(s->src->so_vnet); 406 so_splice_xfer(s); 407 CURVNET_RESTORE(); 408 } 409 } 410 } 411 412 static void 413 so_splice_dispatch_async(struct so_splice *sp) 414 { 415 struct splice_wq *wq; 416 bool running; 417 418 wq = &splice_wq[sp->wq_index]; 419 mtx_lock(&wq->mtx); 420 STAILQ_INSERT_TAIL(&wq->head, sp, next); 421 running = wq->running; 422 mtx_unlock(&wq->mtx); 423 if (!running) 424 wakeup(wq); 425 } 426 427 void 428 so_splice_dispatch(struct so_splice *sp) 429 { 430 mtx_assert(&sp->mtx, MA_OWNED); 431 432 if (sp->state != SPLICE_IDLE) { 433 mtx_unlock(&sp->mtx); 434 } else { 435 sp->state = SPLICE_QUEUED; 436 mtx_unlock(&sp->mtx); 437 so_splice_dispatch_async(sp); 438 } 439 } 440 441 static int 442 splice_zinit(void *mem, int size __unused, int flags __unused) 443 { 444 struct so_splice *s; 445 446 s = (struct so_splice *)mem; 447 mtx_init(&s->mtx, "so_splice", NULL, MTX_DEF); 448 return (0); 449 } 450 451 static void 452 splice_zfini(void *mem, int size) 453 { 454 struct so_splice *s; 455 456 s = (struct so_splice *)mem; 457 mtx_destroy(&s->mtx); 458 } 459 460 static int 461 splice_init(void) 462 { 463 struct thread *td; 464 int error, i, state; 465 466 state = atomic_load_acq_int(&splice_init_state); 467 if (__predict_true(state > 0)) 468 return (0); 469 if (state < 0) 470 return (ENXIO); 471 sx_xlock(&splice_init_lock); 472 if (splice_init_state != 0) { 473 sx_xunlock(&splice_init_lock); 474 return (0); 475 } 476 477 if (splice_num_wq == -1) { 478 /* if no user preference, use all cores */ 479 splice_num_wq = mp_ncpus; 480 } else if (splice_num_wq == 0) { 481 /* allow user to disable */ 482 splice_init_state = -1; 483 sx_xunlock(&splice_init_lock); 484 return (ENXIO); 485 } else if (splice_num_wq > mp_ncpus) { 486 splice_num_wq = mp_ncpus; 487 } 488 489 splice_zone = uma_zcreate("splice", sizeof(struct so_splice), NULL, 490 NULL, splice_zinit, splice_zfini, UMA_ALIGN_CACHE, 0); 491 492 splice_wq = mallocarray(mp_ncpus, sizeof(*splice_wq), M_TEMP, 493 M_WAITOK | M_ZERO); 494 495 /* 496 * Initialize the workqueues to run the splice work. We create a 497 * work queue for each CPU. 498 */ 499 for (i = 0; i < mp_ncpus; i++) { 500 STAILQ_INIT(&splice_wq[i].head); 501 mtx_init(&splice_wq[i].mtx, "splice work queue", NULL, MTX_DEF); 502 } 503 504 /* Start kthreads for each workqueue. */ 505 error = 0; 506 for (i = 0; i < mp_ncpus; i++) { 507 error = kproc_kthread_add(splice_work_thread, &splice_wq[i], 508 &splice_proc, &td, 0, 0, "so_splice", "thr_%d", i); 509 if (error) { 510 printf("Can't add so_splice thread %d error %d\n", 511 i, error); 512 break; 513 } 514 515 /* 516 * It's possible to create loops with SO_SPLICE; ensure that 517 * worker threads aren't able to starve the system too easily. 518 */ 519 thread_lock(td); 520 sched_prio(td, PUSER); 521 thread_unlock(td); 522 } 523 524 splice_init_state = error != 0 ? -1 : 1; 525 sx_xunlock(&splice_init_lock); 526 527 return (error); 528 } 529 530 /* 531 * Lock a pair of socket's I/O locks for splicing. Avoid blocking while holding 532 * one lock in order to avoid potential deadlocks in case there is some other 533 * code path which acquires more than one I/O lock at a time. 534 */ 535 static void 536 splice_lock_pair(struct socket *so_src, struct socket *so_dst) 537 { 538 int error; 539 540 for (;;) { 541 error = SOCK_IO_SEND_LOCK(so_dst, SBL_WAIT | SBL_NOINTR); 542 KASSERT(error == 0, 543 ("%s: failed to lock send I/O lock: %d", __func__, error)); 544 error = SOCK_IO_RECV_LOCK(so_src, 0); 545 KASSERT(error == 0 || error == EWOULDBLOCK, 546 ("%s: failed to lock recv I/O lock: %d", __func__, error)); 547 if (error == 0) 548 break; 549 SOCK_IO_SEND_UNLOCK(so_dst); 550 551 error = SOCK_IO_RECV_LOCK(so_src, SBL_WAIT | SBL_NOINTR); 552 KASSERT(error == 0, 553 ("%s: failed to lock recv I/O lock: %d", __func__, error)); 554 error = SOCK_IO_SEND_LOCK(so_dst, 0); 555 KASSERT(error == 0 || error == EWOULDBLOCK, 556 ("%s: failed to lock send I/O lock: %d", __func__, error)); 557 if (error == 0) 558 break; 559 SOCK_IO_RECV_UNLOCK(so_src); 560 } 561 } 562 563 static void 564 splice_unlock_pair(struct socket *so_src, struct socket *so_dst) 565 { 566 SOCK_IO_RECV_UNLOCK(so_src); 567 SOCK_IO_SEND_UNLOCK(so_dst); 568 } 569 570 /* 571 * Move data from the source to the sink. Assumes that both of the relevant 572 * socket I/O locks are held. 573 */ 574 static int 575 so_splice_xfer_data(struct socket *so_src, struct socket *so_dst, off_t max, 576 ssize_t *lenp) 577 { 578 struct uio uio; 579 struct mbuf *m; 580 struct sockbuf *sb_src, *sb_dst; 581 ssize_t len; 582 long space; 583 int error, flags; 584 585 SOCK_IO_RECV_ASSERT_LOCKED(so_src); 586 SOCK_IO_SEND_ASSERT_LOCKED(so_dst); 587 588 error = 0; 589 m = NULL; 590 memset(&uio, 0, sizeof(uio)); 591 592 sb_src = &so_src->so_rcv; 593 sb_dst = &so_dst->so_snd; 594 595 space = sbspace(sb_dst); 596 if (space < 0) 597 space = 0; 598 len = MIN(max, MIN(space, sbavail(sb_src))); 599 if (len == 0) { 600 SOCK_RECVBUF_LOCK(so_src); 601 if ((sb_src->sb_state & SBS_CANTRCVMORE) != 0) 602 error = EPIPE; 603 SOCK_RECVBUF_UNLOCK(so_src); 604 } else { 605 flags = MSG_DONTWAIT; 606 uio.uio_resid = len; 607 if (splice_receive_stream && sb_src->sb_tls_info == NULL) { 608 error = soreceive_stream_locked(so_src, sb_src, NULL, 609 &uio, &m, NULL, flags); 610 } else { 611 error = soreceive_generic_locked(so_src, NULL, 612 &uio, &m, NULL, &flags); 613 } 614 if (error != 0 && m != NULL) { 615 m_freem(m); 616 m = NULL; 617 } 618 } 619 if (m != NULL) { 620 len -= uio.uio_resid; 621 error = sosend_generic_locked(so_dst, NULL, NULL, m, NULL, 622 MSG_DONTWAIT, curthread); 623 } else if (error == 0) { 624 len = 0; 625 SOCK_SENDBUF_LOCK(so_dst); 626 if ((sb_dst->sb_state & SBS_CANTSENDMORE) != 0) 627 error = EPIPE; 628 SOCK_SENDBUF_UNLOCK(so_dst); 629 } 630 if (error == 0) 631 *lenp = len; 632 return (error); 633 } 634 635 /* 636 * Transfer data from the source to the sink. 637 */ 638 static void 639 so_splice_xfer(struct so_splice *sp) 640 { 641 struct socket *so_src, *so_dst; 642 off_t max; 643 ssize_t len; 644 int error; 645 646 mtx_assert(&sp->mtx, MA_OWNED); 647 KASSERT(sp->state == SPLICE_QUEUED || sp->state == SPLICE_CLOSING, 648 ("so_splice_xfer: invalid state %d", sp->state)); 649 KASSERT(sp->max != 0, ("so_splice_xfer: max == 0")); 650 651 if (sp->state == SPLICE_CLOSING) { 652 /* Userspace asked us to close the splice. */ 653 goto closing; 654 } 655 656 sp->state = SPLICE_RUNNING; 657 so_src = sp->src; 658 so_dst = sp->dst; 659 max = sp->max > 0 ? sp->max - so_src->so_splice_sent : OFF_MAX; 660 if (max < 0) 661 max = 0; 662 663 /* 664 * Lock the sockets in order to block userspace from doing anything 665 * sneaky. If an error occurs or one of the sockets can no longer 666 * transfer data, we will automatically unsplice. 667 */ 668 mtx_unlock(&sp->mtx); 669 splice_lock_pair(so_src, so_dst); 670 671 error = so_splice_xfer_data(so_src, so_dst, max, &len); 672 673 mtx_lock(&sp->mtx); 674 675 /* 676 * Update our stats while still holding the socket locks. This 677 * synchronizes with getsockopt(SO_SPLICE), see the comment there. 678 */ 679 if (error == 0) { 680 KASSERT(len >= 0, ("%s: len %zd < 0", __func__, len)); 681 so_src->so_splice_sent += len; 682 } 683 splice_unlock_pair(so_src, so_dst); 684 685 switch (sp->state) { 686 case SPLICE_CLOSING: 687 closing: 688 sp->state = SPLICE_CLOSED; 689 wakeup(sp); 690 mtx_unlock(&sp->mtx); 691 break; 692 case SPLICE_RUNNING: 693 if (error != 0 || 694 (sp->max > 0 && so_src->so_splice_sent >= sp->max)) { 695 sp->state = SPLICE_EXCEPTION; 696 soref(so_src); 697 mtx_unlock(&sp->mtx); 698 (void)so_unsplice(so_src, false); 699 sorele(so_src); 700 } else { 701 /* 702 * Locklessly check for additional bytes in the source's 703 * receive buffer and queue more work if possible. We 704 * may end up queuing needless work, but that's ok, and 705 * if we race with a thread inserting more data into the 706 * buffer and observe sbavail() == 0, the splice mutex 707 * ensures that splice_push() will queue more work for 708 * us. 709 */ 710 if (sbavail(&so_src->so_rcv) > 0 && 711 sbspace(&so_dst->so_snd) > 0) { 712 sp->state = SPLICE_QUEUED; 713 mtx_unlock(&sp->mtx); 714 so_splice_dispatch_async(sp); 715 } else { 716 sp->state = SPLICE_IDLE; 717 mtx_unlock(&sp->mtx); 718 } 719 } 720 break; 721 default: 722 __assert_unreachable(); 723 } 724 } 725 726 static void 727 socket_init(void *tag) 728 { 729 730 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL, 731 NULL, NULL, UMA_ALIGN_PTR, 0); 732 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 733 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached"); 734 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL, 735 EVENTHANDLER_PRI_FIRST); 736 } 737 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL); 738 739 #ifdef SOCKET_HHOOK 740 static void 741 socket_hhook_register(int subtype) 742 { 743 744 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype, 745 &V_socket_hhh[subtype], 746 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0) 747 printf("%s: WARNING: unable to register hook\n", __func__); 748 } 749 750 static void 751 socket_hhook_deregister(int subtype) 752 { 753 754 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0) 755 printf("%s: WARNING: unable to deregister hook\n", __func__); 756 } 757 758 static void 759 socket_vnet_init(const void *unused __unused) 760 { 761 int i; 762 763 /* We expect a contiguous range */ 764 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 765 socket_hhook_register(i); 766 } 767 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 768 socket_vnet_init, NULL); 769 770 static void 771 socket_vnet_uninit(const void *unused __unused) 772 { 773 int i; 774 775 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 776 socket_hhook_deregister(i); 777 } 778 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 779 socket_vnet_uninit, NULL); 780 #endif /* SOCKET_HHOOK */ 781 782 /* 783 * Initialise maxsockets. This SYSINIT must be run after 784 * tunable_mbinit(). 785 */ 786 static void 787 init_maxsockets(void *ignored) 788 { 789 790 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 791 maxsockets = imax(maxsockets, maxfiles); 792 } 793 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 794 795 /* 796 * Sysctl to get and set the maximum global sockets limit. Notify protocols 797 * of the change so that they can update their dependent limits as required. 798 */ 799 static int 800 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 801 { 802 int error, newmaxsockets; 803 804 newmaxsockets = maxsockets; 805 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req); 806 if (error == 0 && req->newptr && newmaxsockets != maxsockets) { 807 if (newmaxsockets > maxsockets && 808 newmaxsockets <= maxfiles) { 809 maxsockets = newmaxsockets; 810 EVENTHANDLER_INVOKE(maxsockets_change); 811 } else 812 error = EINVAL; 813 } 814 return (error); 815 } 816 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, 817 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, 818 &maxsockets, 0, sysctl_maxsockets, "IU", 819 "Maximum number of sockets available"); 820 821 /* 822 * Socket operation routines. These routines are called by the routines in 823 * sys_socket.c or from a system process, and implement the semantics of 824 * socket operations by switching out to the protocol specific routines. 825 */ 826 827 /* 828 * Get a socket structure from our zone, and initialize it. Note that it 829 * would probably be better to allocate socket and PCB at the same time, but 830 * I'm not convinced that all the protocols can be easily modified to do 831 * this. 832 * 833 * soalloc() returns a socket with a ref count of 0. 834 */ 835 static struct socket * 836 soalloc(struct vnet *vnet) 837 { 838 struct socket *so; 839 840 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); 841 if (so == NULL) 842 return (NULL); 843 #ifdef MAC 844 if (mac_socket_init(so, M_NOWAIT) != 0) { 845 uma_zfree(socket_zone, so); 846 return (NULL); 847 } 848 #endif 849 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { 850 uma_zfree(socket_zone, so); 851 return (NULL); 852 } 853 854 /* 855 * The socket locking protocol allows to lock 2 sockets at a time, 856 * however, the first one must be a listening socket. WITNESS lacks 857 * a feature to change class of an existing lock, so we use DUPOK. 858 */ 859 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); 860 so->so_rcv.sb_sel = &so->so_rdsel; 861 so->so_snd.sb_sel = &so->so_wrsel; 862 sx_init(&so->so_snd_sx, "so_snd_sx"); 863 sx_init(&so->so_rcv_sx, "so_rcv_sx"); 864 TAILQ_INIT(&so->so_snd.sb_aiojobq); 865 TAILQ_INIT(&so->so_rcv.sb_aiojobq); 866 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); 867 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); 868 #ifdef VIMAGE 869 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", 870 __func__, __LINE__, so)); 871 so->so_vnet = vnet; 872 #endif 873 #ifdef SOCKET_HHOOK 874 /* We shouldn't need the so_global_mtx */ 875 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { 876 /* Do we need more comprehensive error returns? */ 877 uma_zfree(socket_zone, so); 878 return (NULL); 879 } 880 #endif 881 mtx_lock(&so_global_mtx); 882 so->so_gencnt = ++so_gencnt; 883 ++numopensockets; 884 #ifdef VIMAGE 885 vnet->vnet_sockcnt++; 886 #endif 887 mtx_unlock(&so_global_mtx); 888 889 return (so); 890 } 891 892 /* 893 * Free the storage associated with a socket at the socket layer, tear down 894 * locks, labels, etc. All protocol state is assumed already to have been 895 * torn down (and possibly never set up) by the caller. 896 */ 897 void 898 sodealloc(struct socket *so) 899 { 900 901 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 902 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 903 904 mtx_lock(&so_global_mtx); 905 so->so_gencnt = ++so_gencnt; 906 --numopensockets; /* Could be below, but faster here. */ 907 #ifdef VIMAGE 908 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", 909 __func__, __LINE__, so)); 910 so->so_vnet->vnet_sockcnt--; 911 #endif 912 mtx_unlock(&so_global_mtx); 913 #ifdef MAC 914 mac_socket_destroy(so); 915 #endif 916 #ifdef SOCKET_HHOOK 917 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); 918 #endif 919 920 khelp_destroy_osd(&so->osd); 921 if (SOLISTENING(so)) { 922 if (so->sol_accept_filter != NULL) 923 accept_filt_setopt(so, NULL); 924 } else { 925 if (so->so_rcv.sb_hiwat) 926 (void)chgsbsize(so->so_cred->cr_uidinfo, 927 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 928 if (so->so_snd.sb_hiwat) 929 (void)chgsbsize(so->so_cred->cr_uidinfo, 930 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 931 sx_destroy(&so->so_snd_sx); 932 sx_destroy(&so->so_rcv_sx); 933 } 934 crfree(so->so_cred); 935 mtx_destroy(&so->so_lock); 936 uma_zfree(socket_zone, so); 937 } 938 939 /* 940 * Shim to accomodate protocols that already do their own socket buffers 941 * management (marked with PR_SOCKBUF) with protocols that yet do not. 942 * 943 * Attach via socket(2) is different from attach via accept(2). In case of 944 * normal socket(2) syscall it is the pr_attach that calls soreserve(), even 945 * for protocols that don't yet do PR_SOCKBUF. In case of accepted connection 946 * it is our shim that calls soreserve() and the hiwat values are taken from 947 * the parent socket. The SCTP's sopeeloff() hands us a non-listening parent 948 * socket. 949 * 950 * This whole shim should go away when all major protocols fully manage their 951 * socket buffers. 952 */ 953 static int 954 soattach(struct socket *so, int proto, struct thread *td, struct socket *head) 955 { 956 int error; 957 958 VNET_ASSERT(curvnet == so->so_vnet, 959 ("%s: %p != %p", __func__, curvnet, so->so_vnet)); 960 961 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { 962 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); 963 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); 964 so->so_snd.sb_mtx = &so->so_snd_mtx; 965 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 966 } 967 if (head == NULL || (error = soreserve(so, 968 SOLISTENING(head) ? head->sol_sbsnd_hiwat : head->so_snd.sb_hiwat, 969 SOLISTENING(head) ? head->sol_sbrcv_hiwat : head->so_rcv.sb_hiwat)) 970 == 0) 971 error = so->so_proto->pr_attach(so, proto, td); 972 if (error != 0 && (so->so_proto->pr_flags & PR_SOCKBUF) == 0) { 973 mtx_destroy(&so->so_snd_mtx); 974 mtx_destroy(&so->so_rcv_mtx); 975 } 976 977 return (error); 978 } 979 980 /* 981 * socreate returns a socket with a ref count of 1 and a file descriptor 982 * reference. The socket should be closed with soclose(). 983 */ 984 int 985 socreate(int dom, struct socket **aso, int type, int proto, 986 struct ucred *cred, struct thread *td) 987 { 988 struct protosw *prp; 989 struct socket *so; 990 int error; 991 992 prp = pffindproto(dom, type, proto); 993 if (prp == NULL) { 994 /* No support for domain. */ 995 if (pffinddomain(dom) == NULL) 996 return (EAFNOSUPPORT); 997 /* No support for socket type. */ 998 if (proto == 0 && type != 0) 999 return (EPROTOTYPE); 1000 return (EPROTONOSUPPORT); 1001 } 1002 1003 MPASS(prp->pr_attach); 1004 1005 if ((prp->pr_flags & PR_CAPATTACH) == 0) { 1006 if (CAP_TRACING(td)) 1007 ktrcapfail(CAPFAIL_PROTO, &proto); 1008 if (IN_CAPABILITY_MODE(td)) 1009 return (ECAPMODE); 1010 } 1011 1012 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0) 1013 return (EPROTONOSUPPORT); 1014 1015 so = soalloc(CRED_TO_VNET(cred)); 1016 if (so == NULL) 1017 return (ENOBUFS); 1018 1019 so->so_type = type; 1020 so->so_cred = crhold(cred); 1021 if ((prp->pr_domain->dom_family == PF_INET) || 1022 (prp->pr_domain->dom_family == PF_INET6) || 1023 (prp->pr_domain->dom_family == PF_ROUTE)) 1024 so->so_fibnum = td->td_proc->p_fibnum; 1025 else 1026 so->so_fibnum = 0; 1027 so->so_proto = prp; 1028 #ifdef MAC 1029 mac_socket_create(cred, so); 1030 #endif 1031 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1032 so_rdknl_assert_lock); 1033 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1034 so_wrknl_assert_lock); 1035 CURVNET_SET(so->so_vnet); 1036 error = soattach(so, proto, td, NULL); 1037 CURVNET_RESTORE(); 1038 if (error) { 1039 sodealloc(so); 1040 return (error); 1041 } 1042 soref(so); 1043 *aso = so; 1044 return (0); 1045 } 1046 1047 #ifdef REGRESSION 1048 static int regression_sonewconn_earlytest = 1; 1049 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 1050 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 1051 #endif 1052 1053 static int sooverprio = LOG_DEBUG; 1054 SYSCTL_INT(_kern_ipc, OID_AUTO, sooverprio, CTLFLAG_RW, 1055 &sooverprio, 0, "Log priority for listen socket overflows: 0..7 or -1 to disable"); 1056 1057 static struct timeval overinterval = { 60, 0 }; 1058 SYSCTL_TIMEVAL_SEC(_kern_ipc, OID_AUTO, sooverinterval, CTLFLAG_RW, 1059 &overinterval, 1060 "Delay in seconds between warnings for listen socket overflows"); 1061 1062 /* 1063 * When an attempt at a new connection is noted on a socket which supports 1064 * accept(2), the protocol has two options: 1065 * 1) Call legacy sonewconn() function, which would call protocol attach 1066 * method, same as used for socket(2). 1067 * 2) Call solisten_clone(), do attach that is specific to a cloned connection, 1068 * and then call solisten_enqueue(). 1069 * 1070 * Note: the ref count on the socket is 0 on return. 1071 */ 1072 struct socket * 1073 solisten_clone(struct socket *head) 1074 { 1075 struct sbuf descrsb; 1076 struct socket *so; 1077 int len, overcount; 1078 u_int qlen; 1079 const char localprefix[] = "local:"; 1080 char descrbuf[SUNPATHLEN + sizeof(localprefix)]; 1081 #if defined(INET6) 1082 char addrbuf[INET6_ADDRSTRLEN]; 1083 #elif defined(INET) 1084 char addrbuf[INET_ADDRSTRLEN]; 1085 #endif 1086 bool dolog, over; 1087 1088 SOLISTEN_LOCK(head); 1089 over = (head->sol_qlen > 3 * head->sol_qlimit / 2); 1090 #ifdef REGRESSION 1091 if (regression_sonewconn_earlytest && over) { 1092 #else 1093 if (over) { 1094 #endif 1095 head->sol_overcount++; 1096 dolog = (sooverprio >= 0) && 1097 !!ratecheck(&head->sol_lastover, &overinterval); 1098 1099 /* 1100 * If we're going to log, copy the overflow count and queue 1101 * length from the listen socket before dropping the lock. 1102 * Also, reset the overflow count. 1103 */ 1104 if (dolog) { 1105 overcount = head->sol_overcount; 1106 head->sol_overcount = 0; 1107 qlen = head->sol_qlen; 1108 } 1109 SOLISTEN_UNLOCK(head); 1110 1111 if (dolog) { 1112 /* 1113 * Try to print something descriptive about the 1114 * socket for the error message. 1115 */ 1116 sbuf_new(&descrsb, descrbuf, sizeof(descrbuf), 1117 SBUF_FIXEDLEN); 1118 switch (head->so_proto->pr_domain->dom_family) { 1119 #if defined(INET) || defined(INET6) 1120 #ifdef INET 1121 case AF_INET: 1122 #endif 1123 #ifdef INET6 1124 case AF_INET6: 1125 if (head->so_proto->pr_domain->dom_family == 1126 AF_INET6 || 1127 (sotoinpcb(head)->inp_inc.inc_flags & 1128 INC_ISIPV6)) { 1129 ip6_sprintf(addrbuf, 1130 &sotoinpcb(head)->inp_inc.inc6_laddr); 1131 sbuf_printf(&descrsb, "[%s]", addrbuf); 1132 } else 1133 #endif 1134 { 1135 #ifdef INET 1136 inet_ntoa_r( 1137 sotoinpcb(head)->inp_inc.inc_laddr, 1138 addrbuf); 1139 sbuf_cat(&descrsb, addrbuf); 1140 #endif 1141 } 1142 sbuf_printf(&descrsb, ":%hu (proto %u)", 1143 ntohs(sotoinpcb(head)->inp_inc.inc_lport), 1144 head->so_proto->pr_protocol); 1145 break; 1146 #endif /* INET || INET6 */ 1147 case AF_UNIX: 1148 sbuf_cat(&descrsb, localprefix); 1149 if (sotounpcb(head)->unp_addr != NULL) 1150 len = 1151 sotounpcb(head)->unp_addr->sun_len - 1152 offsetof(struct sockaddr_un, 1153 sun_path); 1154 else 1155 len = 0; 1156 if (len > 0) 1157 sbuf_bcat(&descrsb, 1158 sotounpcb(head)->unp_addr->sun_path, 1159 len); 1160 else 1161 sbuf_cat(&descrsb, "(unknown)"); 1162 break; 1163 } 1164 1165 /* 1166 * If we can't print something more specific, at least 1167 * print the domain name. 1168 */ 1169 if (sbuf_finish(&descrsb) != 0 || 1170 sbuf_len(&descrsb) <= 0) { 1171 sbuf_clear(&descrsb); 1172 sbuf_cat(&descrsb, 1173 head->so_proto->pr_domain->dom_name ?: 1174 "unknown"); 1175 sbuf_finish(&descrsb); 1176 } 1177 KASSERT(sbuf_len(&descrsb) > 0, 1178 ("%s: sbuf creation failed", __func__)); 1179 /* 1180 * Preserve the historic listen queue overflow log 1181 * message, that starts with "sonewconn:". It has 1182 * been known to sysadmins for years and also test 1183 * sys/kern/sonewconn_overflow checks for it. 1184 */ 1185 if (head->so_cred == 0) { 1186 log(LOG_PRI(sooverprio), 1187 "sonewconn: pcb %p (%s): " 1188 "Listen queue overflow: %i already in " 1189 "queue awaiting acceptance (%d " 1190 "occurrences)\n", head->so_pcb, 1191 sbuf_data(&descrsb), 1192 qlen, overcount); 1193 } else { 1194 log(LOG_PRI(sooverprio), 1195 "sonewconn: pcb %p (%s): " 1196 "Listen queue overflow: " 1197 "%i already in queue awaiting acceptance " 1198 "(%d occurrences), euid %d, rgid %d, jail %s\n", 1199 head->so_pcb, sbuf_data(&descrsb), qlen, 1200 overcount, head->so_cred->cr_uid, 1201 head->so_cred->cr_rgid, 1202 head->so_cred->cr_prison ? 1203 head->so_cred->cr_prison->pr_name : 1204 "not_jailed"); 1205 } 1206 sbuf_delete(&descrsb); 1207 1208 overcount = 0; 1209 } 1210 1211 return (NULL); 1212 } 1213 SOLISTEN_UNLOCK(head); 1214 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL", 1215 __func__, head)); 1216 so = soalloc(head->so_vnet); 1217 if (so == NULL) { 1218 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 1219 "limit reached or out of memory\n", 1220 __func__, head->so_pcb); 1221 return (NULL); 1222 } 1223 so->so_listen = head; 1224 so->so_type = head->so_type; 1225 /* 1226 * POSIX is ambiguous on what options an accept(2)ed socket should 1227 * inherit from the listener. Words "create a new socket" may be 1228 * interpreted as not inheriting anything. Best programming practice 1229 * for application developers is to not rely on such inheritance. 1230 * FreeBSD had historically inherited all so_options excluding 1231 * SO_ACCEPTCONN, which virtually means all SOL_SOCKET level options, 1232 * including those completely irrelevant to a new born socket. For 1233 * compatibility with older versions we will inherit a list of 1234 * meaningful options. 1235 * The crucial bit to inherit is SO_ACCEPTFILTER. We need it present 1236 * in the child socket for soisconnected() promoting socket from the 1237 * incomplete queue to complete. It will be cleared before the child 1238 * gets available to accept(2). 1239 */ 1240 so->so_options = head->so_options & (SO_ACCEPTFILTER | SO_KEEPALIVE | 1241 SO_DONTROUTE | SO_LINGER | SO_OOBINLINE | SO_NOSIGPIPE); 1242 so->so_linger = head->so_linger; 1243 so->so_state = head->so_state; 1244 so->so_fibnum = head->so_fibnum; 1245 so->so_proto = head->so_proto; 1246 so->so_cred = crhold(head->so_cred); 1247 #ifdef SOCKET_HHOOK 1248 if (V_socket_hhh[HHOOK_SOCKET_NEWCONN]->hhh_nhooks > 0) { 1249 if (hhook_run_socket(so, head, HHOOK_SOCKET_NEWCONN)) { 1250 sodealloc(so); 1251 log(LOG_DEBUG, "%s: hhook run failed\n", __func__); 1252 return (NULL); 1253 } 1254 } 1255 #endif 1256 #ifdef MAC 1257 mac_socket_newconn(head, so); 1258 #endif 1259 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1260 so_rdknl_assert_lock); 1261 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1262 so_wrknl_assert_lock); 1263 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; 1264 so->so_snd.sb_lowat = head->sol_sbsnd_lowat; 1265 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; 1266 so->so_snd.sb_timeo = head->sol_sbsnd_timeo; 1267 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE; 1268 so->so_snd.sb_flags = head->sol_sbsnd_flags & 1269 (SB_AUTOSIZE | SB_AUTOLOWAT); 1270 1271 return (so); 1272 } 1273 1274 /* Connstatus may be 0 or SS_ISCONNECTED. */ 1275 struct socket * 1276 sonewconn(struct socket *head, int connstatus) 1277 { 1278 struct socket *so; 1279 1280 if ((so = solisten_clone(head)) == NULL) 1281 return (NULL); 1282 1283 if (soattach(so, 0, NULL, head) != 0) { 1284 sodealloc(so); 1285 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", 1286 __func__, head->so_pcb); 1287 return (NULL); 1288 } 1289 1290 (void)solisten_enqueue(so, connstatus); 1291 1292 return (so); 1293 } 1294 1295 /* 1296 * Enqueue socket cloned by solisten_clone() to the listen queue of the 1297 * listener it has been cloned from. 1298 * 1299 * Return 'true' if socket landed on complete queue, otherwise 'false'. 1300 */ 1301 bool 1302 solisten_enqueue(struct socket *so, int connstatus) 1303 { 1304 struct socket *head = so->so_listen; 1305 1306 MPASS(refcount_load(&so->so_count) == 0); 1307 refcount_init(&so->so_count, 1); 1308 1309 SOLISTEN_LOCK(head); 1310 if (head->sol_accept_filter != NULL) 1311 connstatus = 0; 1312 so->so_state |= connstatus; 1313 soref(head); /* A socket on (in)complete queue refs head. */ 1314 if (connstatus) { 1315 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 1316 so->so_qstate = SQ_COMP; 1317 head->sol_qlen++; 1318 solisten_wakeup(head); /* unlocks */ 1319 return (true); 1320 } else { 1321 /* 1322 * Keep removing sockets from the head until there's room for 1323 * us to insert on the tail. In pre-locking revisions, this 1324 * was a simple if(), but as we could be racing with other 1325 * threads and soabort() requires dropping locks, we must 1326 * loop waiting for the condition to be true. 1327 */ 1328 while (head->sol_incqlen > head->sol_qlimit) { 1329 struct socket *sp; 1330 1331 sp = TAILQ_FIRST(&head->sol_incomp); 1332 TAILQ_REMOVE(&head->sol_incomp, sp, so_list); 1333 head->sol_incqlen--; 1334 SOCK_LOCK(sp); 1335 sp->so_qstate = SQ_NONE; 1336 sp->so_listen = NULL; 1337 SOCK_UNLOCK(sp); 1338 sorele_locked(head); /* does SOLISTEN_UNLOCK, head stays */ 1339 soabort(sp); 1340 SOLISTEN_LOCK(head); 1341 } 1342 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list); 1343 so->so_qstate = SQ_INCOMP; 1344 head->sol_incqlen++; 1345 SOLISTEN_UNLOCK(head); 1346 return (false); 1347 } 1348 } 1349 1350 #if defined(SCTP) || defined(SCTP_SUPPORT) 1351 /* 1352 * Socket part of sctp_peeloff(). Create a new socket for an 1353 * association. The new socket is returned with a reference. 1354 * 1355 * XXXGL: reduce copy-paste with solisten_clone(). 1356 */ 1357 struct socket * 1358 sopeeloff(struct socket *head, struct protosw *so_proto) 1359 { 1360 struct socket *so; 1361 1362 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", 1363 __func__, __LINE__, head)); 1364 KASSERT(head->so_type == SOCK_SEQPACKET, 1365 ("%s: unexpecte so_type: %d", __func__, head->so_type)); 1366 so = soalloc(head->so_vnet); 1367 if (so == NULL) { 1368 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 1369 "limit reached or out of memory\n", 1370 __func__, head->so_pcb); 1371 return (NULL); 1372 } 1373 so->so_type = SOCK_STREAM; 1374 so->so_options = head->so_options; 1375 so->so_linger = head->so_linger; 1376 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; 1377 so->so_fibnum = head->so_fibnum; 1378 so->so_proto = so_proto; 1379 so->so_cred = crhold(head->so_cred); 1380 #ifdef MAC 1381 mac_socket_newconn(head, so); 1382 #endif 1383 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1384 so_rdknl_assert_lock); 1385 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1386 so_wrknl_assert_lock); 1387 if (soattach(so, 0, NULL, head)) { 1388 sodealloc(so); 1389 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", 1390 __func__, head->so_pcb); 1391 return (NULL); 1392 } 1393 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 1394 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 1395 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 1396 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 1397 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 1398 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 1399 1400 soref(so); 1401 1402 return (so); 1403 } 1404 #endif /* SCTP */ 1405 1406 int 1407 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) 1408 { 1409 int error; 1410 1411 CURVNET_SET(so->so_vnet); 1412 error = so->so_proto->pr_bind(so, nam, td); 1413 CURVNET_RESTORE(); 1414 return (error); 1415 } 1416 1417 int 1418 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 1419 { 1420 int error; 1421 1422 CURVNET_SET(so->so_vnet); 1423 error = so->so_proto->pr_bindat(fd, so, nam, td); 1424 CURVNET_RESTORE(); 1425 return (error); 1426 } 1427 1428 /* 1429 * solisten() transitions a socket from a non-listening state to a listening 1430 * state, but can also be used to update the listen queue depth on an 1431 * existing listen socket. The protocol will call back into the sockets 1432 * layer using solisten_proto_check() and solisten_proto() to check and set 1433 * socket-layer listen state. Call backs are used so that the protocol can 1434 * acquire both protocol and socket layer locks in whatever order is required 1435 * by the protocol. 1436 * 1437 * Protocol implementors are advised to hold the socket lock across the 1438 * socket-layer test and set to avoid races at the socket layer. 1439 */ 1440 int 1441 solisten(struct socket *so, int backlog, struct thread *td) 1442 { 1443 int error; 1444 1445 CURVNET_SET(so->so_vnet); 1446 error = so->so_proto->pr_listen(so, backlog, td); 1447 CURVNET_RESTORE(); 1448 return (error); 1449 } 1450 1451 /* 1452 * Prepare for a call to solisten_proto(). Acquire all socket buffer locks in 1453 * order to interlock with socket I/O. 1454 */ 1455 int 1456 solisten_proto_check(struct socket *so) 1457 { 1458 SOCK_LOCK_ASSERT(so); 1459 1460 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 1461 SS_ISDISCONNECTING)) != 0) 1462 return (EINVAL); 1463 1464 /* 1465 * Sleeping is not permitted here, so simply fail if userspace is 1466 * attempting to transmit or receive on the socket. This kind of 1467 * transient failure is not ideal, but it should occur only if userspace 1468 * is misusing the socket interfaces. 1469 */ 1470 if (!sx_try_xlock(&so->so_snd_sx)) 1471 return (EAGAIN); 1472 if (!sx_try_xlock(&so->so_rcv_sx)) { 1473 sx_xunlock(&so->so_snd_sx); 1474 return (EAGAIN); 1475 } 1476 mtx_lock(&so->so_snd_mtx); 1477 mtx_lock(&so->so_rcv_mtx); 1478 1479 /* Interlock with soo_aio_queue() and KTLS. */ 1480 if (!SOLISTENING(so)) { 1481 bool ktls; 1482 1483 #ifdef KERN_TLS 1484 ktls = so->so_snd.sb_tls_info != NULL || 1485 so->so_rcv.sb_tls_info != NULL; 1486 #else 1487 ktls = false; 1488 #endif 1489 if (ktls || 1490 (so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 || 1491 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0) { 1492 solisten_proto_abort(so); 1493 return (EINVAL); 1494 } 1495 } 1496 1497 return (0); 1498 } 1499 1500 /* 1501 * Undo the setup done by solisten_proto_check(). 1502 */ 1503 void 1504 solisten_proto_abort(struct socket *so) 1505 { 1506 mtx_unlock(&so->so_snd_mtx); 1507 mtx_unlock(&so->so_rcv_mtx); 1508 sx_xunlock(&so->so_snd_sx); 1509 sx_xunlock(&so->so_rcv_sx); 1510 } 1511 1512 void 1513 solisten_proto(struct socket *so, int backlog) 1514 { 1515 int sbrcv_lowat, sbsnd_lowat; 1516 u_int sbrcv_hiwat, sbsnd_hiwat; 1517 short sbrcv_flags, sbsnd_flags; 1518 sbintime_t sbrcv_timeo, sbsnd_timeo; 1519 1520 SOCK_LOCK_ASSERT(so); 1521 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 1522 SS_ISDISCONNECTING)) == 0, 1523 ("%s: bad socket state %p", __func__, so)); 1524 1525 if (SOLISTENING(so)) 1526 goto listening; 1527 1528 /* 1529 * Change this socket to listening state. 1530 */ 1531 sbrcv_lowat = so->so_rcv.sb_lowat; 1532 sbsnd_lowat = so->so_snd.sb_lowat; 1533 sbrcv_hiwat = so->so_rcv.sb_hiwat; 1534 sbsnd_hiwat = so->so_snd.sb_hiwat; 1535 sbrcv_flags = so->so_rcv.sb_flags; 1536 sbsnd_flags = so->so_snd.sb_flags; 1537 sbrcv_timeo = so->so_rcv.sb_timeo; 1538 sbsnd_timeo = so->so_snd.sb_timeo; 1539 1540 #ifdef MAC 1541 mac_socketpeer_label_free(so->so_peerlabel); 1542 #endif 1543 1544 if (!(so->so_proto->pr_flags & PR_SOCKBUF)) { 1545 sbdestroy(so, SO_SND); 1546 sbdestroy(so, SO_RCV); 1547 } 1548 1549 #ifdef INVARIANTS 1550 bzero(&so->so_rcv, 1551 sizeof(struct socket) - offsetof(struct socket, so_rcv)); 1552 #endif 1553 1554 so->sol_sbrcv_lowat = sbrcv_lowat; 1555 so->sol_sbsnd_lowat = sbsnd_lowat; 1556 so->sol_sbrcv_hiwat = sbrcv_hiwat; 1557 so->sol_sbsnd_hiwat = sbsnd_hiwat; 1558 so->sol_sbrcv_flags = sbrcv_flags; 1559 so->sol_sbsnd_flags = sbsnd_flags; 1560 so->sol_sbrcv_timeo = sbrcv_timeo; 1561 so->sol_sbsnd_timeo = sbsnd_timeo; 1562 1563 so->sol_qlen = so->sol_incqlen = 0; 1564 TAILQ_INIT(&so->sol_incomp); 1565 TAILQ_INIT(&so->sol_comp); 1566 1567 so->sol_accept_filter = NULL; 1568 so->sol_accept_filter_arg = NULL; 1569 so->sol_accept_filter_str = NULL; 1570 1571 so->sol_upcall = NULL; 1572 so->sol_upcallarg = NULL; 1573 1574 so->so_options |= SO_ACCEPTCONN; 1575 1576 listening: 1577 if (backlog < 0 || backlog > V_somaxconn) 1578 backlog = V_somaxconn; 1579 so->sol_qlimit = backlog; 1580 1581 mtx_unlock(&so->so_snd_mtx); 1582 mtx_unlock(&so->so_rcv_mtx); 1583 sx_xunlock(&so->so_snd_sx); 1584 sx_xunlock(&so->so_rcv_sx); 1585 } 1586 1587 /* 1588 * Wakeup listeners/subsystems once we have a complete connection. 1589 * Enters with lock, returns unlocked. 1590 */ 1591 void 1592 solisten_wakeup(struct socket *sol) 1593 { 1594 1595 if (sol->sol_upcall != NULL) 1596 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT); 1597 else { 1598 selwakeuppri(&sol->so_rdsel, PSOCK); 1599 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0); 1600 } 1601 SOLISTEN_UNLOCK(sol); 1602 wakeup_one(&sol->sol_comp); 1603 if ((sol->so_state & SS_ASYNC) && sol->so_sigio != NULL) 1604 pgsigio(&sol->so_sigio, SIGIO, 0); 1605 } 1606 1607 /* 1608 * Return single connection off a listening socket queue. Main consumer of 1609 * the function is kern_accept4(). Some modules, that do their own accept 1610 * management also use the function. The socket reference held by the 1611 * listen queue is handed to the caller. 1612 * 1613 * Listening socket must be locked on entry and is returned unlocked on 1614 * return. 1615 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT. 1616 */ 1617 int 1618 solisten_dequeue(struct socket *head, struct socket **ret, int flags) 1619 { 1620 struct socket *so; 1621 int error; 1622 1623 SOLISTEN_LOCK_ASSERT(head); 1624 1625 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) && 1626 head->so_error == 0) { 1627 error = msleep(&head->sol_comp, SOCK_MTX(head), PSOCK | PCATCH, 1628 "accept", 0); 1629 if (error != 0) { 1630 SOLISTEN_UNLOCK(head); 1631 return (error); 1632 } 1633 } 1634 if (head->so_error) { 1635 error = head->so_error; 1636 head->so_error = 0; 1637 } else if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp)) 1638 error = EWOULDBLOCK; 1639 else 1640 error = 0; 1641 if (error) { 1642 SOLISTEN_UNLOCK(head); 1643 return (error); 1644 } 1645 so = TAILQ_FIRST(&head->sol_comp); 1646 SOCK_LOCK(so); 1647 KASSERT(so->so_qstate == SQ_COMP, 1648 ("%s: so %p not SQ_COMP", __func__, so)); 1649 head->sol_qlen--; 1650 so->so_qstate = SQ_NONE; 1651 so->so_listen = NULL; 1652 TAILQ_REMOVE(&head->sol_comp, so, so_list); 1653 if (flags & ACCEPT4_INHERIT) 1654 so->so_state |= (head->so_state & SS_NBIO); 1655 else 1656 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 1657 SOCK_UNLOCK(so); 1658 sorele_locked(head); 1659 1660 *ret = so; 1661 return (0); 1662 } 1663 1664 static struct so_splice * 1665 so_splice_alloc(off_t max) 1666 { 1667 struct so_splice *sp; 1668 1669 sp = uma_zalloc(splice_zone, M_WAITOK); 1670 sp->src = NULL; 1671 sp->dst = NULL; 1672 sp->max = max > 0 ? max : -1; 1673 sp->wq_index = atomic_fetchadd_32(&splice_index, 1) % splice_num_wq; 1674 sp->state = SPLICE_INIT; 1675 TIMEOUT_TASK_INIT(taskqueue_thread, &sp->timeout, 0, so_splice_timeout, 1676 sp); 1677 return (sp); 1678 } 1679 1680 static void 1681 so_splice_free(struct so_splice *sp) 1682 { 1683 KASSERT(sp->state == SPLICE_CLOSED, 1684 ("so_splice_free: sp %p not closed", sp)); 1685 uma_zfree(splice_zone, sp); 1686 } 1687 1688 static void 1689 so_splice_timeout(void *arg, int pending __unused) 1690 { 1691 struct so_splice *sp; 1692 1693 sp = arg; 1694 (void)so_unsplice(sp->src, true); 1695 } 1696 1697 /* 1698 * Splice the output from so to the input of so2. 1699 */ 1700 static int 1701 so_splice(struct socket *so, struct socket *so2, struct splice *splice) 1702 { 1703 struct so_splice *sp; 1704 int error; 1705 1706 if (splice->sp_max < 0) 1707 return (EINVAL); 1708 /* Handle only TCP for now; TODO: other streaming protos */ 1709 if (so->so_proto->pr_protocol != IPPROTO_TCP || 1710 so2->so_proto->pr_protocol != IPPROTO_TCP) 1711 return (EPROTONOSUPPORT); 1712 if (so->so_vnet != so2->so_vnet) 1713 return (EINVAL); 1714 1715 /* so_splice_xfer() assumes that we're using these implementations. */ 1716 KASSERT(so->so_proto->pr_sosend == sosend_generic, 1717 ("so_splice: sosend not sosend_generic")); 1718 KASSERT(so2->so_proto->pr_soreceive == soreceive_generic || 1719 so2->so_proto->pr_soreceive == soreceive_stream, 1720 ("so_splice: soreceive not soreceive_generic/stream")); 1721 1722 sp = so_splice_alloc(splice->sp_max); 1723 so->so_splice_sent = 0; 1724 sp->src = so; 1725 sp->dst = so2; 1726 1727 error = 0; 1728 SOCK_LOCK(so); 1729 if (SOLISTENING(so)) 1730 error = EINVAL; 1731 else if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0) 1732 error = ENOTCONN; 1733 else if (so->so_splice != NULL) 1734 error = EBUSY; 1735 if (error != 0) { 1736 SOCK_UNLOCK(so); 1737 uma_zfree(splice_zone, sp); 1738 return (error); 1739 } 1740 SOCK_RECVBUF_LOCK(so); 1741 if (so->so_rcv.sb_tls_info != NULL) { 1742 SOCK_RECVBUF_UNLOCK(so); 1743 SOCK_UNLOCK(so); 1744 uma_zfree(splice_zone, sp); 1745 return (EINVAL); 1746 } 1747 so->so_rcv.sb_flags |= SB_SPLICED; 1748 so->so_splice = sp; 1749 soref(so); 1750 SOCK_RECVBUF_UNLOCK(so); 1751 SOCK_UNLOCK(so); 1752 1753 error = 0; 1754 SOCK_LOCK(so2); 1755 if (SOLISTENING(so2)) 1756 error = EINVAL; 1757 else if ((so2->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0) 1758 error = ENOTCONN; 1759 else if (so2->so_splice_back != NULL) 1760 error = EBUSY; 1761 if (error != 0) { 1762 SOCK_UNLOCK(so2); 1763 mtx_lock(&sp->mtx); 1764 sp->dst = NULL; 1765 sp->state = SPLICE_EXCEPTION; 1766 mtx_unlock(&sp->mtx); 1767 so_unsplice(so, false); 1768 return (error); 1769 } 1770 SOCK_SENDBUF_LOCK(so2); 1771 if (so2->so_snd.sb_tls_info != NULL) { 1772 SOCK_SENDBUF_UNLOCK(so2); 1773 SOCK_UNLOCK(so2); 1774 mtx_lock(&sp->mtx); 1775 sp->dst = NULL; 1776 sp->state = SPLICE_EXCEPTION; 1777 mtx_unlock(&sp->mtx); 1778 so_unsplice(so, false); 1779 return (EINVAL); 1780 } 1781 so2->so_snd.sb_flags |= SB_SPLICED; 1782 so2->so_splice_back = sp; 1783 soref(so2); 1784 mtx_lock(&sp->mtx); 1785 SOCK_SENDBUF_UNLOCK(so2); 1786 SOCK_UNLOCK(so2); 1787 1788 if (splice->sp_idle.tv_sec != 0 || splice->sp_idle.tv_usec != 0) { 1789 taskqueue_enqueue_timeout_sbt(taskqueue_thread, &sp->timeout, 1790 tvtosbt(splice->sp_idle), 0, C_PREL(4)); 1791 } 1792 1793 /* 1794 * Transfer any data already present in the socket buffer. 1795 */ 1796 KASSERT(sp->state == SPLICE_INIT, 1797 ("so_splice: splice %p state %d", sp, sp->state)); 1798 sp->state = SPLICE_QUEUED; 1799 so_splice_xfer(sp); 1800 return (0); 1801 } 1802 1803 static int 1804 so_unsplice(struct socket *so, bool timeout) 1805 { 1806 struct socket *so2; 1807 struct so_splice *sp; 1808 bool drain; 1809 1810 /* 1811 * First unset SB_SPLICED and hide the splice structure so that 1812 * wakeup routines will stop enqueuing work. This also ensures that 1813 * a only a single thread will proceed with the unsplice. 1814 */ 1815 SOCK_LOCK(so); 1816 if (SOLISTENING(so)) { 1817 SOCK_UNLOCK(so); 1818 return (EINVAL); 1819 } 1820 SOCK_RECVBUF_LOCK(so); 1821 if ((so->so_rcv.sb_flags & SB_SPLICED) == 0) { 1822 SOCK_RECVBUF_UNLOCK(so); 1823 SOCK_UNLOCK(so); 1824 return (ENOTCONN); 1825 } 1826 sp = so->so_splice; 1827 mtx_lock(&sp->mtx); 1828 if (sp->state == SPLICE_INIT) { 1829 /* 1830 * A splice is in the middle of being set up. 1831 */ 1832 mtx_unlock(&sp->mtx); 1833 SOCK_RECVBUF_UNLOCK(so); 1834 SOCK_UNLOCK(so); 1835 return (ENOTCONN); 1836 } 1837 mtx_unlock(&sp->mtx); 1838 so->so_rcv.sb_flags &= ~SB_SPLICED; 1839 so->so_splice = NULL; 1840 SOCK_RECVBUF_UNLOCK(so); 1841 SOCK_UNLOCK(so); 1842 1843 so2 = sp->dst; 1844 if (so2 != NULL) { 1845 SOCK_LOCK(so2); 1846 KASSERT(!SOLISTENING(so2), ("%s: so2 is listening", __func__)); 1847 SOCK_SENDBUF_LOCK(so2); 1848 KASSERT((so2->so_snd.sb_flags & SB_SPLICED) != 0, 1849 ("%s: so2 is not spliced", __func__)); 1850 KASSERT(so2->so_splice_back == sp, 1851 ("%s: so_splice_back != sp", __func__)); 1852 so2->so_snd.sb_flags &= ~SB_SPLICED; 1853 so2->so_splice_back = NULL; 1854 SOCK_SENDBUF_UNLOCK(so2); 1855 SOCK_UNLOCK(so2); 1856 } 1857 1858 /* 1859 * No new work is being enqueued. The worker thread might be 1860 * splicing data right now, in which case we want to wait for it to 1861 * finish before proceeding. 1862 */ 1863 mtx_lock(&sp->mtx); 1864 switch (sp->state) { 1865 case SPLICE_QUEUED: 1866 case SPLICE_RUNNING: 1867 sp->state = SPLICE_CLOSING; 1868 while (sp->state == SPLICE_CLOSING) 1869 msleep(sp, &sp->mtx, PSOCK, "unsplice", 0); 1870 break; 1871 case SPLICE_INIT: 1872 case SPLICE_IDLE: 1873 case SPLICE_EXCEPTION: 1874 sp->state = SPLICE_CLOSED; 1875 break; 1876 default: 1877 __assert_unreachable(); 1878 } 1879 if (!timeout) { 1880 drain = taskqueue_cancel_timeout(taskqueue_thread, &sp->timeout, 1881 NULL) != 0; 1882 } else { 1883 drain = false; 1884 } 1885 mtx_unlock(&sp->mtx); 1886 if (drain) 1887 taskqueue_drain_timeout(taskqueue_thread, &sp->timeout); 1888 1889 /* 1890 * Now we hold the sole reference to the splice structure. 1891 * Clean up: signal userspace and release socket references. 1892 */ 1893 sorwakeup(so); 1894 CURVNET_SET(so->so_vnet); 1895 sorele(so); 1896 if (so2 != NULL) { 1897 sowwakeup(so2); 1898 sorele(so2); 1899 } 1900 CURVNET_RESTORE(); 1901 so_splice_free(sp); 1902 return (0); 1903 } 1904 1905 /* 1906 * Free socket upon release of the very last reference. 1907 */ 1908 static void 1909 sofree(struct socket *so) 1910 { 1911 struct protosw *pr = so->so_proto; 1912 1913 SOCK_LOCK_ASSERT(so); 1914 KASSERT(refcount_load(&so->so_count) == 0, 1915 ("%s: so %p has references", __func__, so)); 1916 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE, 1917 ("%s: so %p is on listen queue", __func__, so)); 1918 KASSERT(SOLISTENING(so) || (so->so_rcv.sb_flags & SB_SPLICED) == 0, 1919 ("%s: so %p rcvbuf is spliced", __func__, so)); 1920 KASSERT(SOLISTENING(so) || (so->so_snd.sb_flags & SB_SPLICED) == 0, 1921 ("%s: so %p sndbuf is spliced", __func__, so)); 1922 KASSERT(so->so_splice == NULL && so->so_splice_back == NULL, 1923 ("%s: so %p has spliced data", __func__, so)); 1924 1925 SOCK_UNLOCK(so); 1926 1927 if (so->so_dtor != NULL) 1928 so->so_dtor(so); 1929 1930 VNET_SO_ASSERT(so); 1931 if (pr->pr_detach != NULL) 1932 pr->pr_detach(so); 1933 1934 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) { 1935 /* 1936 * From this point on, we assume that no other references to 1937 * this socket exist anywhere else in the stack. Therefore, 1938 * no locks need to be acquired or held. 1939 */ 1940 #ifdef INVARIANTS 1941 SOCK_SENDBUF_LOCK(so); 1942 SOCK_RECVBUF_LOCK(so); 1943 #endif 1944 sbdestroy(so, SO_SND); 1945 sbdestroy(so, SO_RCV); 1946 #ifdef INVARIANTS 1947 SOCK_SENDBUF_UNLOCK(so); 1948 SOCK_RECVBUF_UNLOCK(so); 1949 #endif 1950 mtx_destroy(&so->so_snd_mtx); 1951 mtx_destroy(&so->so_rcv_mtx); 1952 } 1953 seldrain(&so->so_rdsel); 1954 seldrain(&so->so_wrsel); 1955 knlist_destroy(&so->so_rdsel.si_note); 1956 knlist_destroy(&so->so_wrsel.si_note); 1957 sodealloc(so); 1958 } 1959 1960 /* 1961 * Release a reference on a socket while holding the socket lock. 1962 * Unlocks the socket lock before returning. 1963 */ 1964 void 1965 sorele_locked(struct socket *so) 1966 { 1967 SOCK_LOCK_ASSERT(so); 1968 if (refcount_release(&so->so_count)) 1969 sofree(so); 1970 else 1971 SOCK_UNLOCK(so); 1972 } 1973 1974 /* 1975 * Close a socket on last file table reference removal. Initiate disconnect 1976 * if connected. Free socket when disconnect complete. 1977 * 1978 * This function will sorele() the socket. Note that soclose() may be called 1979 * prior to the ref count reaching zero. The actual socket structure will 1980 * not be freed until the ref count reaches zero. 1981 */ 1982 int 1983 soclose(struct socket *so) 1984 { 1985 struct accept_queue lqueue; 1986 int error = 0; 1987 bool listening, last __diagused; 1988 1989 CURVNET_SET(so->so_vnet); 1990 funsetown(&so->so_sigio); 1991 if (so->so_state & SS_ISCONNECTED) { 1992 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 1993 error = sodisconnect(so); 1994 if (error) { 1995 if (error == ENOTCONN) 1996 error = 0; 1997 goto drop; 1998 } 1999 } 2000 2001 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) { 2002 if ((so->so_state & SS_ISDISCONNECTING) && 2003 (so->so_state & SS_NBIO)) 2004 goto drop; 2005 while (so->so_state & SS_ISCONNECTED) { 2006 error = tsleep(&so->so_timeo, 2007 PSOCK | PCATCH, "soclos", 2008 so->so_linger * hz); 2009 if (error) 2010 break; 2011 } 2012 } 2013 } 2014 2015 drop: 2016 if (so->so_proto->pr_close != NULL) 2017 so->so_proto->pr_close(so); 2018 2019 SOCK_LOCK(so); 2020 if ((listening = SOLISTENING(so))) { 2021 struct socket *sp; 2022 2023 TAILQ_INIT(&lqueue); 2024 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list); 2025 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list); 2026 2027 so->sol_qlen = so->sol_incqlen = 0; 2028 2029 TAILQ_FOREACH(sp, &lqueue, so_list) { 2030 SOCK_LOCK(sp); 2031 sp->so_qstate = SQ_NONE; 2032 sp->so_listen = NULL; 2033 SOCK_UNLOCK(sp); 2034 last = refcount_release(&so->so_count); 2035 KASSERT(!last, ("%s: released last reference for %p", 2036 __func__, so)); 2037 } 2038 } 2039 sorele_locked(so); 2040 if (listening) { 2041 struct socket *sp, *tsp; 2042 2043 TAILQ_FOREACH_SAFE(sp, &lqueue, so_list, tsp) 2044 soabort(sp); 2045 } 2046 CURVNET_RESTORE(); 2047 return (error); 2048 } 2049 2050 /* 2051 * soabort() is used to abruptly tear down a connection, such as when a 2052 * resource limit is reached (listen queue depth exceeded), or if a listen 2053 * socket is closed while there are sockets waiting to be accepted. 2054 * 2055 * This interface is tricky, because it is called on an unreferenced socket, 2056 * and must be called only by a thread that has actually removed the socket 2057 * from the listen queue it was on. Likely this thread holds the last 2058 * reference on the socket and soabort() will proceed with sofree(). But 2059 * it might be not the last, as the sockets on the listen queues are seen 2060 * from the protocol side. 2061 * 2062 * This interface will call into the protocol code, so must not be called 2063 * with any socket locks held. Protocols do call it while holding their own 2064 * recursible protocol mutexes, but this is something that should be subject 2065 * to review in the future. 2066 * 2067 * Usually socket should have a single reference left, but this is not a 2068 * requirement. In the past, when we have had named references for file 2069 * descriptor and protocol, we asserted that none of them are being held. 2070 */ 2071 void 2072 soabort(struct socket *so) 2073 { 2074 2075 VNET_SO_ASSERT(so); 2076 2077 if (so->so_proto->pr_abort != NULL) 2078 so->so_proto->pr_abort(so); 2079 SOCK_LOCK(so); 2080 sorele_locked(so); 2081 } 2082 2083 int 2084 soaccept(struct socket *so, struct sockaddr *sa) 2085 { 2086 #ifdef INVARIANTS 2087 u_char len = sa->sa_len; 2088 #endif 2089 int error; 2090 2091 CURVNET_SET(so->so_vnet); 2092 error = so->so_proto->pr_accept(so, sa); 2093 KASSERT(sa->sa_len <= len, 2094 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2095 CURVNET_RESTORE(); 2096 return (error); 2097 } 2098 2099 int 2100 sopeeraddr(struct socket *so, struct sockaddr *sa) 2101 { 2102 #ifdef INVARIANTS 2103 u_char len = sa->sa_len; 2104 #endif 2105 int error; 2106 2107 CURVNET_ASSERT_SET(); 2108 2109 error = so->so_proto->pr_peeraddr(so, sa); 2110 KASSERT(sa->sa_len <= len, 2111 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2112 2113 return (error); 2114 } 2115 2116 int 2117 sosockaddr(struct socket *so, struct sockaddr *sa) 2118 { 2119 #ifdef INVARIANTS 2120 u_char len = sa->sa_len; 2121 #endif 2122 int error; 2123 2124 CURVNET_SET(so->so_vnet); 2125 error = so->so_proto->pr_sockaddr(so, sa); 2126 KASSERT(sa->sa_len <= len, 2127 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2128 CURVNET_RESTORE(); 2129 2130 return (error); 2131 } 2132 2133 int 2134 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) 2135 { 2136 2137 return (soconnectat(AT_FDCWD, so, nam, td)); 2138 } 2139 2140 int 2141 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 2142 { 2143 int error; 2144 2145 CURVNET_SET(so->so_vnet); 2146 2147 /* 2148 * If protocol is connection-based, can only connect once. 2149 * Otherwise, if connected, try to disconnect first. This allows 2150 * user to disconnect by connecting to, e.g., a null address. 2151 * 2152 * Note, this check is racy and may need to be re-evaluated at the 2153 * protocol layer. 2154 */ 2155 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 2156 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 2157 (error = sodisconnect(so)))) { 2158 error = EISCONN; 2159 } else { 2160 /* 2161 * Prevent accumulated error from previous connection from 2162 * biting us. 2163 */ 2164 so->so_error = 0; 2165 if (fd == AT_FDCWD) { 2166 error = so->so_proto->pr_connect(so, nam, td); 2167 } else { 2168 error = so->so_proto->pr_connectat(fd, so, nam, td); 2169 } 2170 } 2171 CURVNET_RESTORE(); 2172 2173 return (error); 2174 } 2175 2176 int 2177 soconnect2(struct socket *so1, struct socket *so2) 2178 { 2179 int error; 2180 2181 CURVNET_SET(so1->so_vnet); 2182 error = so1->so_proto->pr_connect2(so1, so2); 2183 CURVNET_RESTORE(); 2184 return (error); 2185 } 2186 2187 int 2188 sodisconnect(struct socket *so) 2189 { 2190 int error; 2191 2192 if ((so->so_state & SS_ISCONNECTED) == 0) 2193 return (ENOTCONN); 2194 if (so->so_state & SS_ISDISCONNECTING) 2195 return (EALREADY); 2196 VNET_SO_ASSERT(so); 2197 error = so->so_proto->pr_disconnect(so); 2198 return (error); 2199 } 2200 2201 int 2202 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 2203 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2204 { 2205 long space; 2206 ssize_t resid; 2207 int clen = 0, error, dontroute; 2208 2209 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM")); 2210 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 2211 ("sosend_dgram: !PR_ATOMIC")); 2212 2213 if (uio != NULL) 2214 resid = uio->uio_resid; 2215 else 2216 resid = top->m_pkthdr.len; 2217 /* 2218 * In theory resid should be unsigned. However, space must be 2219 * signed, as it might be less than 0 if we over-committed, and we 2220 * must use a signed comparison of space and resid. On the other 2221 * hand, a negative resid causes us to loop sending 0-length 2222 * segments to the protocol. 2223 */ 2224 if (resid < 0) { 2225 error = EINVAL; 2226 goto out; 2227 } 2228 2229 dontroute = 2230 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 2231 if (td != NULL) 2232 td->td_ru.ru_msgsnd++; 2233 if (control != NULL) 2234 clen = control->m_len; 2235 2236 SOCKBUF_LOCK(&so->so_snd); 2237 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2238 SOCKBUF_UNLOCK(&so->so_snd); 2239 error = EPIPE; 2240 goto out; 2241 } 2242 if (so->so_error) { 2243 error = so->so_error; 2244 so->so_error = 0; 2245 SOCKBUF_UNLOCK(&so->so_snd); 2246 goto out; 2247 } 2248 if ((so->so_state & SS_ISCONNECTED) == 0) { 2249 /* 2250 * `sendto' and `sendmsg' is allowed on a connection-based 2251 * socket if it supports implied connect. Return ENOTCONN if 2252 * not connected and no address is supplied. 2253 */ 2254 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 2255 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 2256 if (!(resid == 0 && clen != 0)) { 2257 SOCKBUF_UNLOCK(&so->so_snd); 2258 error = ENOTCONN; 2259 goto out; 2260 } 2261 } else if (addr == NULL) { 2262 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2263 error = ENOTCONN; 2264 else 2265 error = EDESTADDRREQ; 2266 SOCKBUF_UNLOCK(&so->so_snd); 2267 goto out; 2268 } 2269 } 2270 2271 /* 2272 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 2273 * problem and need fixing. 2274 */ 2275 space = sbspace(&so->so_snd); 2276 if (flags & MSG_OOB) 2277 space += 1024; 2278 space -= clen; 2279 SOCKBUF_UNLOCK(&so->so_snd); 2280 if (resid > space) { 2281 error = EMSGSIZE; 2282 goto out; 2283 } 2284 if (uio == NULL) { 2285 resid = 0; 2286 if (flags & MSG_EOR) 2287 top->m_flags |= M_EOR; 2288 } else { 2289 /* 2290 * Copy the data from userland into a mbuf chain. 2291 * If no data is to be copied in, a single empty mbuf 2292 * is returned. 2293 */ 2294 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 2295 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 2296 if (top == NULL) { 2297 error = EFAULT; /* only possible error */ 2298 goto out; 2299 } 2300 space -= resid - uio->uio_resid; 2301 resid = uio->uio_resid; 2302 } 2303 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 2304 /* 2305 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 2306 * than with. 2307 */ 2308 if (dontroute) { 2309 SOCK_LOCK(so); 2310 so->so_options |= SO_DONTROUTE; 2311 SOCK_UNLOCK(so); 2312 } 2313 /* 2314 * XXX all the SBS_CANTSENDMORE checks previously done could be out 2315 * of date. We could have received a reset packet in an interrupt or 2316 * maybe we slept while doing page faults in uiomove() etc. We could 2317 * probably recheck again inside the locking protection here, but 2318 * there are probably other places that this also happens. We must 2319 * rethink this. 2320 */ 2321 VNET_SO_ASSERT(so); 2322 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB : 2323 /* 2324 * If the user set MSG_EOF, the protocol understands this flag and 2325 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 2326 */ 2327 ((flags & MSG_EOF) && 2328 (so->so_proto->pr_flags & PR_IMPLOPCL) && 2329 (resid <= 0)) ? 2330 PRUS_EOF : 2331 /* If there is more to send set PRUS_MORETOCOME */ 2332 (flags & MSG_MORETOCOME) || 2333 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 2334 top, addr, control, td); 2335 if (dontroute) { 2336 SOCK_LOCK(so); 2337 so->so_options &= ~SO_DONTROUTE; 2338 SOCK_UNLOCK(so); 2339 } 2340 clen = 0; 2341 control = NULL; 2342 top = NULL; 2343 out: 2344 if (top != NULL) 2345 m_freem(top); 2346 if (control != NULL) 2347 m_freem(control); 2348 return (error); 2349 } 2350 2351 /* 2352 * Send on a socket. If send must go all at once and message is larger than 2353 * send buffering, then hard error. Lock against other senders. If must go 2354 * all at once and not enough room now, then inform user that this would 2355 * block and do nothing. Otherwise, if nonblocking, send as much as 2356 * possible. The data to be sent is described by "uio" if nonzero, otherwise 2357 * by the mbuf chain "top" (which must be null if uio is not). Data provided 2358 * in mbuf chain must be small enough to send all at once. 2359 * 2360 * Returns nonzero on error, timeout or signal; callers must check for short 2361 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 2362 * on return. 2363 */ 2364 static int 2365 sosend_generic_locked(struct socket *so, struct sockaddr *addr, struct uio *uio, 2366 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2367 { 2368 long space; 2369 ssize_t resid; 2370 int clen = 0, error, dontroute; 2371 int atomic = sosendallatonce(so) || top; 2372 int pr_send_flag; 2373 #ifdef KERN_TLS 2374 struct ktls_session *tls; 2375 int tls_enq_cnt, tls_send_flag; 2376 uint8_t tls_rtype; 2377 2378 tls = NULL; 2379 tls_rtype = TLS_RLTYPE_APP; 2380 #endif 2381 2382 SOCK_IO_SEND_ASSERT_LOCKED(so); 2383 2384 if (uio != NULL) 2385 resid = uio->uio_resid; 2386 else if ((top->m_flags & M_PKTHDR) != 0) 2387 resid = top->m_pkthdr.len; 2388 else 2389 resid = m_length(top, NULL); 2390 /* 2391 * In theory resid should be unsigned. However, space must be 2392 * signed, as it might be less than 0 if we over-committed, and we 2393 * must use a signed comparison of space and resid. On the other 2394 * hand, a negative resid causes us to loop sending 0-length 2395 * segments to the protocol. 2396 * 2397 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 2398 * type sockets since that's an error. 2399 */ 2400 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 2401 error = EINVAL; 2402 goto out; 2403 } 2404 2405 dontroute = 2406 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 2407 (so->so_proto->pr_flags & PR_ATOMIC); 2408 if (td != NULL) 2409 td->td_ru.ru_msgsnd++; 2410 if (control != NULL) 2411 clen = control->m_len; 2412 2413 #ifdef KERN_TLS 2414 tls_send_flag = 0; 2415 tls = ktls_hold(so->so_snd.sb_tls_info); 2416 if (tls != NULL) { 2417 if (tls->mode == TCP_TLS_MODE_SW) 2418 tls_send_flag = PRUS_NOTREADY; 2419 2420 if (control != NULL) { 2421 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 2422 2423 if (clen >= sizeof(*cm) && 2424 cm->cmsg_type == TLS_SET_RECORD_TYPE) { 2425 tls_rtype = *((uint8_t *)CMSG_DATA(cm)); 2426 clen = 0; 2427 m_freem(control); 2428 control = NULL; 2429 atomic = 1; 2430 } 2431 } 2432 2433 if (resid == 0 && !ktls_permit_empty_frames(tls)) { 2434 error = EINVAL; 2435 goto out; 2436 } 2437 } 2438 #endif 2439 2440 restart: 2441 do { 2442 SOCKBUF_LOCK(&so->so_snd); 2443 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2444 SOCKBUF_UNLOCK(&so->so_snd); 2445 error = EPIPE; 2446 goto out; 2447 } 2448 if (so->so_error) { 2449 error = so->so_error; 2450 so->so_error = 0; 2451 SOCKBUF_UNLOCK(&so->so_snd); 2452 goto out; 2453 } 2454 if ((so->so_state & SS_ISCONNECTED) == 0) { 2455 /* 2456 * `sendto' and `sendmsg' is allowed on a connection- 2457 * based socket if it supports implied connect. 2458 * Return ENOTCONN if not connected and no address is 2459 * supplied. 2460 */ 2461 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 2462 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 2463 if (!(resid == 0 && clen != 0)) { 2464 SOCKBUF_UNLOCK(&so->so_snd); 2465 error = ENOTCONN; 2466 goto out; 2467 } 2468 } else if (addr == NULL) { 2469 SOCKBUF_UNLOCK(&so->so_snd); 2470 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2471 error = ENOTCONN; 2472 else 2473 error = EDESTADDRREQ; 2474 goto out; 2475 } 2476 } 2477 space = sbspace(&so->so_snd); 2478 if (flags & MSG_OOB) 2479 space += 1024; 2480 if ((atomic && resid > so->so_snd.sb_hiwat) || 2481 clen > so->so_snd.sb_hiwat) { 2482 SOCKBUF_UNLOCK(&so->so_snd); 2483 error = EMSGSIZE; 2484 goto out; 2485 } 2486 if (space < resid + clen && 2487 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 2488 if ((so->so_state & SS_NBIO) || 2489 (flags & (MSG_NBIO | MSG_DONTWAIT)) != 0) { 2490 SOCKBUF_UNLOCK(&so->so_snd); 2491 error = EWOULDBLOCK; 2492 goto out; 2493 } 2494 error = sbwait(so, SO_SND); 2495 SOCKBUF_UNLOCK(&so->so_snd); 2496 if (error) 2497 goto out; 2498 goto restart; 2499 } 2500 SOCKBUF_UNLOCK(&so->so_snd); 2501 space -= clen; 2502 do { 2503 if (uio == NULL) { 2504 resid = 0; 2505 if (flags & MSG_EOR) 2506 top->m_flags |= M_EOR; 2507 #ifdef KERN_TLS 2508 if (tls != NULL) { 2509 ktls_frame(top, tls, &tls_enq_cnt, 2510 tls_rtype); 2511 tls_rtype = TLS_RLTYPE_APP; 2512 } 2513 #endif 2514 } else { 2515 /* 2516 * Copy the data from userland into a mbuf 2517 * chain. If resid is 0, which can happen 2518 * only if we have control to send, then 2519 * a single empty mbuf is returned. This 2520 * is a workaround to prevent protocol send 2521 * methods to panic. 2522 */ 2523 #ifdef KERN_TLS 2524 if (tls != NULL) { 2525 top = m_uiotombuf(uio, M_WAITOK, space, 2526 tls->params.max_frame_len, 2527 M_EXTPG | 2528 ((flags & MSG_EOR) ? M_EOR : 0)); 2529 if (top != NULL) { 2530 ktls_frame(top, tls, 2531 &tls_enq_cnt, tls_rtype); 2532 } 2533 tls_rtype = TLS_RLTYPE_APP; 2534 } else 2535 #endif 2536 top = m_uiotombuf(uio, M_WAITOK, space, 2537 (atomic ? max_hdr : 0), 2538 (atomic ? M_PKTHDR : 0) | 2539 ((flags & MSG_EOR) ? M_EOR : 0)); 2540 if (top == NULL) { 2541 error = EFAULT; /* only possible error */ 2542 goto out; 2543 } 2544 space -= resid - uio->uio_resid; 2545 resid = uio->uio_resid; 2546 } 2547 if (dontroute) { 2548 SOCK_LOCK(so); 2549 so->so_options |= SO_DONTROUTE; 2550 SOCK_UNLOCK(so); 2551 } 2552 /* 2553 * XXX all the SBS_CANTSENDMORE checks previously 2554 * done could be out of date. We could have received 2555 * a reset packet in an interrupt or maybe we slept 2556 * while doing page faults in uiomove() etc. We 2557 * could probably recheck again inside the locking 2558 * protection here, but there are probably other 2559 * places that this also happens. We must rethink 2560 * this. 2561 */ 2562 VNET_SO_ASSERT(so); 2563 2564 pr_send_flag = (flags & MSG_OOB) ? PRUS_OOB : 2565 /* 2566 * If the user set MSG_EOF, the protocol understands 2567 * this flag and nothing left to send then use 2568 * PRU_SEND_EOF instead of PRU_SEND. 2569 */ 2570 ((flags & MSG_EOF) && 2571 (so->so_proto->pr_flags & PR_IMPLOPCL) && 2572 (resid <= 0)) ? 2573 PRUS_EOF : 2574 /* If there is more to send set PRUS_MORETOCOME. */ 2575 (flags & MSG_MORETOCOME) || 2576 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0; 2577 2578 #ifdef KERN_TLS 2579 pr_send_flag |= tls_send_flag; 2580 #endif 2581 2582 error = so->so_proto->pr_send(so, pr_send_flag, top, 2583 addr, control, td); 2584 2585 if (dontroute) { 2586 SOCK_LOCK(so); 2587 so->so_options &= ~SO_DONTROUTE; 2588 SOCK_UNLOCK(so); 2589 } 2590 2591 #ifdef KERN_TLS 2592 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { 2593 if (error != 0) { 2594 m_freem(top); 2595 top = NULL; 2596 } else { 2597 soref(so); 2598 ktls_enqueue(top, so, tls_enq_cnt); 2599 } 2600 } 2601 #endif 2602 clen = 0; 2603 control = NULL; 2604 top = NULL; 2605 if (error) 2606 goto out; 2607 } while (resid && space > 0); 2608 } while (resid); 2609 2610 out: 2611 #ifdef KERN_TLS 2612 if (tls != NULL) 2613 ktls_free(tls); 2614 #endif 2615 if (top != NULL) 2616 m_freem(top); 2617 if (control != NULL) 2618 m_freem(control); 2619 return (error); 2620 } 2621 2622 int 2623 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, 2624 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2625 { 2626 int error; 2627 2628 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2629 if (error) 2630 return (error); 2631 error = sosend_generic_locked(so, addr, uio, top, control, flags, td); 2632 SOCK_IO_SEND_UNLOCK(so); 2633 return (error); 2634 } 2635 2636 /* 2637 * Send to a socket from a kernel thread. 2638 * 2639 * XXXGL: in almost all cases uio is NULL and the mbuf is supplied. 2640 * Exception is nfs/bootp_subr.c. It is arguable that the VNET context needs 2641 * to be set at all. This function should just boil down to a static inline 2642 * calling the protocol method. 2643 */ 2644 int 2645 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, 2646 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2647 { 2648 int error; 2649 2650 CURVNET_SET(so->so_vnet); 2651 error = so->so_proto->pr_sosend(so, addr, uio, 2652 top, control, flags, td); 2653 CURVNET_RESTORE(); 2654 return (error); 2655 } 2656 2657 /* 2658 * send(2), write(2) or aio_write(2) on a socket. 2659 */ 2660 int 2661 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio, 2662 struct mbuf *control, int flags, struct proc *userproc) 2663 { 2664 struct thread *td; 2665 ssize_t len; 2666 int error; 2667 2668 td = uio->uio_td; 2669 len = uio->uio_resid; 2670 CURVNET_SET(so->so_vnet); 2671 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags, 2672 td); 2673 CURVNET_RESTORE(); 2674 if (error != 0) { 2675 /* 2676 * Clear transient errors for stream protocols if they made 2677 * some progress. Make exclusion for aio(4) that would 2678 * schedule a new write in case of EWOULDBLOCK and clear 2679 * error itself. See soaio_process_job(). 2680 */ 2681 if (uio->uio_resid != len && 2682 (so->so_proto->pr_flags & PR_ATOMIC) == 0 && 2683 userproc == NULL && 2684 (error == ERESTART || error == EINTR || 2685 error == EWOULDBLOCK)) 2686 error = 0; 2687 /* Generation of SIGPIPE can be controlled per socket. */ 2688 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 && 2689 (flags & MSG_NOSIGNAL) == 0) { 2690 if (userproc != NULL) { 2691 /* aio(4) job */ 2692 PROC_LOCK(userproc); 2693 kern_psignal(userproc, SIGPIPE); 2694 PROC_UNLOCK(userproc); 2695 } else { 2696 PROC_LOCK(td->td_proc); 2697 tdsignal(td, SIGPIPE); 2698 PROC_UNLOCK(td->td_proc); 2699 } 2700 } 2701 } 2702 return (error); 2703 } 2704 2705 /* 2706 * The part of soreceive() that implements reading non-inline out-of-band 2707 * data from a socket. For more complete comments, see soreceive(), from 2708 * which this code originated. 2709 * 2710 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 2711 * unable to return an mbuf chain to the caller. 2712 */ 2713 static int 2714 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) 2715 { 2716 struct protosw *pr = so->so_proto; 2717 struct mbuf *m; 2718 int error; 2719 2720 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 2721 VNET_SO_ASSERT(so); 2722 2723 m = m_get(M_WAITOK, MT_DATA); 2724 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK); 2725 if (error) 2726 goto bad; 2727 do { 2728 error = uiomove(mtod(m, void *), 2729 (int) min(uio->uio_resid, m->m_len), uio); 2730 m = m_free(m); 2731 } while (uio->uio_resid && error == 0 && m); 2732 bad: 2733 if (m != NULL) 2734 m_freem(m); 2735 return (error); 2736 } 2737 2738 /* 2739 * Following replacement or removal of the first mbuf on the first mbuf chain 2740 * of a socket buffer, push necessary state changes back into the socket 2741 * buffer so that other consumers see the values consistently. 'nextrecord' 2742 * is the callers locally stored value of the original value of 2743 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 2744 * NOTE: 'nextrecord' may be NULL. 2745 */ 2746 static __inline void 2747 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 2748 { 2749 2750 SOCKBUF_LOCK_ASSERT(sb); 2751 /* 2752 * First, update for the new value of nextrecord. If necessary, make 2753 * it the first record. 2754 */ 2755 if (sb->sb_mb != NULL) 2756 sb->sb_mb->m_nextpkt = nextrecord; 2757 else 2758 sb->sb_mb = nextrecord; 2759 2760 /* 2761 * Now update any dependent socket buffer fields to reflect the new 2762 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 2763 * addition of a second clause that takes care of the case where 2764 * sb_mb has been updated, but remains the last record. 2765 */ 2766 if (sb->sb_mb == NULL) { 2767 sb->sb_mbtail = NULL; 2768 sb->sb_lastrecord = NULL; 2769 } else if (sb->sb_mb->m_nextpkt == NULL) 2770 sb->sb_lastrecord = sb->sb_mb; 2771 } 2772 2773 /* 2774 * Implement receive operations on a socket. We depend on the way that 2775 * records are added to the sockbuf by sbappend. In particular, each record 2776 * (mbufs linked through m_next) must begin with an address if the protocol 2777 * so specifies, followed by an optional mbuf or mbufs containing ancillary 2778 * data, and then zero or more mbufs of data. In order to allow parallelism 2779 * between network receive and copying to user space, as well as avoid 2780 * sleeping with a mutex held, we release the socket buffer mutex during the 2781 * user space copy. Although the sockbuf is locked, new data may still be 2782 * appended, and thus we must maintain consistency of the sockbuf during that 2783 * time. 2784 * 2785 * The caller may receive the data as a single mbuf chain by supplying an 2786 * mbuf **mp for use in returning the chain. The uio is then used only for 2787 * the count in uio_resid. 2788 */ 2789 static int 2790 soreceive_generic_locked(struct socket *so, struct sockaddr **psa, 2791 struct uio *uio, struct mbuf **mp, struct mbuf **controlp, int *flagsp) 2792 { 2793 struct mbuf *m; 2794 int flags, error, offset; 2795 ssize_t len; 2796 struct protosw *pr = so->so_proto; 2797 struct mbuf *nextrecord; 2798 int moff, type = 0; 2799 ssize_t orig_resid = uio->uio_resid; 2800 bool report_real_len = false; 2801 2802 SOCK_IO_RECV_ASSERT_LOCKED(so); 2803 2804 error = 0; 2805 if (flagsp != NULL) { 2806 report_real_len = *flagsp & MSG_TRUNC; 2807 *flagsp &= ~MSG_TRUNC; 2808 flags = *flagsp &~ MSG_EOR; 2809 } else 2810 flags = 0; 2811 2812 restart: 2813 SOCKBUF_LOCK(&so->so_rcv); 2814 m = so->so_rcv.sb_mb; 2815 /* 2816 * If we have less data than requested, block awaiting more (subject 2817 * to any timeout) if: 2818 * 1. the current count is less than the low water mark, or 2819 * 2. MSG_DONTWAIT is not set 2820 */ 2821 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 2822 sbavail(&so->so_rcv) < uio->uio_resid) && 2823 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat && 2824 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 2825 KASSERT(m != NULL || !sbavail(&so->so_rcv), 2826 ("receive: m == %p sbavail == %u", 2827 m, sbavail(&so->so_rcv))); 2828 if (so->so_error || so->so_rerror) { 2829 if (m != NULL) 2830 goto dontblock; 2831 if (so->so_error) 2832 error = so->so_error; 2833 else 2834 error = so->so_rerror; 2835 if ((flags & MSG_PEEK) == 0) { 2836 if (so->so_error) 2837 so->so_error = 0; 2838 else 2839 so->so_rerror = 0; 2840 } 2841 SOCKBUF_UNLOCK(&so->so_rcv); 2842 goto release; 2843 } 2844 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2845 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 2846 if (m != NULL) 2847 goto dontblock; 2848 #ifdef KERN_TLS 2849 else if (so->so_rcv.sb_tlsdcc == 0 && 2850 so->so_rcv.sb_tlscc == 0) { 2851 #else 2852 else { 2853 #endif 2854 SOCKBUF_UNLOCK(&so->so_rcv); 2855 goto release; 2856 } 2857 } 2858 for (; m != NULL; m = m->m_next) 2859 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 2860 m = so->so_rcv.sb_mb; 2861 goto dontblock; 2862 } 2863 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED | 2864 SS_ISDISCONNECTING | SS_ISDISCONNECTED)) == 0 && 2865 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) { 2866 SOCKBUF_UNLOCK(&so->so_rcv); 2867 error = ENOTCONN; 2868 goto release; 2869 } 2870 if (uio->uio_resid == 0 && !report_real_len) { 2871 SOCKBUF_UNLOCK(&so->so_rcv); 2872 goto release; 2873 } 2874 if ((so->so_state & SS_NBIO) || 2875 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 2876 SOCKBUF_UNLOCK(&so->so_rcv); 2877 error = EWOULDBLOCK; 2878 goto release; 2879 } 2880 SBLASTRECORDCHK(&so->so_rcv); 2881 SBLASTMBUFCHK(&so->so_rcv); 2882 error = sbwait(so, SO_RCV); 2883 SOCKBUF_UNLOCK(&so->so_rcv); 2884 if (error) 2885 goto release; 2886 goto restart; 2887 } 2888 dontblock: 2889 /* 2890 * From this point onward, we maintain 'nextrecord' as a cache of the 2891 * pointer to the next record in the socket buffer. We must keep the 2892 * various socket buffer pointers and local stack versions of the 2893 * pointers in sync, pushing out modifications before dropping the 2894 * socket buffer mutex, and re-reading them when picking it up. 2895 * 2896 * Otherwise, we will race with the network stack appending new data 2897 * or records onto the socket buffer by using inconsistent/stale 2898 * versions of the field, possibly resulting in socket buffer 2899 * corruption. 2900 * 2901 * By holding the high-level sblock(), we prevent simultaneous 2902 * readers from pulling off the front of the socket buffer. 2903 */ 2904 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2905 if (uio->uio_td) 2906 uio->uio_td->td_ru.ru_msgrcv++; 2907 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 2908 SBLASTRECORDCHK(&so->so_rcv); 2909 SBLASTMBUFCHK(&so->so_rcv); 2910 nextrecord = m->m_nextpkt; 2911 if (pr->pr_flags & PR_ADDR) { 2912 KASSERT(m->m_type == MT_SONAME, 2913 ("m->m_type == %d", m->m_type)); 2914 orig_resid = 0; 2915 if (psa != NULL) 2916 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 2917 M_NOWAIT); 2918 if (flags & MSG_PEEK) { 2919 m = m->m_next; 2920 } else { 2921 sbfree(&so->so_rcv, m); 2922 so->so_rcv.sb_mb = m_free(m); 2923 m = so->so_rcv.sb_mb; 2924 sockbuf_pushsync(&so->so_rcv, nextrecord); 2925 } 2926 } 2927 2928 /* 2929 * Process one or more MT_CONTROL mbufs present before any data mbufs 2930 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 2931 * just copy the data; if !MSG_PEEK, we call into the protocol to 2932 * perform externalization (or freeing if controlp == NULL). 2933 */ 2934 if (m != NULL && m->m_type == MT_CONTROL) { 2935 struct mbuf *cm = NULL, *cmn; 2936 struct mbuf **cme = &cm; 2937 #ifdef KERN_TLS 2938 struct cmsghdr *cmsg; 2939 struct tls_get_record tgr; 2940 2941 /* 2942 * For MSG_TLSAPPDATA, check for an alert record. 2943 * If found, return ENXIO without removing 2944 * it from the receive queue. This allows a subsequent 2945 * call without MSG_TLSAPPDATA to receive it. 2946 * Note that, for TLS, there should only be a single 2947 * control mbuf with the TLS_GET_RECORD message in it. 2948 */ 2949 if (flags & MSG_TLSAPPDATA) { 2950 cmsg = mtod(m, struct cmsghdr *); 2951 if (cmsg->cmsg_type == TLS_GET_RECORD && 2952 cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) { 2953 memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr)); 2954 if (__predict_false(tgr.tls_type == 2955 TLS_RLTYPE_ALERT)) { 2956 SOCKBUF_UNLOCK(&so->so_rcv); 2957 error = ENXIO; 2958 goto release; 2959 } 2960 } 2961 } 2962 #endif 2963 2964 do { 2965 if (flags & MSG_PEEK) { 2966 if (controlp != NULL) { 2967 *controlp = m_copym(m, 0, m->m_len, 2968 M_NOWAIT); 2969 controlp = &(*controlp)->m_next; 2970 } 2971 m = m->m_next; 2972 } else { 2973 sbfree(&so->so_rcv, m); 2974 so->so_rcv.sb_mb = m->m_next; 2975 m->m_next = NULL; 2976 *cme = m; 2977 cme = &(*cme)->m_next; 2978 m = so->so_rcv.sb_mb; 2979 } 2980 } while (m != NULL && m->m_type == MT_CONTROL); 2981 if ((flags & MSG_PEEK) == 0) 2982 sockbuf_pushsync(&so->so_rcv, nextrecord); 2983 while (cm != NULL) { 2984 cmn = cm->m_next; 2985 cm->m_next = NULL; 2986 if (controlp != NULL) 2987 *controlp = cm; 2988 else 2989 m_freem(cm); 2990 if (controlp != NULL) { 2991 while (*controlp != NULL) 2992 controlp = &(*controlp)->m_next; 2993 } 2994 cm = cmn; 2995 } 2996 if (m != NULL) 2997 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 2998 else 2999 nextrecord = so->so_rcv.sb_mb; 3000 orig_resid = 0; 3001 } 3002 if (m != NULL) { 3003 if ((flags & MSG_PEEK) == 0) { 3004 KASSERT(m->m_nextpkt == nextrecord, 3005 ("soreceive: post-control, nextrecord !sync")); 3006 if (nextrecord == NULL) { 3007 KASSERT(so->so_rcv.sb_mb == m, 3008 ("soreceive: post-control, sb_mb!=m")); 3009 KASSERT(so->so_rcv.sb_lastrecord == m, 3010 ("soreceive: post-control, lastrecord!=m")); 3011 } 3012 } 3013 type = m->m_type; 3014 if (type == MT_OOBDATA) 3015 flags |= MSG_OOB; 3016 } else { 3017 if ((flags & MSG_PEEK) == 0) { 3018 KASSERT(so->so_rcv.sb_mb == nextrecord, 3019 ("soreceive: sb_mb != nextrecord")); 3020 if (so->so_rcv.sb_mb == NULL) { 3021 KASSERT(so->so_rcv.sb_lastrecord == NULL, 3022 ("soreceive: sb_lastercord != NULL")); 3023 } 3024 } 3025 } 3026 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3027 SBLASTRECORDCHK(&so->so_rcv); 3028 SBLASTMBUFCHK(&so->so_rcv); 3029 3030 /* 3031 * Now continue to read any data mbufs off of the head of the socket 3032 * buffer until the read request is satisfied. Note that 'type' is 3033 * used to store the type of any mbuf reads that have happened so far 3034 * such that soreceive() can stop reading if the type changes, which 3035 * causes soreceive() to return only one of regular data and inline 3036 * out-of-band data in a single socket receive operation. 3037 */ 3038 moff = 0; 3039 offset = 0; 3040 while (m != NULL && !(m->m_flags & M_NOTREADY) && uio->uio_resid > 0 && 3041 error == 0) { 3042 /* 3043 * If the type of mbuf has changed since the last mbuf 3044 * examined ('type'), end the receive operation. 3045 */ 3046 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3047 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) { 3048 if (type != m->m_type) 3049 break; 3050 } else if (type == MT_OOBDATA) 3051 break; 3052 else 3053 KASSERT(m->m_type == MT_DATA, 3054 ("m->m_type == %d", m->m_type)); 3055 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 3056 len = uio->uio_resid; 3057 if (so->so_oobmark && len > so->so_oobmark - offset) 3058 len = so->so_oobmark - offset; 3059 if (len > m->m_len - moff) 3060 len = m->m_len - moff; 3061 /* 3062 * If mp is set, just pass back the mbufs. Otherwise copy 3063 * them out via the uio, then free. Sockbuf must be 3064 * consistent here (points to current mbuf, it points to next 3065 * record) when we drop priority; we must note any additions 3066 * to the sockbuf when we block interrupts again. 3067 */ 3068 if (mp == NULL) { 3069 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3070 SBLASTRECORDCHK(&so->so_rcv); 3071 SBLASTMBUFCHK(&so->so_rcv); 3072 SOCKBUF_UNLOCK(&so->so_rcv); 3073 if ((m->m_flags & M_EXTPG) != 0) 3074 error = m_unmapped_uiomove(m, moff, uio, 3075 (int)len); 3076 else 3077 error = uiomove(mtod(m, char *) + moff, 3078 (int)len, uio); 3079 SOCKBUF_LOCK(&so->so_rcv); 3080 if (error) { 3081 /* 3082 * The MT_SONAME mbuf has already been removed 3083 * from the record, so it is necessary to 3084 * remove the data mbufs, if any, to preserve 3085 * the invariant in the case of PR_ADDR that 3086 * requires MT_SONAME mbufs at the head of 3087 * each record. 3088 */ 3089 if (pr->pr_flags & PR_ATOMIC && 3090 ((flags & MSG_PEEK) == 0)) 3091 (void)sbdroprecord_locked(&so->so_rcv); 3092 SOCKBUF_UNLOCK(&so->so_rcv); 3093 goto release; 3094 } 3095 } else 3096 uio->uio_resid -= len; 3097 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3098 if (len == m->m_len - moff) { 3099 if (m->m_flags & M_EOR) 3100 flags |= MSG_EOR; 3101 if (flags & MSG_PEEK) { 3102 m = m->m_next; 3103 moff = 0; 3104 } else { 3105 nextrecord = m->m_nextpkt; 3106 sbfree(&so->so_rcv, m); 3107 if (mp != NULL) { 3108 m->m_nextpkt = NULL; 3109 *mp = m; 3110 mp = &m->m_next; 3111 so->so_rcv.sb_mb = m = m->m_next; 3112 *mp = NULL; 3113 } else { 3114 so->so_rcv.sb_mb = m_free(m); 3115 m = so->so_rcv.sb_mb; 3116 } 3117 sockbuf_pushsync(&so->so_rcv, nextrecord); 3118 SBLASTRECORDCHK(&so->so_rcv); 3119 SBLASTMBUFCHK(&so->so_rcv); 3120 } 3121 } else { 3122 if (flags & MSG_PEEK) 3123 moff += len; 3124 else { 3125 if (mp != NULL) { 3126 if (flags & MSG_DONTWAIT) { 3127 *mp = m_copym(m, 0, len, 3128 M_NOWAIT); 3129 if (*mp == NULL) { 3130 /* 3131 * m_copym() couldn't 3132 * allocate an mbuf. 3133 * Adjust uio_resid back 3134 * (it was adjusted 3135 * down by len bytes, 3136 * which we didn't end 3137 * up "copying" over). 3138 */ 3139 uio->uio_resid += len; 3140 break; 3141 } 3142 } else { 3143 SOCKBUF_UNLOCK(&so->so_rcv); 3144 *mp = m_copym(m, 0, len, 3145 M_WAITOK); 3146 SOCKBUF_LOCK(&so->so_rcv); 3147 } 3148 } 3149 sbcut_locked(&so->so_rcv, len); 3150 } 3151 } 3152 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3153 if (so->so_oobmark) { 3154 if ((flags & MSG_PEEK) == 0) { 3155 so->so_oobmark -= len; 3156 if (so->so_oobmark == 0) { 3157 so->so_rcv.sb_state |= SBS_RCVATMARK; 3158 break; 3159 } 3160 } else { 3161 offset += len; 3162 if (offset == so->so_oobmark) 3163 break; 3164 } 3165 } 3166 if (flags & MSG_EOR) 3167 break; 3168 /* 3169 * If the MSG_WAITALL flag is set (for non-atomic socket), we 3170 * must not quit until "uio->uio_resid == 0" or an error 3171 * termination. If a signal/timeout occurs, return with a 3172 * short count but without error. Keep sockbuf locked 3173 * against other readers. 3174 */ 3175 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 3176 !sosendallatonce(so) && nextrecord == NULL) { 3177 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3178 if (so->so_error || so->so_rerror || 3179 so->so_rcv.sb_state & SBS_CANTRCVMORE) 3180 break; 3181 /* 3182 * Notify the protocol that some data has been 3183 * drained before blocking. 3184 */ 3185 if (pr->pr_flags & PR_WANTRCVD) { 3186 SOCKBUF_UNLOCK(&so->so_rcv); 3187 VNET_SO_ASSERT(so); 3188 pr->pr_rcvd(so, flags); 3189 SOCKBUF_LOCK(&so->so_rcv); 3190 if (__predict_false(so->so_rcv.sb_mb == NULL && 3191 (so->so_error || so->so_rerror || 3192 so->so_rcv.sb_state & SBS_CANTRCVMORE))) 3193 break; 3194 } 3195 SBLASTRECORDCHK(&so->so_rcv); 3196 SBLASTMBUFCHK(&so->so_rcv); 3197 /* 3198 * We could receive some data while was notifying 3199 * the protocol. Skip blocking in this case. 3200 */ 3201 if (so->so_rcv.sb_mb == NULL) { 3202 error = sbwait(so, SO_RCV); 3203 if (error) { 3204 SOCKBUF_UNLOCK(&so->so_rcv); 3205 goto release; 3206 } 3207 } 3208 m = so->so_rcv.sb_mb; 3209 if (m != NULL) 3210 nextrecord = m->m_nextpkt; 3211 } 3212 } 3213 3214 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3215 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 3216 if (report_real_len) 3217 uio->uio_resid -= m_length(m, NULL) - moff; 3218 flags |= MSG_TRUNC; 3219 if ((flags & MSG_PEEK) == 0) 3220 (void) sbdroprecord_locked(&so->so_rcv); 3221 } 3222 if ((flags & MSG_PEEK) == 0) { 3223 if (m == NULL) { 3224 /* 3225 * First part is an inline SB_EMPTY_FIXUP(). Second 3226 * part makes sure sb_lastrecord is up-to-date if 3227 * there is still data in the socket buffer. 3228 */ 3229 so->so_rcv.sb_mb = nextrecord; 3230 if (so->so_rcv.sb_mb == NULL) { 3231 so->so_rcv.sb_mbtail = NULL; 3232 so->so_rcv.sb_lastrecord = NULL; 3233 } else if (nextrecord->m_nextpkt == NULL) 3234 so->so_rcv.sb_lastrecord = nextrecord; 3235 } 3236 SBLASTRECORDCHK(&so->so_rcv); 3237 SBLASTMBUFCHK(&so->so_rcv); 3238 /* 3239 * If soreceive() is being done from the socket callback, 3240 * then don't need to generate ACK to peer to update window, 3241 * since ACK will be generated on return to TCP. 3242 */ 3243 if (!(flags & MSG_SOCALLBCK) && 3244 (pr->pr_flags & PR_WANTRCVD)) { 3245 SOCKBUF_UNLOCK(&so->so_rcv); 3246 VNET_SO_ASSERT(so); 3247 pr->pr_rcvd(so, flags); 3248 SOCKBUF_LOCK(&so->so_rcv); 3249 } 3250 } 3251 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3252 if (orig_resid == uio->uio_resid && orig_resid && 3253 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 3254 SOCKBUF_UNLOCK(&so->so_rcv); 3255 goto restart; 3256 } 3257 SOCKBUF_UNLOCK(&so->so_rcv); 3258 3259 if (flagsp != NULL) 3260 *flagsp |= flags; 3261 release: 3262 return (error); 3263 } 3264 3265 int 3266 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, 3267 struct mbuf **mp, struct mbuf **controlp, int *flagsp) 3268 { 3269 int error, flags; 3270 3271 if (psa != NULL) 3272 *psa = NULL; 3273 if (controlp != NULL) 3274 *controlp = NULL; 3275 if (flagsp != NULL) { 3276 flags = *flagsp; 3277 if ((flags & MSG_OOB) != 0) 3278 return (soreceive_rcvoob(so, uio, flags)); 3279 } else { 3280 flags = 0; 3281 } 3282 if (mp != NULL) 3283 *mp = NULL; 3284 3285 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 3286 if (error) 3287 return (error); 3288 error = soreceive_generic_locked(so, psa, uio, mp, controlp, flagsp); 3289 SOCK_IO_RECV_UNLOCK(so); 3290 return (error); 3291 } 3292 3293 /* 3294 * Optimized version of soreceive() for stream (TCP) sockets. 3295 */ 3296 static int 3297 soreceive_stream_locked(struct socket *so, struct sockbuf *sb, 3298 struct sockaddr **psa, struct uio *uio, struct mbuf **mp0, 3299 struct mbuf **controlp, int flags) 3300 { 3301 int len = 0, error = 0, oresid; 3302 struct mbuf *m, *n = NULL; 3303 3304 SOCK_IO_RECV_ASSERT_LOCKED(so); 3305 3306 /* Easy one, no space to copyout anything. */ 3307 if (uio->uio_resid == 0) 3308 return (EINVAL); 3309 oresid = uio->uio_resid; 3310 3311 SOCKBUF_LOCK(sb); 3312 /* We will never ever get anything unless we are or were connected. */ 3313 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { 3314 error = ENOTCONN; 3315 goto out; 3316 } 3317 3318 restart: 3319 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3320 3321 /* Abort if socket has reported problems. */ 3322 if (so->so_error) { 3323 if (sbavail(sb) > 0) 3324 goto deliver; 3325 if (oresid > uio->uio_resid) 3326 goto out; 3327 error = so->so_error; 3328 if (!(flags & MSG_PEEK)) 3329 so->so_error = 0; 3330 goto out; 3331 } 3332 3333 /* Door is closed. Deliver what is left, if any. */ 3334 if (sb->sb_state & SBS_CANTRCVMORE) { 3335 if (sbavail(sb) > 0) 3336 goto deliver; 3337 else 3338 goto out; 3339 } 3340 3341 /* Socket buffer is empty and we shall not block. */ 3342 if (sbavail(sb) == 0 && 3343 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 3344 error = EAGAIN; 3345 goto out; 3346 } 3347 3348 /* Socket buffer got some data that we shall deliver now. */ 3349 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) && 3350 ((so->so_state & SS_NBIO) || 3351 (flags & (MSG_DONTWAIT|MSG_NBIO)) || 3352 sbavail(sb) >= sb->sb_lowat || 3353 sbavail(sb) >= uio->uio_resid || 3354 sbavail(sb) >= sb->sb_hiwat) ) { 3355 goto deliver; 3356 } 3357 3358 /* On MSG_WAITALL we must wait until all data or error arrives. */ 3359 if ((flags & MSG_WAITALL) && 3360 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat)) 3361 goto deliver; 3362 3363 /* 3364 * Wait and block until (more) data comes in. 3365 * NB: Drops the sockbuf lock during wait. 3366 */ 3367 error = sbwait(so, SO_RCV); 3368 if (error) 3369 goto out; 3370 goto restart; 3371 3372 deliver: 3373 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3374 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__)); 3375 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 3376 3377 /* Statistics. */ 3378 if (uio->uio_td) 3379 uio->uio_td->td_ru.ru_msgrcv++; 3380 3381 /* Fill uio until full or current end of socket buffer is reached. */ 3382 len = min(uio->uio_resid, sbavail(sb)); 3383 if (mp0 != NULL) { 3384 /* Dequeue as many mbufs as possible. */ 3385 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 3386 if (*mp0 == NULL) 3387 *mp0 = sb->sb_mb; 3388 else 3389 m_cat(*mp0, sb->sb_mb); 3390 for (m = sb->sb_mb; 3391 m != NULL && m->m_len <= len; 3392 m = m->m_next) { 3393 KASSERT(!(m->m_flags & M_NOTREADY), 3394 ("%s: m %p not available", __func__, m)); 3395 len -= m->m_len; 3396 uio->uio_resid -= m->m_len; 3397 sbfree(sb, m); 3398 n = m; 3399 } 3400 n->m_next = NULL; 3401 sb->sb_mb = m; 3402 sb->sb_lastrecord = sb->sb_mb; 3403 if (sb->sb_mb == NULL) 3404 SB_EMPTY_FIXUP(sb); 3405 } 3406 /* Copy the remainder. */ 3407 if (len > 0) { 3408 KASSERT(sb->sb_mb != NULL, 3409 ("%s: len > 0 && sb->sb_mb empty", __func__)); 3410 3411 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT); 3412 if (m == NULL) 3413 len = 0; /* Don't flush data from sockbuf. */ 3414 else 3415 uio->uio_resid -= len; 3416 if (*mp0 != NULL) 3417 m_cat(*mp0, m); 3418 else 3419 *mp0 = m; 3420 if (*mp0 == NULL) { 3421 error = ENOBUFS; 3422 goto out; 3423 } 3424 } 3425 } else { 3426 /* NB: Must unlock socket buffer as uiomove may sleep. */ 3427 SOCKBUF_UNLOCK(sb); 3428 error = m_mbuftouio(uio, sb->sb_mb, len); 3429 SOCKBUF_LOCK(sb); 3430 if (error) 3431 goto out; 3432 } 3433 SBLASTRECORDCHK(sb); 3434 SBLASTMBUFCHK(sb); 3435 3436 /* 3437 * Remove the delivered data from the socket buffer unless we 3438 * were only peeking. 3439 */ 3440 if (!(flags & MSG_PEEK)) { 3441 if (len > 0) 3442 sbdrop_locked(sb, len); 3443 3444 /* Notify protocol that we drained some data. */ 3445 if ((so->so_proto->pr_flags & PR_WANTRCVD) && 3446 (((flags & MSG_WAITALL) && uio->uio_resid > 0) || 3447 !(flags & MSG_SOCALLBCK))) { 3448 SOCKBUF_UNLOCK(sb); 3449 VNET_SO_ASSERT(so); 3450 so->so_proto->pr_rcvd(so, flags); 3451 SOCKBUF_LOCK(sb); 3452 } 3453 } 3454 3455 /* 3456 * For MSG_WAITALL we may have to loop again and wait for 3457 * more data to come in. 3458 */ 3459 if ((flags & MSG_WAITALL) && uio->uio_resid > 0) 3460 goto restart; 3461 out: 3462 SBLASTRECORDCHK(sb); 3463 SBLASTMBUFCHK(sb); 3464 SOCKBUF_UNLOCK(sb); 3465 return (error); 3466 } 3467 3468 int 3469 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, 3470 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3471 { 3472 struct sockbuf *sb; 3473 int error, flags; 3474 3475 sb = &so->so_rcv; 3476 3477 /* We only do stream sockets. */ 3478 if (so->so_type != SOCK_STREAM) 3479 return (EINVAL); 3480 if (psa != NULL) 3481 *psa = NULL; 3482 if (flagsp != NULL) 3483 flags = *flagsp & ~MSG_EOR; 3484 else 3485 flags = 0; 3486 if (controlp != NULL) 3487 *controlp = NULL; 3488 if (flags & MSG_OOB) 3489 return (soreceive_rcvoob(so, uio, flags)); 3490 if (mp0 != NULL) 3491 *mp0 = NULL; 3492 3493 #ifdef KERN_TLS 3494 /* 3495 * KTLS store TLS records as records with a control message to 3496 * describe the framing. 3497 * 3498 * We check once here before acquiring locks to optimize the 3499 * common case. 3500 */ 3501 if (sb->sb_tls_info != NULL) 3502 return (soreceive_generic(so, psa, uio, mp0, controlp, 3503 flagsp)); 3504 #endif 3505 3506 /* 3507 * Prevent other threads from reading from the socket. This lock may be 3508 * dropped in order to sleep waiting for data to arrive. 3509 */ 3510 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 3511 if (error) 3512 return (error); 3513 #ifdef KERN_TLS 3514 if (__predict_false(sb->sb_tls_info != NULL)) { 3515 SOCK_IO_RECV_UNLOCK(so); 3516 return (soreceive_generic(so, psa, uio, mp0, controlp, 3517 flagsp)); 3518 } 3519 #endif 3520 error = soreceive_stream_locked(so, sb, psa, uio, mp0, controlp, flags); 3521 SOCK_IO_RECV_UNLOCK(so); 3522 return (error); 3523 } 3524 3525 /* 3526 * Optimized version of soreceive() for simple datagram cases from userspace. 3527 * Unlike in the stream case, we're able to drop a datagram if copyout() 3528 * fails, and because we handle datagrams atomically, we don't need to use a 3529 * sleep lock to prevent I/O interlacing. 3530 */ 3531 int 3532 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 3533 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3534 { 3535 struct mbuf *m, *m2; 3536 int flags, error; 3537 ssize_t len; 3538 struct protosw *pr = so->so_proto; 3539 struct mbuf *nextrecord; 3540 3541 if (psa != NULL) 3542 *psa = NULL; 3543 if (controlp != NULL) 3544 *controlp = NULL; 3545 if (flagsp != NULL) 3546 flags = *flagsp &~ MSG_EOR; 3547 else 3548 flags = 0; 3549 3550 /* 3551 * For any complicated cases, fall back to the full 3552 * soreceive_generic(). 3553 */ 3554 if (mp0 != NULL || (flags & (MSG_PEEK | MSG_OOB | MSG_TRUNC))) 3555 return (soreceive_generic(so, psa, uio, mp0, controlp, 3556 flagsp)); 3557 3558 /* 3559 * Enforce restrictions on use. 3560 */ 3561 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0, 3562 ("soreceive_dgram: wantrcvd")); 3563 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic")); 3564 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0, 3565 ("soreceive_dgram: SBS_RCVATMARK")); 3566 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0, 3567 ("soreceive_dgram: P_CONNREQUIRED")); 3568 3569 /* 3570 * Loop blocking while waiting for a datagram. 3571 */ 3572 SOCKBUF_LOCK(&so->so_rcv); 3573 while ((m = so->so_rcv.sb_mb) == NULL) { 3574 KASSERT(sbavail(&so->so_rcv) == 0, 3575 ("soreceive_dgram: sb_mb NULL but sbavail %u", 3576 sbavail(&so->so_rcv))); 3577 if (so->so_error) { 3578 error = so->so_error; 3579 so->so_error = 0; 3580 SOCKBUF_UNLOCK(&so->so_rcv); 3581 return (error); 3582 } 3583 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 3584 uio->uio_resid == 0) { 3585 SOCKBUF_UNLOCK(&so->so_rcv); 3586 return (0); 3587 } 3588 if ((so->so_state & SS_NBIO) || 3589 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 3590 SOCKBUF_UNLOCK(&so->so_rcv); 3591 return (EWOULDBLOCK); 3592 } 3593 SBLASTRECORDCHK(&so->so_rcv); 3594 SBLASTMBUFCHK(&so->so_rcv); 3595 error = sbwait(so, SO_RCV); 3596 if (error) { 3597 SOCKBUF_UNLOCK(&so->so_rcv); 3598 return (error); 3599 } 3600 } 3601 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3602 3603 if (uio->uio_td) 3604 uio->uio_td->td_ru.ru_msgrcv++; 3605 SBLASTRECORDCHK(&so->so_rcv); 3606 SBLASTMBUFCHK(&so->so_rcv); 3607 nextrecord = m->m_nextpkt; 3608 if (nextrecord == NULL) { 3609 KASSERT(so->so_rcv.sb_lastrecord == m, 3610 ("soreceive_dgram: lastrecord != m")); 3611 } 3612 3613 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord, 3614 ("soreceive_dgram: m_nextpkt != nextrecord")); 3615 3616 /* 3617 * Pull 'm' and its chain off the front of the packet queue. 3618 */ 3619 so->so_rcv.sb_mb = NULL; 3620 sockbuf_pushsync(&so->so_rcv, nextrecord); 3621 3622 /* 3623 * Walk 'm's chain and free that many bytes from the socket buffer. 3624 */ 3625 for (m2 = m; m2 != NULL; m2 = m2->m_next) 3626 sbfree(&so->so_rcv, m2); 3627 3628 /* 3629 * Do a few last checks before we let go of the lock. 3630 */ 3631 SBLASTRECORDCHK(&so->so_rcv); 3632 SBLASTMBUFCHK(&so->so_rcv); 3633 SOCKBUF_UNLOCK(&so->so_rcv); 3634 3635 if (pr->pr_flags & PR_ADDR) { 3636 KASSERT(m->m_type == MT_SONAME, 3637 ("m->m_type == %d", m->m_type)); 3638 if (psa != NULL) 3639 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 3640 M_WAITOK); 3641 m = m_free(m); 3642 } 3643 KASSERT(m, ("%s: no data or control after soname", __func__)); 3644 3645 /* 3646 * Packet to copyout() is now in 'm' and it is disconnected from the 3647 * queue. 3648 * 3649 * Process one or more MT_CONTROL mbufs present before any data mbufs 3650 * in the first mbuf chain on the socket buffer. We call into the 3651 * protocol to perform externalization (or freeing if controlp == 3652 * NULL). In some cases there can be only MT_CONTROL mbufs without 3653 * MT_DATA mbufs. 3654 */ 3655 if (m->m_type == MT_CONTROL) { 3656 struct mbuf *cm = NULL, *cmn; 3657 struct mbuf **cme = &cm; 3658 3659 do { 3660 m2 = m->m_next; 3661 m->m_next = NULL; 3662 *cme = m; 3663 cme = &(*cme)->m_next; 3664 m = m2; 3665 } while (m != NULL && m->m_type == MT_CONTROL); 3666 while (cm != NULL) { 3667 cmn = cm->m_next; 3668 cm->m_next = NULL; 3669 if (controlp != NULL) 3670 *controlp = cm; 3671 else 3672 m_freem(cm); 3673 if (controlp != NULL) { 3674 while (*controlp != NULL) 3675 controlp = &(*controlp)->m_next; 3676 } 3677 cm = cmn; 3678 } 3679 } 3680 KASSERT(m == NULL || m->m_type == MT_DATA, 3681 ("soreceive_dgram: !data")); 3682 while (m != NULL && uio->uio_resid > 0) { 3683 len = uio->uio_resid; 3684 if (len > m->m_len) 3685 len = m->m_len; 3686 error = uiomove(mtod(m, char *), (int)len, uio); 3687 if (error) { 3688 m_freem(m); 3689 return (error); 3690 } 3691 if (len == m->m_len) 3692 m = m_free(m); 3693 else { 3694 m->m_data += len; 3695 m->m_len -= len; 3696 } 3697 } 3698 if (m != NULL) { 3699 flags |= MSG_TRUNC; 3700 m_freem(m); 3701 } 3702 if (flagsp != NULL) 3703 *flagsp |= flags; 3704 return (0); 3705 } 3706 3707 int 3708 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, 3709 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3710 { 3711 int error; 3712 3713 CURVNET_SET(so->so_vnet); 3714 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp); 3715 CURVNET_RESTORE(); 3716 return (error); 3717 } 3718 3719 int 3720 soshutdown(struct socket *so, enum shutdown_how how) 3721 { 3722 int error; 3723 3724 CURVNET_SET(so->so_vnet); 3725 error = so->so_proto->pr_shutdown(so, how); 3726 CURVNET_RESTORE(); 3727 3728 return (error); 3729 } 3730 3731 /* 3732 * Used by several pr_shutdown implementations that use generic socket buffers. 3733 */ 3734 void 3735 sorflush(struct socket *so) 3736 { 3737 int error; 3738 3739 VNET_SO_ASSERT(so); 3740 3741 /* 3742 * Dislodge threads currently blocked in receive and wait to acquire 3743 * a lock against other simultaneous readers before clearing the 3744 * socket buffer. Don't let our acquire be interrupted by a signal 3745 * despite any existing socket disposition on interruptable waiting. 3746 * 3747 * The SOCK_IO_RECV_LOCK() is important here as there some pr_soreceive 3748 * methods that read the top of the socket buffer without acquisition 3749 * of the socket buffer mutex, assuming that top of the buffer 3750 * exclusively belongs to the read(2) syscall. This is handy when 3751 * performing MSG_PEEK. 3752 */ 3753 socantrcvmore(so); 3754 3755 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 3756 if (error != 0) { 3757 KASSERT(SOLISTENING(so), 3758 ("%s: soiolock(%p) failed", __func__, so)); 3759 return; 3760 } 3761 3762 sbrelease(so, SO_RCV); 3763 SOCK_IO_RECV_UNLOCK(so); 3764 3765 } 3766 3767 int 3768 sosetfib(struct socket *so, int fibnum) 3769 { 3770 if (fibnum < 0 || fibnum >= rt_numfibs) 3771 return (EINVAL); 3772 3773 SOCK_LOCK(so); 3774 so->so_fibnum = fibnum; 3775 SOCK_UNLOCK(so); 3776 3777 return (0); 3778 } 3779 3780 #ifdef SOCKET_HHOOK 3781 /* 3782 * Wrapper for Socket established helper hook. 3783 * Parameters: socket, context of the hook point, hook id. 3784 */ 3785 static inline int 3786 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) 3787 { 3788 struct socket_hhook_data hhook_data = { 3789 .so = so, 3790 .hctx = hctx, 3791 .m = NULL, 3792 .status = 0 3793 }; 3794 3795 CURVNET_SET(so->so_vnet); 3796 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd); 3797 CURVNET_RESTORE(); 3798 3799 /* Ugly but needed, since hhooks return void for now */ 3800 return (hhook_data.status); 3801 } 3802 #endif 3803 3804 /* 3805 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 3806 * additional variant to handle the case where the option value needs to be 3807 * some kind of integer, but not a specific size. In addition to their use 3808 * here, these functions are also called by the protocol-level pr_ctloutput() 3809 * routines. 3810 */ 3811 static int 3812 _sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen, 3813 bool copycaps) 3814 { 3815 size_t valsize; 3816 3817 /* 3818 * If the user gives us more than we wanted, we ignore it, but if we 3819 * don't get the minimum length the caller wants, we return EINVAL. 3820 * On success, sopt->sopt_valsize is set to however much we actually 3821 * retrieved. 3822 */ 3823 if ((valsize = sopt->sopt_valsize) < minlen) 3824 return EINVAL; 3825 if (valsize > len) 3826 sopt->sopt_valsize = valsize = len; 3827 3828 if (sopt->sopt_td != NULL) { 3829 if (copycaps) 3830 return (copyinptr(sopt->sopt_val, buf, valsize)); 3831 else 3832 return (copyin(sopt->sopt_val, buf, valsize)); 3833 } 3834 3835 if (copycaps) 3836 memcpy(buf, sopt->sopt_val, valsize); 3837 else 3838 memcpy_data(buf, sopt->sopt_val, valsize); 3839 return (0); 3840 } 3841 3842 int 3843 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 3844 { 3845 return (_sooptcopyin(sopt, buf, len, minlen, false)); 3846 } 3847 3848 #ifdef __CHERI__ 3849 int 3850 sooptcopyinptr(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 3851 { 3852 return (_sooptcopyin(sopt, buf, len, minlen, true)); 3853 } 3854 #endif 3855 3856 /* 3857 * Kernel version of setsockopt(2). 3858 * 3859 * XXX: optlen is size_t, not socklen_t 3860 */ 3861 int 3862 so_setsockopt(struct socket *so, int level, int optname, void *optval, 3863 size_t optlen) 3864 { 3865 struct sockopt sopt; 3866 3867 sopt.sopt_level = level; 3868 sopt.sopt_name = optname; 3869 sopt.sopt_dir = SOPT_SET; 3870 sopt.sopt_val = optval; 3871 sopt.sopt_valsize = optlen; 3872 sopt.sopt_td = NULL; 3873 return (sosetopt(so, &sopt)); 3874 } 3875 3876 int 3877 sosetopt(struct socket *so, struct sockopt *sopt) 3878 { 3879 int error, optval; 3880 struct linger l; 3881 struct timeval tv; 3882 sbintime_t val, *valp; 3883 uint32_t val32; 3884 #ifdef MAC 3885 struct mac extmac; 3886 #endif 3887 3888 CURVNET_SET(so->so_vnet); 3889 error = 0; 3890 if (sopt->sopt_level != SOL_SOCKET) { 3891 error = so->so_proto->pr_ctloutput(so, sopt); 3892 } else { 3893 switch (sopt->sopt_name) { 3894 case SO_ACCEPTFILTER: 3895 error = accept_filt_setopt(so, sopt); 3896 if (error) 3897 goto bad; 3898 break; 3899 3900 case SO_LINGER: 3901 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 3902 if (error) 3903 goto bad; 3904 if (l.l_linger < 0 || 3905 l.l_linger > USHRT_MAX || 3906 l.l_linger > (INT_MAX / hz)) { 3907 error = EDOM; 3908 goto bad; 3909 } 3910 SOCK_LOCK(so); 3911 so->so_linger = l.l_linger; 3912 if (l.l_onoff) 3913 so->so_options |= SO_LINGER; 3914 else 3915 so->so_options &= ~SO_LINGER; 3916 SOCK_UNLOCK(so); 3917 break; 3918 3919 case SO_DEBUG: 3920 case SO_KEEPALIVE: 3921 case SO_DONTROUTE: 3922 case SO_USELOOPBACK: 3923 case SO_BROADCAST: 3924 case SO_REUSEADDR: 3925 case SO_REUSEPORT: 3926 case SO_REUSEPORT_LB: 3927 case SO_OOBINLINE: 3928 case SO_TIMESTAMP: 3929 case SO_BINTIME: 3930 case SO_NOSIGPIPE: 3931 case SO_NO_DDP: 3932 case SO_NO_OFFLOAD: 3933 case SO_RERROR: 3934 stdopt: 3935 error = sooptcopyin(sopt, &optval, sizeof optval, 3936 sizeof optval); 3937 if (error) 3938 goto bad; 3939 SOCK_LOCK(so); 3940 if (optval) 3941 so->so_options |= sopt->sopt_name; 3942 else 3943 so->so_options &= ~sopt->sopt_name; 3944 SOCK_UNLOCK(so); 3945 break; 3946 3947 case SO_PASSRIGHTS: 3948 if (so->so_proto->pr_domain->dom_family != AF_LOCAL) { 3949 error = EOPNOTSUPP; 3950 goto bad; 3951 } 3952 3953 goto stdopt; 3954 3955 case SO_SETFIB: 3956 error = so->so_proto->pr_ctloutput(so, sopt); 3957 break; 3958 3959 case SO_USER_COOKIE: 3960 error = sooptcopyin(sopt, &val32, sizeof val32, 3961 sizeof val32); 3962 if (error) 3963 goto bad; 3964 so->so_user_cookie = val32; 3965 break; 3966 3967 case SO_SNDBUF: 3968 case SO_RCVBUF: 3969 case SO_SNDLOWAT: 3970 case SO_RCVLOWAT: 3971 error = so->so_proto->pr_setsbopt(so, sopt); 3972 if (error) 3973 goto bad; 3974 break; 3975 3976 case SO_SNDTIMEO: 3977 case SO_RCVTIMEO: 3978 #ifdef COMPAT_FREEBSD32 3979 if (SV_CURPROC_FLAG(SV_ILP32)) { 3980 struct timeval32 tv32; 3981 3982 error = sooptcopyin(sopt, &tv32, sizeof tv32, 3983 sizeof tv32); 3984 CP(tv32, tv, tv_sec); 3985 CP(tv32, tv, tv_usec); 3986 } else 3987 #endif 3988 error = sooptcopyin(sopt, &tv, sizeof tv, 3989 sizeof tv); 3990 if (error) 3991 goto bad; 3992 if (tv.tv_sec < 0 || tv.tv_usec < 0 || 3993 tv.tv_usec >= 1000000) { 3994 error = EDOM; 3995 goto bad; 3996 } 3997 if (tv.tv_sec > INT32_MAX) 3998 val = SBT_MAX; 3999 else 4000 val = tvtosbt(tv); 4001 SOCK_LOCK(so); 4002 valp = sopt->sopt_name == SO_SNDTIMEO ? 4003 (SOLISTENING(so) ? &so->sol_sbsnd_timeo : 4004 &so->so_snd.sb_timeo) : 4005 (SOLISTENING(so) ? &so->sol_sbrcv_timeo : 4006 &so->so_rcv.sb_timeo); 4007 *valp = val; 4008 SOCK_UNLOCK(so); 4009 break; 4010 4011 case SO_LABEL: 4012 #ifdef MAC 4013 error = sooptcopyin(sopt, &extmac, sizeof extmac, 4014 sizeof extmac); 4015 if (error) 4016 goto bad; 4017 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 4018 so, &extmac); 4019 #else 4020 error = EOPNOTSUPP; 4021 #endif 4022 break; 4023 4024 case SO_TS_CLOCK: 4025 error = sooptcopyin(sopt, &optval, sizeof optval, 4026 sizeof optval); 4027 if (error) 4028 goto bad; 4029 if (optval < 0 || optval > SO_TS_CLOCK_MAX) { 4030 error = EINVAL; 4031 goto bad; 4032 } 4033 so->so_ts_clock = optval; 4034 break; 4035 4036 case SO_MAX_PACING_RATE: 4037 error = sooptcopyin(sopt, &val32, sizeof(val32), 4038 sizeof(val32)); 4039 if (error) 4040 goto bad; 4041 so->so_max_pacing_rate = val32; 4042 break; 4043 4044 case SO_SPLICE: { 4045 struct splice splice; 4046 4047 #ifdef COMPAT_FREEBSD32 4048 if (SV_CURPROC_FLAG(SV_ILP32)) { 4049 struct splice32 splice32; 4050 4051 error = sooptcopyin(sopt, &splice32, 4052 sizeof(splice32), sizeof(splice32)); 4053 if (error == 0) { 4054 splice.sp_fd = splice32.sp_fd; 4055 splice.sp_max = splice32.sp_max; 4056 CP(splice32.sp_idle, splice.sp_idle, 4057 tv_sec); 4058 CP(splice32.sp_idle, splice.sp_idle, 4059 tv_usec); 4060 } 4061 } else 4062 #endif 4063 { 4064 error = sooptcopyin(sopt, &splice, 4065 sizeof(splice), sizeof(splice)); 4066 } 4067 if (error) 4068 goto bad; 4069 #ifdef KTRACE 4070 if (KTRPOINT(curthread, KTR_STRUCT)) 4071 ktrsplice(&splice); 4072 #endif 4073 4074 error = splice_init(); 4075 if (error != 0) 4076 goto bad; 4077 4078 if (splice.sp_fd >= 0) { 4079 struct file *fp; 4080 struct socket *so2; 4081 4082 if (!cap_rights_contains(sopt->sopt_rights, 4083 &cap_recv_rights)) { 4084 error = ENOTCAPABLE; 4085 goto bad; 4086 } 4087 error = getsock(sopt->sopt_td, splice.sp_fd, 4088 &cap_send_rights, &fp); 4089 if (error != 0) 4090 goto bad; 4091 so2 = fp->f_data; 4092 4093 error = so_splice(so, so2, &splice); 4094 fdrop(fp, sopt->sopt_td); 4095 } else { 4096 error = so_unsplice(so, false); 4097 } 4098 break; 4099 } 4100 default: 4101 #ifdef SOCKET_HHOOK 4102 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 4103 error = hhook_run_socket(so, sopt, 4104 HHOOK_SOCKET_OPT); 4105 else 4106 #endif 4107 error = ENOPROTOOPT; 4108 break; 4109 } 4110 if (error == 0) 4111 (void)so->so_proto->pr_ctloutput(so, sopt); 4112 } 4113 bad: 4114 CURVNET_RESTORE(); 4115 return (error); 4116 } 4117 4118 /* 4119 * Helper routine for getsockopt. 4120 */ 4121 int 4122 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 4123 { 4124 int error; 4125 size_t valsize; 4126 4127 error = 0; 4128 4129 /* 4130 * Documented get behavior is that we always return a value, possibly 4131 * truncated to fit in the user's buffer. Traditional behavior is 4132 * that we always tell the user precisely how much we copied, rather 4133 * than something useful like the total amount we had available for 4134 * her. Note that this interface is not idempotent; the entire 4135 * answer must be generated ahead of time. 4136 */ 4137 valsize = min(len, sopt->sopt_valsize); 4138 sopt->sopt_valsize = valsize; 4139 if (sopt->sopt_val != NULL) { 4140 if (sopt->sopt_td != NULL) 4141 error = copyout(buf, sopt->sopt_val, valsize); 4142 else 4143 memcpy_data(sopt->sopt_val, buf, valsize); 4144 } 4145 return (error); 4146 } 4147 4148 int 4149 sogetopt(struct socket *so, struct sockopt *sopt) 4150 { 4151 int error, optval; 4152 struct linger l; 4153 struct timeval tv; 4154 #ifdef MAC 4155 struct mac extmac; 4156 #endif 4157 4158 CURVNET_SET(so->so_vnet); 4159 error = 0; 4160 if (sopt->sopt_level != SOL_SOCKET) { 4161 error = so->so_proto->pr_ctloutput(so, sopt); 4162 CURVNET_RESTORE(); 4163 return (error); 4164 } else { 4165 switch (sopt->sopt_name) { 4166 case SO_ACCEPTFILTER: 4167 error = accept_filt_getopt(so, sopt); 4168 break; 4169 4170 case SO_LINGER: 4171 SOCK_LOCK(so); 4172 l.l_onoff = so->so_options & SO_LINGER; 4173 l.l_linger = so->so_linger; 4174 SOCK_UNLOCK(so); 4175 error = sooptcopyout(sopt, &l, sizeof l); 4176 break; 4177 4178 case SO_USELOOPBACK: 4179 case SO_DONTROUTE: 4180 case SO_DEBUG: 4181 case SO_KEEPALIVE: 4182 case SO_REUSEADDR: 4183 case SO_REUSEPORT: 4184 case SO_REUSEPORT_LB: 4185 case SO_BROADCAST: 4186 case SO_OOBINLINE: 4187 case SO_ACCEPTCONN: 4188 case SO_TIMESTAMP: 4189 case SO_BINTIME: 4190 case SO_NOSIGPIPE: 4191 case SO_NO_DDP: 4192 case SO_NO_OFFLOAD: 4193 case SO_RERROR: 4194 stdopt: 4195 optval = so->so_options & sopt->sopt_name; 4196 integer: 4197 error = sooptcopyout(sopt, &optval, sizeof optval); 4198 break; 4199 4200 case SO_PASSRIGHTS: 4201 if (so->so_proto->pr_domain->dom_family != AF_LOCAL) { 4202 error = EOPNOTSUPP; 4203 goto bad; 4204 } 4205 4206 goto stdopt; 4207 4208 case SO_FIB: 4209 SOCK_LOCK(so); 4210 optval = so->so_fibnum; 4211 SOCK_UNLOCK(so); 4212 goto integer; 4213 4214 case SO_DOMAIN: 4215 optval = so->so_proto->pr_domain->dom_family; 4216 goto integer; 4217 4218 case SO_TYPE: 4219 optval = so->so_type; 4220 goto integer; 4221 4222 case SO_PROTOCOL: 4223 optval = so->so_proto->pr_protocol; 4224 goto integer; 4225 4226 case SO_ERROR: 4227 SOCK_LOCK(so); 4228 if (so->so_error) { 4229 optval = so->so_error; 4230 so->so_error = 0; 4231 } else { 4232 optval = so->so_rerror; 4233 so->so_rerror = 0; 4234 } 4235 SOCK_UNLOCK(so); 4236 goto integer; 4237 4238 case SO_SNDBUF: 4239 SOCK_LOCK(so); 4240 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat : 4241 so->so_snd.sb_hiwat; 4242 SOCK_UNLOCK(so); 4243 goto integer; 4244 4245 case SO_RCVBUF: 4246 SOCK_LOCK(so); 4247 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat : 4248 so->so_rcv.sb_hiwat; 4249 SOCK_UNLOCK(so); 4250 goto integer; 4251 4252 case SO_SNDLOWAT: 4253 SOCK_LOCK(so); 4254 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat : 4255 so->so_snd.sb_lowat; 4256 SOCK_UNLOCK(so); 4257 goto integer; 4258 4259 case SO_RCVLOWAT: 4260 SOCK_LOCK(so); 4261 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat : 4262 so->so_rcv.sb_lowat; 4263 SOCK_UNLOCK(so); 4264 goto integer; 4265 4266 case SO_SNDTIMEO: 4267 case SO_RCVTIMEO: 4268 SOCK_LOCK(so); 4269 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ? 4270 (SOLISTENING(so) ? so->sol_sbsnd_timeo : 4271 so->so_snd.sb_timeo) : 4272 (SOLISTENING(so) ? so->sol_sbrcv_timeo : 4273 so->so_rcv.sb_timeo)); 4274 SOCK_UNLOCK(so); 4275 #ifdef COMPAT_FREEBSD32 4276 if (SV_CURPROC_FLAG(SV_ILP32)) { 4277 struct timeval32 tv32; 4278 4279 CP(tv, tv32, tv_sec); 4280 CP(tv, tv32, tv_usec); 4281 error = sooptcopyout(sopt, &tv32, sizeof tv32); 4282 } else 4283 #endif 4284 error = sooptcopyout(sopt, &tv, sizeof tv); 4285 break; 4286 4287 case SO_LABEL: 4288 #ifdef MAC 4289 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 4290 sizeof(extmac)); 4291 if (error) 4292 goto bad; 4293 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 4294 so, &extmac); 4295 if (error) 4296 goto bad; 4297 /* Don't copy out extmac, it is unchanged. */ 4298 #else 4299 error = EOPNOTSUPP; 4300 #endif 4301 break; 4302 4303 case SO_PEERLABEL: 4304 #ifdef MAC 4305 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 4306 sizeof(extmac)); 4307 if (error) 4308 goto bad; 4309 error = mac_getsockopt_peerlabel( 4310 sopt->sopt_td->td_ucred, so, &extmac); 4311 if (error) 4312 goto bad; 4313 /* Don't copy out extmac, it is unchanged. */ 4314 #else 4315 error = EOPNOTSUPP; 4316 #endif 4317 break; 4318 4319 case SO_LISTENQLIMIT: 4320 SOCK_LOCK(so); 4321 optval = SOLISTENING(so) ? so->sol_qlimit : 0; 4322 SOCK_UNLOCK(so); 4323 goto integer; 4324 4325 case SO_LISTENQLEN: 4326 SOCK_LOCK(so); 4327 optval = SOLISTENING(so) ? so->sol_qlen : 0; 4328 SOCK_UNLOCK(so); 4329 goto integer; 4330 4331 case SO_LISTENINCQLEN: 4332 SOCK_LOCK(so); 4333 optval = SOLISTENING(so) ? so->sol_incqlen : 0; 4334 SOCK_UNLOCK(so); 4335 goto integer; 4336 4337 case SO_TS_CLOCK: 4338 optval = so->so_ts_clock; 4339 goto integer; 4340 4341 case SO_MAX_PACING_RATE: 4342 optval = so->so_max_pacing_rate; 4343 goto integer; 4344 4345 case SO_SPLICE: { 4346 off_t n; 4347 4348 /* 4349 * Acquire the I/O lock to serialize with 4350 * so_splice_xfer(). This is not required for 4351 * correctness, but makes testing simpler: once a byte 4352 * has been transmitted to the sink and observed (e.g., 4353 * by reading from the socket to which the sink is 4354 * connected), a subsequent getsockopt(SO_SPLICE) will 4355 * return an up-to-date value. 4356 */ 4357 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT); 4358 if (error != 0) 4359 goto bad; 4360 SOCK_LOCK(so); 4361 if (SOLISTENING(so)) { 4362 n = 0; 4363 } else { 4364 n = so->so_splice_sent; 4365 } 4366 SOCK_UNLOCK(so); 4367 SOCK_IO_RECV_UNLOCK(so); 4368 error = sooptcopyout(sopt, &n, sizeof(n)); 4369 break; 4370 } 4371 4372 default: 4373 #ifdef SOCKET_HHOOK 4374 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 4375 error = hhook_run_socket(so, sopt, 4376 HHOOK_SOCKET_OPT); 4377 else 4378 #endif 4379 error = ENOPROTOOPT; 4380 break; 4381 } 4382 } 4383 bad: 4384 CURVNET_RESTORE(); 4385 return (error); 4386 } 4387 4388 int 4389 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 4390 { 4391 struct mbuf *m, *m_prev; 4392 int sopt_size = sopt->sopt_valsize; 4393 4394 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 4395 if (m == NULL) 4396 return ENOBUFS; 4397 if (sopt_size > MLEN) { 4398 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT); 4399 if ((m->m_flags & M_EXT) == 0) { 4400 m_free(m); 4401 return ENOBUFS; 4402 } 4403 m->m_len = min(MCLBYTES, sopt_size); 4404 } else { 4405 m->m_len = min(MLEN, sopt_size); 4406 } 4407 sopt_size -= m->m_len; 4408 *mp = m; 4409 m_prev = m; 4410 4411 while (sopt_size) { 4412 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 4413 if (m == NULL) { 4414 m_freem(*mp); 4415 return ENOBUFS; 4416 } 4417 if (sopt_size > MLEN) { 4418 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK : 4419 M_NOWAIT); 4420 if ((m->m_flags & M_EXT) == 0) { 4421 m_freem(m); 4422 m_freem(*mp); 4423 return ENOBUFS; 4424 } 4425 m->m_len = min(MCLBYTES, sopt_size); 4426 } else { 4427 m->m_len = min(MLEN, sopt_size); 4428 } 4429 sopt_size -= m->m_len; 4430 m_prev->m_next = m; 4431 m_prev = m; 4432 } 4433 return (0); 4434 } 4435 4436 int 4437 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 4438 { 4439 struct mbuf *m0 = m; 4440 4441 if (sopt->sopt_val == NULL) 4442 return (0); 4443 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 4444 if (sopt->sopt_td != NULL) { 4445 int error; 4446 4447 error = copyin(sopt->sopt_val, mtod(m, char *), 4448 m->m_len); 4449 if (error != 0) { 4450 m_freem(m0); 4451 return(error); 4452 } 4453 } else 4454 memcpy(mtod(m, char *), sopt->sopt_val, m->m_len); 4455 sopt->sopt_valsize -= m->m_len; 4456 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 4457 m = m->m_next; 4458 } 4459 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 4460 panic("ip6_sooptmcopyin"); 4461 return (0); 4462 } 4463 4464 int 4465 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 4466 { 4467 struct mbuf *m0 = m; 4468 size_t valsize = 0; 4469 4470 if (sopt->sopt_val == NULL) 4471 return (0); 4472 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 4473 if (sopt->sopt_td != NULL) { 4474 int error; 4475 4476 error = copyout(mtod(m, char *), sopt->sopt_val, 4477 m->m_len); 4478 if (error != 0) { 4479 m_freem(m0); 4480 return(error); 4481 } 4482 } else 4483 memcpy(sopt->sopt_val, mtod(m, char *), m->m_len); 4484 sopt->sopt_valsize -= m->m_len; 4485 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 4486 valsize += m->m_len; 4487 m = m->m_next; 4488 } 4489 if (m != NULL) { 4490 /* enough soopt buffer should be given from user-land */ 4491 m_freem(m0); 4492 return(EINVAL); 4493 } 4494 sopt->sopt_valsize = valsize; 4495 return (0); 4496 } 4497 4498 /* 4499 * sohasoutofband(): protocol notifies socket layer of the arrival of new 4500 * out-of-band data, which will then notify socket consumers. 4501 */ 4502 void 4503 sohasoutofband(struct socket *so) 4504 { 4505 4506 if (so->so_sigio != NULL) 4507 pgsigio(&so->so_sigio, SIGURG, 0); 4508 selwakeuppri(&so->so_rdsel, PSOCK); 4509 } 4510 4511 int 4512 sopoll_generic(struct socket *so, int events, struct thread *td) 4513 { 4514 int revents; 4515 4516 SOCK_LOCK(so); 4517 if (SOLISTENING(so)) { 4518 if (!(events & (POLLIN | POLLRDNORM))) 4519 revents = 0; 4520 else if (!TAILQ_EMPTY(&so->sol_comp)) 4521 revents = events & (POLLIN | POLLRDNORM); 4522 else if ((events & POLLINIGNEOF) == 0 && so->so_error) 4523 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 4524 else { 4525 selrecord(td, &so->so_rdsel); 4526 revents = 0; 4527 } 4528 } else { 4529 revents = 0; 4530 SOCK_SENDBUF_LOCK(so); 4531 SOCK_RECVBUF_LOCK(so); 4532 if (events & (POLLIN | POLLRDNORM)) 4533 if (soreadabledata(so) && !isspliced(so)) 4534 revents |= events & (POLLIN | POLLRDNORM); 4535 if (events & (POLLOUT | POLLWRNORM)) 4536 if (sowriteable(so) && !issplicedback(so)) 4537 revents |= events & (POLLOUT | POLLWRNORM); 4538 if (events & (POLLPRI | POLLRDBAND)) 4539 if (so->so_oobmark || 4540 (so->so_rcv.sb_state & SBS_RCVATMARK)) 4541 revents |= events & (POLLPRI | POLLRDBAND); 4542 if ((events & POLLINIGNEOF) == 0) { 4543 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 4544 revents |= events & (POLLIN | POLLRDNORM); 4545 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 4546 revents |= POLLHUP; 4547 } 4548 } 4549 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 4550 revents |= events & POLLRDHUP; 4551 if (revents == 0) { 4552 if (events & 4553 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND | POLLRDHUP)) { 4554 selrecord(td, &so->so_rdsel); 4555 so->so_rcv.sb_flags |= SB_SEL; 4556 } 4557 if (events & (POLLOUT | POLLWRNORM)) { 4558 selrecord(td, &so->so_wrsel); 4559 so->so_snd.sb_flags |= SB_SEL; 4560 } 4561 } 4562 SOCK_RECVBUF_UNLOCK(so); 4563 SOCK_SENDBUF_UNLOCK(so); 4564 } 4565 SOCK_UNLOCK(so); 4566 return (revents); 4567 } 4568 4569 int 4570 sokqfilter_generic(struct socket *so, struct knote *kn) 4571 { 4572 struct sockbuf *sb; 4573 sb_which which; 4574 struct knlist *knl; 4575 4576 switch (kn->kn_filter) { 4577 case EVFILT_READ: 4578 kn->kn_fop = &soread_filtops; 4579 knl = &so->so_rdsel.si_note; 4580 sb = &so->so_rcv; 4581 which = SO_RCV; 4582 break; 4583 case EVFILT_WRITE: 4584 kn->kn_fop = &sowrite_filtops; 4585 knl = &so->so_wrsel.si_note; 4586 sb = &so->so_snd; 4587 which = SO_SND; 4588 break; 4589 case EVFILT_EMPTY: 4590 kn->kn_fop = &soempty_filtops; 4591 knl = &so->so_wrsel.si_note; 4592 sb = &so->so_snd; 4593 which = SO_SND; 4594 break; 4595 default: 4596 return (EINVAL); 4597 } 4598 4599 SOCK_LOCK(so); 4600 if (SOLISTENING(so)) { 4601 knlist_add(knl, kn, 1); 4602 } else { 4603 SOCK_BUF_LOCK(so, which); 4604 knlist_add(knl, kn, 1); 4605 sb->sb_flags |= SB_KNOTE; 4606 if ((kn->kn_sfflags & NOTE_LOWAT) && 4607 (sb->sb_flags & SB_AUTOLOWAT)) 4608 sb->sb_flags &= ~SB_AUTOLOWAT; 4609 SOCK_BUF_UNLOCK(so, which); 4610 } 4611 SOCK_UNLOCK(so); 4612 return (0); 4613 } 4614 4615 static void 4616 filt_sordetach(struct knote *kn) 4617 { 4618 struct socket *so = kn->kn_fp->f_data; 4619 4620 so_rdknl_lock(so); 4621 knlist_remove(&so->so_rdsel.si_note, kn, 1); 4622 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note)) 4623 so->so_rcv.sb_flags &= ~SB_KNOTE; 4624 so_rdknl_unlock(so); 4625 } 4626 4627 /*ARGSUSED*/ 4628 static int 4629 filt_soread(struct knote *kn, long hint) 4630 { 4631 struct socket *so; 4632 4633 so = kn->kn_fp->f_data; 4634 4635 if (SOLISTENING(so)) { 4636 SOCK_LOCK_ASSERT(so); 4637 kn->kn_data = so->sol_qlen; 4638 if (so->so_error) { 4639 kn->kn_flags |= EV_EOF; 4640 kn->kn_fflags = so->so_error; 4641 return (1); 4642 } 4643 return (!TAILQ_EMPTY(&so->sol_comp)); 4644 } 4645 4646 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) 4647 return (0); 4648 4649 SOCK_RECVBUF_LOCK_ASSERT(so); 4650 4651 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl; 4652 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 4653 kn->kn_flags |= EV_EOF; 4654 kn->kn_fflags = so->so_error; 4655 return (1); 4656 } else if (so->so_error || so->so_rerror) 4657 return (1); 4658 4659 if (kn->kn_sfflags & NOTE_LOWAT) { 4660 if (kn->kn_data >= kn->kn_sdata) 4661 return (1); 4662 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat) 4663 return (1); 4664 4665 #ifdef SOCKET_HHOOK 4666 /* This hook returning non-zero indicates an event, not error */ 4667 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD)); 4668 #else 4669 return (0); 4670 #endif 4671 } 4672 4673 static void 4674 filt_sowdetach(struct knote *kn) 4675 { 4676 struct socket *so = kn->kn_fp->f_data; 4677 4678 so_wrknl_lock(so); 4679 knlist_remove(&so->so_wrsel.si_note, kn, 1); 4680 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note)) 4681 so->so_snd.sb_flags &= ~SB_KNOTE; 4682 so_wrknl_unlock(so); 4683 } 4684 4685 /*ARGSUSED*/ 4686 static int 4687 filt_sowrite(struct knote *kn, long hint) 4688 { 4689 struct socket *so; 4690 4691 so = kn->kn_fp->f_data; 4692 4693 if (SOLISTENING(so)) 4694 return (0); 4695 4696 SOCK_SENDBUF_LOCK_ASSERT(so); 4697 kn->kn_data = sbspace(&so->so_snd); 4698 4699 #ifdef SOCKET_HHOOK 4700 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE); 4701 #endif 4702 4703 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 4704 kn->kn_flags |= EV_EOF; 4705 kn->kn_fflags = so->so_error; 4706 return (1); 4707 } else if (so->so_error) /* temporary udp error */ 4708 return (1); 4709 else if (((so->so_state & SS_ISCONNECTED) == 0) && 4710 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 4711 return (0); 4712 else if (kn->kn_sfflags & NOTE_LOWAT) 4713 return (kn->kn_data >= kn->kn_sdata); 4714 else 4715 return (kn->kn_data >= so->so_snd.sb_lowat); 4716 } 4717 4718 static int 4719 filt_soempty(struct knote *kn, long hint) 4720 { 4721 struct socket *so; 4722 4723 so = kn->kn_fp->f_data; 4724 4725 if (SOLISTENING(so)) 4726 return (1); 4727 4728 SOCK_SENDBUF_LOCK_ASSERT(so); 4729 kn->kn_data = sbused(&so->so_snd); 4730 4731 if (kn->kn_data == 0) 4732 return (1); 4733 else 4734 return (0); 4735 } 4736 4737 int 4738 socheckuid(struct socket *so, uid_t uid) 4739 { 4740 4741 if (so == NULL) 4742 return (EPERM); 4743 if (so->so_cred->cr_uid != uid) 4744 return (EPERM); 4745 return (0); 4746 } 4747 4748 /* 4749 * These functions are used by protocols to notify the socket layer (and its 4750 * consumers) of state changes in the sockets driven by protocol-side events. 4751 */ 4752 4753 /* 4754 * Procedures to manipulate state flags of socket and do appropriate wakeups. 4755 * 4756 * Normal sequence from the active (originating) side is that 4757 * soisconnecting() is called during processing of connect() call, resulting 4758 * in an eventual call to soisconnected() if/when the connection is 4759 * established. When the connection is torn down soisdisconnecting() is 4760 * called during processing of disconnect() call, and soisdisconnected() is 4761 * called when the connection to the peer is totally severed. The semantics 4762 * of these routines are such that connectionless protocols can call 4763 * soisconnected() and soisdisconnected() only, bypassing the in-progress 4764 * calls when setting up a ``connection'' takes no time. 4765 * 4766 * From the passive side, a socket is created with two queues of sockets: 4767 * so_incomp for connections in progress and so_comp for connections already 4768 * made and awaiting user acceptance. As a protocol is preparing incoming 4769 * connections, it creates a socket structure queued on so_incomp by calling 4770 * sonewconn(). When the connection is established, soisconnected() is 4771 * called, and transfers the socket structure to so_comp, making it available 4772 * to accept(). 4773 * 4774 * If a socket is closed with sockets on either so_incomp or so_comp, these 4775 * sockets are dropped. 4776 * 4777 * If higher-level protocols are implemented in the kernel, the wakeups done 4778 * here will sometimes cause software-interrupt process scheduling. 4779 */ 4780 void 4781 soisconnecting(struct socket *so) 4782 { 4783 4784 SOCK_LOCK(so); 4785 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 4786 so->so_state |= SS_ISCONNECTING; 4787 SOCK_UNLOCK(so); 4788 } 4789 4790 void 4791 soisconnected(struct socket *so) 4792 { 4793 bool last __diagused; 4794 4795 SOCK_LOCK(so); 4796 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING); 4797 so->so_state |= SS_ISCONNECTED; 4798 4799 if (so->so_qstate == SQ_INCOMP) { 4800 struct socket *head = so->so_listen; 4801 int ret; 4802 4803 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so)); 4804 /* 4805 * Promoting a socket from incomplete queue to complete, we 4806 * need to go through reverse order of locking. We first do 4807 * trylock, and if that doesn't succeed, we go the hard way 4808 * leaving a reference and rechecking consistency after proper 4809 * locking. 4810 */ 4811 if (__predict_false(SOLISTEN_TRYLOCK(head) == 0)) { 4812 soref(head); 4813 SOCK_UNLOCK(so); 4814 SOLISTEN_LOCK(head); 4815 SOCK_LOCK(so); 4816 if (__predict_false(head != so->so_listen)) { 4817 /* 4818 * The socket went off the listen queue, 4819 * should be lost race to close(2) of sol. 4820 * The socket is about to soabort(). 4821 */ 4822 SOCK_UNLOCK(so); 4823 sorele_locked(head); 4824 return; 4825 } 4826 last = refcount_release(&head->so_count); 4827 KASSERT(!last, ("%s: released last reference for %p", 4828 __func__, head)); 4829 } 4830 again: 4831 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 4832 TAILQ_REMOVE(&head->sol_incomp, so, so_list); 4833 head->sol_incqlen--; 4834 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 4835 head->sol_qlen++; 4836 so->so_qstate = SQ_COMP; 4837 SOCK_UNLOCK(so); 4838 solisten_wakeup(head); /* unlocks */ 4839 } else { 4840 SOCK_RECVBUF_LOCK(so); 4841 soupcall_set(so, SO_RCV, 4842 head->sol_accept_filter->accf_callback, 4843 head->sol_accept_filter_arg); 4844 so->so_options &= ~SO_ACCEPTFILTER; 4845 ret = head->sol_accept_filter->accf_callback(so, 4846 head->sol_accept_filter_arg, M_NOWAIT); 4847 if (ret == SU_ISCONNECTED) { 4848 soupcall_clear(so, SO_RCV); 4849 SOCK_RECVBUF_UNLOCK(so); 4850 goto again; 4851 } 4852 SOCK_RECVBUF_UNLOCK(so); 4853 SOCK_UNLOCK(so); 4854 SOLISTEN_UNLOCK(head); 4855 } 4856 return; 4857 } 4858 SOCK_UNLOCK(so); 4859 wakeup(&so->so_timeo); 4860 sorwakeup(so); 4861 sowwakeup(so); 4862 } 4863 4864 void 4865 soisdisconnecting(struct socket *so) 4866 { 4867 4868 SOCK_LOCK(so); 4869 so->so_state &= ~SS_ISCONNECTING; 4870 so->so_state |= SS_ISDISCONNECTING; 4871 4872 if (!SOLISTENING(so)) { 4873 SOCK_RECVBUF_LOCK(so); 4874 socantrcvmore_locked(so); 4875 SOCK_SENDBUF_LOCK(so); 4876 socantsendmore_locked(so); 4877 } 4878 SOCK_UNLOCK(so); 4879 wakeup(&so->so_timeo); 4880 } 4881 4882 void 4883 soisdisconnected(struct socket *so) 4884 { 4885 4886 SOCK_LOCK(so); 4887 4888 /* 4889 * There is at least one reader of so_state that does not 4890 * acquire socket lock, namely soreceive_generic(). Ensure 4891 * that it never sees all flags that track connection status 4892 * cleared, by ordering the update with a barrier semantic of 4893 * our release thread fence. 4894 */ 4895 so->so_state |= SS_ISDISCONNECTED; 4896 atomic_thread_fence_rel(); 4897 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 4898 4899 if (!SOLISTENING(so)) { 4900 SOCK_UNLOCK(so); 4901 SOCK_RECVBUF_LOCK(so); 4902 socantrcvmore_locked(so); 4903 SOCK_SENDBUF_LOCK(so); 4904 sbdrop_locked(&so->so_snd, sbused(&so->so_snd)); 4905 socantsendmore_locked(so); 4906 } else 4907 SOCK_UNLOCK(so); 4908 wakeup(&so->so_timeo); 4909 } 4910 4911 int 4912 soiolock(struct socket *so, struct sx *sx, int flags) 4913 { 4914 int error; 4915 4916 KASSERT((flags & SBL_VALID) == flags, 4917 ("soiolock: invalid flags %#x", flags)); 4918 4919 if ((flags & SBL_WAIT) != 0) { 4920 if ((flags & SBL_NOINTR) != 0) { 4921 sx_xlock(sx); 4922 } else { 4923 error = sx_xlock_sig(sx); 4924 if (error != 0) 4925 return (error); 4926 } 4927 } else if (!sx_try_xlock(sx)) { 4928 return (EWOULDBLOCK); 4929 } 4930 4931 if (__predict_false(SOLISTENING(so))) { 4932 sx_xunlock(sx); 4933 return (ENOTCONN); 4934 } 4935 return (0); 4936 } 4937 4938 void 4939 soiounlock(struct sx *sx) 4940 { 4941 sx_xunlock(sx); 4942 } 4943 4944 /* 4945 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME. 4946 */ 4947 struct sockaddr * 4948 sodupsockaddr(const struct sockaddr *sa, int mflags) 4949 { 4950 struct sockaddr *sa2; 4951 4952 sa2 = malloc(sa->sa_len, M_SONAME, mflags); 4953 if (sa2) 4954 memcpy(sa2, sa, sa->sa_len); 4955 return sa2; 4956 } 4957 4958 /* 4959 * Register per-socket destructor. 4960 */ 4961 void 4962 sodtor_set(struct socket *so, so_dtor_t *func) 4963 { 4964 4965 SOCK_LOCK_ASSERT(so); 4966 so->so_dtor = func; 4967 } 4968 4969 /* 4970 * Register per-socket buffer upcalls. 4971 */ 4972 void 4973 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg) 4974 { 4975 struct sockbuf *sb; 4976 4977 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 4978 4979 switch (which) { 4980 case SO_RCV: 4981 sb = &so->so_rcv; 4982 break; 4983 case SO_SND: 4984 sb = &so->so_snd; 4985 break; 4986 } 4987 SOCK_BUF_LOCK_ASSERT(so, which); 4988 sb->sb_upcall = func; 4989 sb->sb_upcallarg = arg; 4990 sb->sb_flags |= SB_UPCALL; 4991 } 4992 4993 void 4994 soupcall_clear(struct socket *so, sb_which which) 4995 { 4996 struct sockbuf *sb; 4997 4998 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 4999 5000 switch (which) { 5001 case SO_RCV: 5002 sb = &so->so_rcv; 5003 break; 5004 case SO_SND: 5005 sb = &so->so_snd; 5006 break; 5007 } 5008 SOCK_BUF_LOCK_ASSERT(so, which); 5009 KASSERT(sb->sb_upcall != NULL, 5010 ("%s: so %p no upcall to clear", __func__, so)); 5011 sb->sb_upcall = NULL; 5012 sb->sb_upcallarg = NULL; 5013 sb->sb_flags &= ~SB_UPCALL; 5014 } 5015 5016 void 5017 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) 5018 { 5019 5020 SOLISTEN_LOCK_ASSERT(so); 5021 so->sol_upcall = func; 5022 so->sol_upcallarg = arg; 5023 } 5024 5025 static void 5026 so_rdknl_lock(void *arg) 5027 { 5028 struct socket *so = arg; 5029 5030 retry: 5031 if (SOLISTENING(so)) { 5032 SOLISTEN_LOCK(so); 5033 } else { 5034 SOCK_RECVBUF_LOCK(so); 5035 if (__predict_false(SOLISTENING(so))) { 5036 SOCK_RECVBUF_UNLOCK(so); 5037 goto retry; 5038 } 5039 } 5040 } 5041 5042 static void 5043 so_rdknl_unlock(void *arg) 5044 { 5045 struct socket *so = arg; 5046 5047 if (SOLISTENING(so)) 5048 SOLISTEN_UNLOCK(so); 5049 else 5050 SOCK_RECVBUF_UNLOCK(so); 5051 } 5052 5053 static void 5054 so_rdknl_assert_lock(void *arg, int what) 5055 { 5056 struct socket *so = arg; 5057 5058 if (what == LA_LOCKED) { 5059 if (SOLISTENING(so)) 5060 SOLISTEN_LOCK_ASSERT(so); 5061 else 5062 SOCK_RECVBUF_LOCK_ASSERT(so); 5063 } else { 5064 if (SOLISTENING(so)) 5065 SOLISTEN_UNLOCK_ASSERT(so); 5066 else 5067 SOCK_RECVBUF_UNLOCK_ASSERT(so); 5068 } 5069 } 5070 5071 static void 5072 so_wrknl_lock(void *arg) 5073 { 5074 struct socket *so = arg; 5075 5076 retry: 5077 if (SOLISTENING(so)) { 5078 SOLISTEN_LOCK(so); 5079 } else { 5080 SOCK_SENDBUF_LOCK(so); 5081 if (__predict_false(SOLISTENING(so))) { 5082 SOCK_SENDBUF_UNLOCK(so); 5083 goto retry; 5084 } 5085 } 5086 } 5087 5088 static void 5089 so_wrknl_unlock(void *arg) 5090 { 5091 struct socket *so = arg; 5092 5093 if (SOLISTENING(so)) 5094 SOLISTEN_UNLOCK(so); 5095 else 5096 SOCK_SENDBUF_UNLOCK(so); 5097 } 5098 5099 static void 5100 so_wrknl_assert_lock(void *arg, int what) 5101 { 5102 struct socket *so = arg; 5103 5104 if (what == LA_LOCKED) { 5105 if (SOLISTENING(so)) 5106 SOLISTEN_LOCK_ASSERT(so); 5107 else 5108 SOCK_SENDBUF_LOCK_ASSERT(so); 5109 } else { 5110 if (SOLISTENING(so)) 5111 SOLISTEN_UNLOCK_ASSERT(so); 5112 else 5113 SOCK_SENDBUF_UNLOCK_ASSERT(so); 5114 } 5115 } 5116 5117 /* 5118 * Create an external-format (``xsocket'') structure using the information in 5119 * the kernel-format socket structure pointed to by so. This is done to 5120 * reduce the spew of irrelevant information over this interface, to isolate 5121 * user code from changes in the kernel structure, and potentially to provide 5122 * information-hiding if we decide that some of this information should be 5123 * hidden from users. 5124 */ 5125 void 5126 sotoxsocket(struct socket *so, struct xsocket *xso) 5127 { 5128 5129 bzero(xso, sizeof(*xso)); 5130 xso->xso_len = sizeof *xso; 5131 xso->xso_so = (uintptr_t)so; 5132 xso->so_type = so->so_type; 5133 xso->so_options = so->so_options; 5134 xso->so_linger = so->so_linger; 5135 xso->so_state = so->so_state; 5136 xso->so_pcb = (uintptr_t)so->so_pcb; 5137 xso->xso_protocol = so->so_proto->pr_protocol; 5138 xso->xso_family = so->so_proto->pr_domain->dom_family; 5139 xso->so_timeo = so->so_timeo; 5140 xso->so_error = so->so_error; 5141 xso->so_uid = so->so_cred->cr_uid; 5142 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0; 5143 SOCK_LOCK(so); 5144 xso->so_fibnum = so->so_fibnum; 5145 if (SOLISTENING(so)) { 5146 xso->so_qlen = so->sol_qlen; 5147 xso->so_incqlen = so->sol_incqlen; 5148 xso->so_qlimit = so->sol_qlimit; 5149 xso->so_oobmark = 0; 5150 } else { 5151 xso->so_state |= so->so_qstate; 5152 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0; 5153 xso->so_oobmark = so->so_oobmark; 5154 sbtoxsockbuf(&so->so_snd, &xso->so_snd); 5155 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv); 5156 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) 5157 xso->so_splice_so = (uintptr_t)so->so_splice->dst; 5158 } 5159 SOCK_UNLOCK(so); 5160 } 5161 5162 int 5163 so_options_get(const struct socket *so) 5164 { 5165 5166 return (so->so_options); 5167 } 5168 5169 void 5170 so_options_set(struct socket *so, int val) 5171 { 5172 5173 so->so_options = val; 5174 } 5175 5176 int 5177 so_error_get(const struct socket *so) 5178 { 5179 5180 return (so->so_error); 5181 } 5182 5183 void 5184 so_error_set(struct socket *so, int val) 5185 { 5186 5187 so->so_error = val; 5188 } 5189