1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1988, 1990, 1993 5 * The Regents of the University of California. 6 * Copyright (c) 2004 The FreeBSD Foundation 7 * Copyright (c) 2004-2008 Robert N. M. Watson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pru_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pru_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pru_attach() has 50 * been successfully called. If pru_attach() returned an error, 51 * pru_detach() will not be called. Socket layer private. 52 * 53 * pru_abort() and pru_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pru_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. 58 * 59 * socreate() creates a socket and attaches protocol state. This is a public 60 * interface that may be used by socket layer consumers to create new 61 * sockets. 62 * 63 * sonewconn() creates a socket and attaches protocol state. This is a 64 * public interface that may be used by protocols to create new sockets when 65 * a new connection is received and will be available for accept() on a 66 * listen socket. 67 * 68 * soclose() destroys a socket after possibly waiting for it to disconnect. 69 * This is a public interface that socket consumers should use to close and 70 * release a socket when done with it. 71 * 72 * soabort() destroys a socket without waiting for it to disconnect (used 73 * only for incoming connections that are already partially or fully 74 * connected). This is used internally by the socket layer when clearing 75 * listen socket queues (due to overflow or close on the listen socket), but 76 * is also a public interface protocols may use to abort connections in 77 * their incomplete listen queues should they no longer be required. Sockets 78 * placed in completed connection listen queues should not be aborted for 79 * reasons described in the comment above the soclose() implementation. This 80 * is not a general purpose close routine, and except in the specific 81 * circumstances described here, should not be used. 82 * 83 * sofree() will free a socket and its protocol state if all references on 84 * the socket have been released, and is the public interface to attempt to 85 * free a socket when a reference is removed. This is a socket layer private 86 * interface. 87 * 88 * NOTE: In addition to socreate() and soclose(), which provide a single 89 * socket reference to the consumer to be managed as required, there are two 90 * calls to explicitly manage socket references, soref(), and sorele(). 91 * Currently, these are generally required only when transitioning a socket 92 * from a listen queue to a file descriptor, in order to prevent garbage 93 * collection of the socket at an untimely moment. For a number of reasons, 94 * these interfaces are not preferred, and should be avoided. 95 * 96 * NOTE: With regard to VNETs the general rule is that callers do not set 97 * curvnet. Exceptions to this rule include soabort(), sodisconnect(), 98 * sofree(), sorele(), sonewconn() and sorflush(), which are usually called 99 * from a pre-set VNET context. sopoll_generic() currently does not need a 100 * VNET context to be set. 101 */ 102 103 #include <sys/cdefs.h> 104 #include "opt_inet.h" 105 #include "opt_inet6.h" 106 #include "opt_kern_tls.h" 107 #include "opt_ktrace.h" 108 #include "opt_sctp.h" 109 110 #include <sys/param.h> 111 #include <sys/systm.h> 112 #include <sys/capsicum.h> 113 #include <sys/fcntl.h> 114 #include <sys/limits.h> 115 #include <sys/lock.h> 116 #include <sys/mac.h> 117 #include <sys/malloc.h> 118 #include <sys/mbuf.h> 119 #include <sys/mutex.h> 120 #include <sys/domain.h> 121 #include <sys/file.h> /* for struct knote */ 122 #include <sys/hhook.h> 123 #include <sys/kernel.h> 124 #include <sys/khelp.h> 125 #include <sys/kthread.h> 126 #include <sys/ktls.h> 127 #include <sys/event.h> 128 #include <sys/eventhandler.h> 129 #include <sys/poll.h> 130 #include <sys/proc.h> 131 #include <sys/protosw.h> 132 #include <sys/sbuf.h> 133 #include <sys/socket.h> 134 #include <sys/socketvar.h> 135 #include <sys/resourcevar.h> 136 #include <net/route.h> 137 #include <sys/sched.h> 138 #include <sys/signalvar.h> 139 #include <sys/smp.h> 140 #include <sys/stat.h> 141 #include <sys/sx.h> 142 #include <sys/sysctl.h> 143 #include <sys/taskqueue.h> 144 #include <sys/uio.h> 145 #include <sys/un.h> 146 #include <sys/unpcb.h> 147 #include <sys/jail.h> 148 #include <sys/syslog.h> 149 #include <netinet/in.h> 150 #include <netinet/in_pcb.h> 151 #include <netinet/tcp.h> 152 153 #include <net/vnet.h> 154 155 #include <security/mac/mac_framework.h> 156 #include <security/mac/mac_internal.h> 157 158 #include <vm/uma.h> 159 160 #ifdef COMPAT_FREEBSD32 161 #include <sys/mount.h> 162 #include <sys/sysent.h> 163 #include <compat/freebsd32/freebsd32.h> 164 #endif 165 166 static int soreceive_generic_locked(struct socket *so, 167 struct sockaddr **psa, struct uio *uio, struct mbuf **mp, 168 struct mbuf **controlp, int *flagsp); 169 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 170 int flags); 171 static int soreceive_stream_locked(struct socket *so, struct sockbuf *sb, 172 struct sockaddr **psa, struct uio *uio, struct mbuf **mp, 173 struct mbuf **controlp, int flags); 174 static int sosend_generic_locked(struct socket *so, struct sockaddr *addr, 175 struct uio *uio, struct mbuf *top, struct mbuf *control, 176 int flags, struct thread *td); 177 static void so_rdknl_lock(void *); 178 static void so_rdknl_unlock(void *); 179 static void so_rdknl_assert_lock(void *, int); 180 static void so_wrknl_lock(void *); 181 static void so_wrknl_unlock(void *); 182 static void so_wrknl_assert_lock(void *, int); 183 184 static void filt_sordetach(struct knote *kn); 185 static int filt_soread(struct knote *kn, long hint); 186 static void filt_sowdetach(struct knote *kn); 187 static int filt_sowrite(struct knote *kn, long hint); 188 static int filt_soempty(struct knote *kn, long hint); 189 fo_kqfilter_t soo_kqfilter; 190 191 static const struct filterops soread_filtops = { 192 .f_isfd = 1, 193 .f_detach = filt_sordetach, 194 .f_event = filt_soread, 195 }; 196 static const struct filterops sowrite_filtops = { 197 .f_isfd = 1, 198 .f_detach = filt_sowdetach, 199 .f_event = filt_sowrite, 200 }; 201 static const struct filterops soempty_filtops = { 202 .f_isfd = 1, 203 .f_detach = filt_sowdetach, 204 .f_event = filt_soempty, 205 }; 206 207 so_gen_t so_gencnt; /* generation count for sockets */ 208 209 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 210 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 211 212 #define VNET_SO_ASSERT(so) \ 213 VNET_ASSERT(curvnet != NULL, \ 214 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so))); 215 216 #ifdef SOCKET_HHOOK 217 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]); 218 #define V_socket_hhh VNET(socket_hhh) 219 static inline int hhook_run_socket(struct socket *, void *, int32_t); 220 #endif 221 222 #ifdef COMPAT_FREEBSD32 223 #ifdef __amd64__ 224 /* off_t has 4-byte alignment on i386 but not on other 32-bit platforms. */ 225 #define __splice32_packed __packed 226 #else 227 #define __splice32_packed 228 #endif 229 struct splice32 { 230 int32_t sp_fd; 231 int64_t sp_max; 232 struct timeval32 sp_idle; 233 } __splice32_packed; 234 #undef __splice32_packed 235 #endif 236 237 /* 238 * Limit on the number of connections in the listen queue waiting 239 * for accept(2). 240 * NB: The original sysctl somaxconn is still available but hidden 241 * to prevent confusion about the actual purpose of this number. 242 */ 243 VNET_DEFINE_STATIC(u_int, somaxconn) = SOMAXCONN; 244 #define V_somaxconn VNET(somaxconn) 245 246 static int 247 sysctl_somaxconn(SYSCTL_HANDLER_ARGS) 248 { 249 int error; 250 u_int val; 251 252 val = V_somaxconn; 253 error = sysctl_handle_int(oidp, &val, 0, req); 254 if (error || !req->newptr ) 255 return (error); 256 257 /* 258 * The purpose of the UINT_MAX / 3 limit, is so that the formula 259 * 3 * sol_qlimit / 2 260 * below, will not overflow. 261 */ 262 263 if (val < 1 || val > UINT_MAX / 3) 264 return (EINVAL); 265 266 V_somaxconn = val; 267 return (0); 268 } 269 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue, 270 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, sizeof(u_int), 271 sysctl_somaxconn, "IU", 272 "Maximum listen socket pending connection accept queue size"); 273 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, 274 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, 275 sizeof(u_int), sysctl_somaxconn, "IU", 276 "Maximum listen socket pending connection accept queue size (compat)"); 277 278 static u_int numopensockets; 279 static int 280 sysctl_numopensockets(SYSCTL_HANDLER_ARGS) 281 { 282 u_int val; 283 284 #ifdef VIMAGE 285 if(!IS_DEFAULT_VNET(curvnet)) 286 val = curvnet->vnet_sockcnt; 287 else 288 #endif 289 val = numopensockets; 290 return (sysctl_handle_int(oidp, &val, 0, req)); 291 } 292 SYSCTL_PROC(_kern_ipc, OID_AUTO, numopensockets, 293 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, sizeof(u_int), 294 sysctl_numopensockets, "IU", "Number of open sockets"); 295 296 /* 297 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 298 * so_gencnt field. 299 */ 300 static struct mtx so_global_mtx; 301 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 302 303 /* 304 * General IPC sysctl name space, used by sockets and a variety of other IPC 305 * types. 306 */ 307 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 308 "IPC"); 309 310 /* 311 * Initialize the socket subsystem and set up the socket 312 * memory allocator. 313 */ 314 static uma_zone_t socket_zone; 315 int maxsockets; 316 317 static void 318 socket_zone_change(void *tag) 319 { 320 321 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 322 } 323 324 static int splice_init_state; 325 static struct sx splice_init_lock; 326 SX_SYSINIT(splice_init_lock, &splice_init_lock, "splice_init"); 327 328 static SYSCTL_NODE(_kern_ipc, OID_AUTO, splice, CTLFLAG_RW, 0, 329 "Settings relating to the SO_SPLICE socket option"); 330 331 static bool splice_receive_stream = true; 332 SYSCTL_BOOL(_kern_ipc_splice, OID_AUTO, receive_stream, CTLFLAG_RWTUN, 333 &splice_receive_stream, 0, 334 "Use soreceive_stream() for stream splices"); 335 336 static uma_zone_t splice_zone; 337 static struct proc *splice_proc; 338 struct splice_wq { 339 struct mtx mtx; 340 STAILQ_HEAD(, so_splice) head; 341 bool running; 342 } __aligned(CACHE_LINE_SIZE); 343 static struct splice_wq *splice_wq; 344 static uint32_t splice_index = 0; 345 346 static void so_splice_timeout(void *arg, int pending); 347 static void so_splice_xfer(struct so_splice *s); 348 static int so_unsplice(struct socket *so, bool timeout); 349 350 static void 351 splice_work_thread(void *ctx) 352 { 353 struct splice_wq *wq = ctx; 354 struct so_splice *s, *s_temp; 355 STAILQ_HEAD(, so_splice) local_head; 356 int cpu; 357 358 cpu = wq - splice_wq; 359 if (bootverbose) 360 printf("starting so_splice worker thread for CPU %d\n", cpu); 361 362 for (;;) { 363 mtx_lock(&wq->mtx); 364 while (STAILQ_EMPTY(&wq->head)) { 365 wq->running = false; 366 mtx_sleep(wq, &wq->mtx, 0, "-", 0); 367 wq->running = true; 368 } 369 STAILQ_INIT(&local_head); 370 STAILQ_CONCAT(&local_head, &wq->head); 371 STAILQ_INIT(&wq->head); 372 mtx_unlock(&wq->mtx); 373 STAILQ_FOREACH_SAFE(s, &local_head, next, s_temp) { 374 mtx_lock(&s->mtx); 375 CURVNET_SET(s->src->so_vnet); 376 so_splice_xfer(s); 377 CURVNET_RESTORE(); 378 } 379 } 380 } 381 382 static void 383 so_splice_dispatch_async(struct so_splice *sp) 384 { 385 struct splice_wq *wq; 386 bool running; 387 388 wq = &splice_wq[sp->wq_index]; 389 mtx_lock(&wq->mtx); 390 STAILQ_INSERT_TAIL(&wq->head, sp, next); 391 running = wq->running; 392 mtx_unlock(&wq->mtx); 393 if (!running) 394 wakeup(wq); 395 } 396 397 void 398 so_splice_dispatch(struct so_splice *sp) 399 { 400 mtx_assert(&sp->mtx, MA_OWNED); 401 402 if (sp->state != SPLICE_IDLE) { 403 mtx_unlock(&sp->mtx); 404 } else { 405 sp->state = SPLICE_QUEUED; 406 mtx_unlock(&sp->mtx); 407 so_splice_dispatch_async(sp); 408 } 409 } 410 411 static int 412 splice_zinit(void *mem, int size __unused, int flags __unused) 413 { 414 struct so_splice *s; 415 416 s = (struct so_splice *)mem; 417 mtx_init(&s->mtx, "so_splice", NULL, MTX_DEF); 418 return (0); 419 } 420 421 static void 422 splice_zfini(void *mem, int size) 423 { 424 struct so_splice *s; 425 426 s = (struct so_splice *)mem; 427 mtx_destroy(&s->mtx); 428 } 429 430 static int 431 splice_init(void) 432 { 433 struct thread *td; 434 int error, i, state; 435 436 state = atomic_load_acq_int(&splice_init_state); 437 if (__predict_true(state > 0)) 438 return (0); 439 if (state < 0) 440 return (ENXIO); 441 sx_xlock(&splice_init_lock); 442 if (splice_init_state != 0) { 443 sx_xunlock(&splice_init_lock); 444 return (0); 445 } 446 447 splice_zone = uma_zcreate("splice", sizeof(struct so_splice), NULL, 448 NULL, splice_zinit, splice_zfini, UMA_ALIGN_CACHE, 0); 449 450 splice_wq = mallocarray(mp_maxid + 1, sizeof(*splice_wq), M_TEMP, 451 M_WAITOK | M_ZERO); 452 453 /* 454 * Initialize the workqueues to run the splice work. We create a 455 * work queue for each CPU. 456 */ 457 CPU_FOREACH(i) { 458 STAILQ_INIT(&splice_wq[i].head); 459 mtx_init(&splice_wq[i].mtx, "splice work queue", NULL, MTX_DEF); 460 } 461 462 /* Start kthreads for each workqueue. */ 463 error = 0; 464 CPU_FOREACH(i) { 465 error = kproc_kthread_add(splice_work_thread, &splice_wq[i], 466 &splice_proc, &td, 0, 0, "so_splice", "thr_%d", i); 467 if (error) { 468 printf("Can't add so_splice thread %d error %d\n", 469 i, error); 470 break; 471 } 472 473 /* 474 * It's possible to create loops with SO_SPLICE; ensure that 475 * worker threads aren't able to starve the system too easily. 476 */ 477 thread_lock(td); 478 sched_prio(td, PUSER); 479 thread_unlock(td); 480 } 481 482 splice_init_state = error != 0 ? -1 : 1; 483 sx_xunlock(&splice_init_lock); 484 485 return (error); 486 } 487 488 /* 489 * Lock a pair of socket's I/O locks for splicing. Avoid blocking while holding 490 * one lock in order to avoid potential deadlocks in case there is some other 491 * code path which acquires more than one I/O lock at a time. 492 */ 493 static void 494 splice_lock_pair(struct socket *so_src, struct socket *so_dst) 495 { 496 int error; 497 498 for (;;) { 499 error = SOCK_IO_SEND_LOCK(so_dst, SBL_WAIT | SBL_NOINTR); 500 KASSERT(error == 0, 501 ("%s: failed to lock send I/O lock: %d", __func__, error)); 502 error = SOCK_IO_RECV_LOCK(so_src, 0); 503 KASSERT(error == 0 || error == EWOULDBLOCK, 504 ("%s: failed to lock recv I/O lock: %d", __func__, error)); 505 if (error == 0) 506 break; 507 SOCK_IO_SEND_UNLOCK(so_dst); 508 509 error = SOCK_IO_RECV_LOCK(so_src, SBL_WAIT | SBL_NOINTR); 510 KASSERT(error == 0, 511 ("%s: failed to lock recv I/O lock: %d", __func__, error)); 512 error = SOCK_IO_SEND_LOCK(so_dst, 0); 513 KASSERT(error == 0 || error == EWOULDBLOCK, 514 ("%s: failed to lock send I/O lock: %d", __func__, error)); 515 if (error == 0) 516 break; 517 SOCK_IO_RECV_UNLOCK(so_src); 518 } 519 } 520 521 static void 522 splice_unlock_pair(struct socket *so_src, struct socket *so_dst) 523 { 524 SOCK_IO_RECV_UNLOCK(so_src); 525 SOCK_IO_SEND_UNLOCK(so_dst); 526 } 527 528 /* 529 * Move data from the source to the sink. Assumes that both of the relevant 530 * socket I/O locks are held. 531 */ 532 static int 533 so_splice_xfer_data(struct socket *so_src, struct socket *so_dst, off_t max, 534 ssize_t *lenp) 535 { 536 struct uio uio; 537 struct mbuf *m; 538 struct sockbuf *sb_src, *sb_dst; 539 ssize_t len; 540 long space; 541 int error, flags; 542 543 SOCK_IO_RECV_ASSERT_LOCKED(so_src); 544 SOCK_IO_SEND_ASSERT_LOCKED(so_dst); 545 546 error = 0; 547 m = NULL; 548 memset(&uio, 0, sizeof(uio)); 549 550 sb_src = &so_src->so_rcv; 551 sb_dst = &so_dst->so_snd; 552 553 space = sbspace(sb_dst); 554 if (space < 0) 555 space = 0; 556 len = MIN(max, MIN(space, sbavail(sb_src))); 557 if (len == 0) { 558 SOCK_RECVBUF_LOCK(so_src); 559 if ((sb_src->sb_state & SBS_CANTRCVMORE) != 0) 560 error = EPIPE; 561 SOCK_RECVBUF_UNLOCK(so_src); 562 } else { 563 flags = MSG_DONTWAIT; 564 uio.uio_resid = len; 565 if (splice_receive_stream && sb_src->sb_tls_info == NULL) { 566 error = soreceive_stream_locked(so_src, sb_src, NULL, 567 &uio, &m, NULL, flags); 568 } else { 569 error = soreceive_generic_locked(so_src, NULL, 570 &uio, &m, NULL, &flags); 571 } 572 if (error != 0 && m != NULL) { 573 m_freem(m); 574 m = NULL; 575 } 576 } 577 if (m != NULL) { 578 len -= uio.uio_resid; 579 error = sosend_generic_locked(so_dst, NULL, NULL, m, NULL, 580 MSG_DONTWAIT, curthread); 581 } else if (error == 0) { 582 len = 0; 583 SOCK_SENDBUF_LOCK(so_dst); 584 if ((sb_dst->sb_state & SBS_CANTSENDMORE) != 0) 585 error = EPIPE; 586 SOCK_SENDBUF_UNLOCK(so_dst); 587 } 588 if (error == 0) 589 *lenp = len; 590 return (error); 591 } 592 593 /* 594 * Transfer data from the source to the sink. 595 */ 596 static void 597 so_splice_xfer(struct so_splice *sp) 598 { 599 struct socket *so_src, *so_dst; 600 off_t max; 601 ssize_t len; 602 int error; 603 604 mtx_assert(&sp->mtx, MA_OWNED); 605 KASSERT(sp->state == SPLICE_QUEUED || sp->state == SPLICE_CLOSING, 606 ("so_splice_xfer: invalid state %d", sp->state)); 607 KASSERT(sp->max != 0, ("so_splice_xfer: max == 0")); 608 609 if (sp->state == SPLICE_CLOSING) { 610 /* Userspace asked us to close the splice. */ 611 goto closing; 612 } 613 614 sp->state = SPLICE_RUNNING; 615 so_src = sp->src; 616 so_dst = sp->dst; 617 max = sp->max > 0 ? sp->max - so_src->so_splice_sent : OFF_MAX; 618 if (max < 0) 619 max = 0; 620 621 /* 622 * Lock the sockets in order to block userspace from doing anything 623 * sneaky. If an error occurs or one of the sockets can no longer 624 * transfer data, we will automatically unsplice. 625 */ 626 mtx_unlock(&sp->mtx); 627 splice_lock_pair(so_src, so_dst); 628 629 error = so_splice_xfer_data(so_src, so_dst, max, &len); 630 631 mtx_lock(&sp->mtx); 632 633 /* 634 * Update our stats while still holding the socket locks. This 635 * synchronizes with getsockopt(SO_SPLICE), see the comment there. 636 */ 637 if (error == 0) { 638 KASSERT(len >= 0, ("%s: len %zd < 0", __func__, len)); 639 so_src->so_splice_sent += len; 640 } 641 splice_unlock_pair(so_src, so_dst); 642 643 switch (sp->state) { 644 case SPLICE_CLOSING: 645 closing: 646 sp->state = SPLICE_CLOSED; 647 wakeup(sp); 648 mtx_unlock(&sp->mtx); 649 break; 650 case SPLICE_RUNNING: 651 if (error != 0 || 652 (sp->max > 0 && so_src->so_splice_sent >= sp->max)) { 653 sp->state = SPLICE_EXCEPTION; 654 soref(so_src); 655 mtx_unlock(&sp->mtx); 656 (void)so_unsplice(so_src, false); 657 sorele(so_src); 658 } else { 659 /* 660 * Locklessly check for additional bytes in the source's 661 * receive buffer and queue more work if possible. We 662 * may end up queuing needless work, but that's ok, and 663 * if we race with a thread inserting more data into the 664 * buffer and observe sbavail() == 0, the splice mutex 665 * ensures that splice_push() will queue more work for 666 * us. 667 */ 668 if (sbavail(&so_src->so_rcv) > 0 && 669 sbspace(&so_dst->so_snd) > 0) { 670 sp->state = SPLICE_QUEUED; 671 mtx_unlock(&sp->mtx); 672 so_splice_dispatch_async(sp); 673 } else { 674 sp->state = SPLICE_IDLE; 675 mtx_unlock(&sp->mtx); 676 } 677 } 678 break; 679 default: 680 __assert_unreachable(); 681 } 682 } 683 684 static void 685 socket_init(void *tag) 686 { 687 688 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL, 689 NULL, NULL, UMA_ALIGN_PTR, 0); 690 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 691 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached"); 692 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL, 693 EVENTHANDLER_PRI_FIRST); 694 } 695 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL); 696 697 #ifdef SOCKET_HHOOK 698 static void 699 socket_hhook_register(int subtype) 700 { 701 702 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype, 703 &V_socket_hhh[subtype], 704 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0) 705 printf("%s: WARNING: unable to register hook\n", __func__); 706 } 707 708 static void 709 socket_hhook_deregister(int subtype) 710 { 711 712 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0) 713 printf("%s: WARNING: unable to deregister hook\n", __func__); 714 } 715 716 static void 717 socket_vnet_init(const void *unused __unused) 718 { 719 int i; 720 721 /* We expect a contiguous range */ 722 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 723 socket_hhook_register(i); 724 } 725 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 726 socket_vnet_init, NULL); 727 728 static void 729 socket_vnet_uninit(const void *unused __unused) 730 { 731 int i; 732 733 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 734 socket_hhook_deregister(i); 735 } 736 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 737 socket_vnet_uninit, NULL); 738 #endif /* SOCKET_HHOOK */ 739 740 /* 741 * Initialise maxsockets. This SYSINIT must be run after 742 * tunable_mbinit(). 743 */ 744 static void 745 init_maxsockets(void *ignored) 746 { 747 748 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 749 maxsockets = imax(maxsockets, maxfiles); 750 } 751 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 752 753 /* 754 * Sysctl to get and set the maximum global sockets limit. Notify protocols 755 * of the change so that they can update their dependent limits as required. 756 */ 757 static int 758 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 759 { 760 int error, newmaxsockets; 761 762 newmaxsockets = maxsockets; 763 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req); 764 if (error == 0 && req->newptr && newmaxsockets != maxsockets) { 765 if (newmaxsockets > maxsockets && 766 newmaxsockets <= maxfiles) { 767 maxsockets = newmaxsockets; 768 EVENTHANDLER_INVOKE(maxsockets_change); 769 } else 770 error = EINVAL; 771 } 772 return (error); 773 } 774 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, 775 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, 776 &maxsockets, 0, sysctl_maxsockets, "IU", 777 "Maximum number of sockets available"); 778 779 /* 780 * Socket operation routines. These routines are called by the routines in 781 * sys_socket.c or from a system process, and implement the semantics of 782 * socket operations by switching out to the protocol specific routines. 783 */ 784 785 /* 786 * Get a socket structure from our zone, and initialize it. Note that it 787 * would probably be better to allocate socket and PCB at the same time, but 788 * I'm not convinced that all the protocols can be easily modified to do 789 * this. 790 * 791 * soalloc() returns a socket with a ref count of 0. 792 */ 793 static struct socket * 794 soalloc(struct vnet *vnet) 795 { 796 struct socket *so; 797 798 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); 799 if (so == NULL) 800 return (NULL); 801 #ifdef MAC 802 if (mac_socket_init(so, M_NOWAIT) != 0) { 803 uma_zfree(socket_zone, so); 804 return (NULL); 805 } 806 #endif 807 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { 808 uma_zfree(socket_zone, so); 809 return (NULL); 810 } 811 812 /* 813 * The socket locking protocol allows to lock 2 sockets at a time, 814 * however, the first one must be a listening socket. WITNESS lacks 815 * a feature to change class of an existing lock, so we use DUPOK. 816 */ 817 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); 818 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); 819 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); 820 so->so_rcv.sb_sel = &so->so_rdsel; 821 so->so_snd.sb_sel = &so->so_wrsel; 822 sx_init(&so->so_snd_sx, "so_snd_sx"); 823 sx_init(&so->so_rcv_sx, "so_rcv_sx"); 824 TAILQ_INIT(&so->so_snd.sb_aiojobq); 825 TAILQ_INIT(&so->so_rcv.sb_aiojobq); 826 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); 827 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); 828 #ifdef VIMAGE 829 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", 830 __func__, __LINE__, so)); 831 so->so_vnet = vnet; 832 #endif 833 #ifdef SOCKET_HHOOK 834 /* We shouldn't need the so_global_mtx */ 835 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { 836 /* Do we need more comprehensive error returns? */ 837 uma_zfree(socket_zone, so); 838 return (NULL); 839 } 840 #endif 841 mtx_lock(&so_global_mtx); 842 so->so_gencnt = ++so_gencnt; 843 ++numopensockets; 844 #ifdef VIMAGE 845 vnet->vnet_sockcnt++; 846 #endif 847 mtx_unlock(&so_global_mtx); 848 849 return (so); 850 } 851 852 /* 853 * Free the storage associated with a socket at the socket layer, tear down 854 * locks, labels, etc. All protocol state is assumed already to have been 855 * torn down (and possibly never set up) by the caller. 856 */ 857 void 858 sodealloc(struct socket *so) 859 { 860 861 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 862 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 863 864 mtx_lock(&so_global_mtx); 865 so->so_gencnt = ++so_gencnt; 866 --numopensockets; /* Could be below, but faster here. */ 867 #ifdef VIMAGE 868 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", 869 __func__, __LINE__, so)); 870 so->so_vnet->vnet_sockcnt--; 871 #endif 872 mtx_unlock(&so_global_mtx); 873 #ifdef MAC 874 mac_socket_destroy(so); 875 #endif 876 #ifdef SOCKET_HHOOK 877 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); 878 #endif 879 880 khelp_destroy_osd(&so->osd); 881 if (SOLISTENING(so)) { 882 if (so->sol_accept_filter != NULL) 883 accept_filt_setopt(so, NULL); 884 } else { 885 if (so->so_rcv.sb_hiwat) 886 (void)chgsbsize(so->so_cred->cr_uidinfo, 887 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 888 if (so->so_snd.sb_hiwat) 889 (void)chgsbsize(so->so_cred->cr_uidinfo, 890 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 891 sx_destroy(&so->so_snd_sx); 892 sx_destroy(&so->so_rcv_sx); 893 mtx_destroy(&so->so_snd_mtx); 894 mtx_destroy(&so->so_rcv_mtx); 895 } 896 crfree(so->so_cred); 897 mtx_destroy(&so->so_lock); 898 uma_zfree(socket_zone, so); 899 } 900 901 /* 902 * socreate returns a socket with a ref count of 1 and a file descriptor 903 * reference. The socket should be closed with soclose(). 904 */ 905 int 906 socreate(int dom, struct socket **aso, int type, int proto, 907 struct ucred *cred, struct thread *td) 908 { 909 struct protosw *prp; 910 struct socket *so; 911 int error; 912 913 /* 914 * XXX: divert(4) historically abused PF_INET. Keep this compatibility 915 * shim until all applications have been updated. 916 */ 917 if (__predict_false(dom == PF_INET && type == SOCK_RAW && 918 proto == IPPROTO_DIVERT)) { 919 dom = PF_DIVERT; 920 printf("%s uses obsolete way to create divert(4) socket\n", 921 td->td_proc->p_comm); 922 } 923 924 prp = pffindproto(dom, type, proto); 925 if (prp == NULL) { 926 /* No support for domain. */ 927 if (pffinddomain(dom) == NULL) 928 return (EAFNOSUPPORT); 929 /* No support for socket type. */ 930 if (proto == 0 && type != 0) 931 return (EPROTOTYPE); 932 return (EPROTONOSUPPORT); 933 } 934 935 MPASS(prp->pr_attach); 936 937 if ((prp->pr_flags & PR_CAPATTACH) == 0) { 938 if (CAP_TRACING(td)) 939 ktrcapfail(CAPFAIL_PROTO, &proto); 940 if (IN_CAPABILITY_MODE(td)) 941 return (ECAPMODE); 942 } 943 944 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0) 945 return (EPROTONOSUPPORT); 946 947 so = soalloc(CRED_TO_VNET(cred)); 948 if (so == NULL) 949 return (ENOBUFS); 950 951 so->so_type = type; 952 so->so_cred = crhold(cred); 953 if ((prp->pr_domain->dom_family == PF_INET) || 954 (prp->pr_domain->dom_family == PF_INET6) || 955 (prp->pr_domain->dom_family == PF_ROUTE)) 956 so->so_fibnum = td->td_proc->p_fibnum; 957 else 958 so->so_fibnum = 0; 959 so->so_proto = prp; 960 #ifdef MAC 961 mac_socket_create(cred, so); 962 #endif 963 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 964 so_rdknl_assert_lock); 965 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 966 so_wrknl_assert_lock); 967 if ((prp->pr_flags & PR_SOCKBUF) == 0) { 968 so->so_snd.sb_mtx = &so->so_snd_mtx; 969 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 970 } 971 /* 972 * Auto-sizing of socket buffers is managed by the protocols and 973 * the appropriate flags must be set in the pru_attach function. 974 */ 975 CURVNET_SET(so->so_vnet); 976 error = prp->pr_attach(so, proto, td); 977 CURVNET_RESTORE(); 978 if (error) { 979 sodealloc(so); 980 return (error); 981 } 982 soref(so); 983 *aso = so; 984 return (0); 985 } 986 987 #ifdef REGRESSION 988 static int regression_sonewconn_earlytest = 1; 989 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 990 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 991 #endif 992 993 static int sooverprio = LOG_DEBUG; 994 SYSCTL_INT(_kern_ipc, OID_AUTO, sooverprio, CTLFLAG_RW, 995 &sooverprio, 0, "Log priority for listen socket overflows: 0..7 or -1 to disable"); 996 997 static struct timeval overinterval = { 60, 0 }; 998 SYSCTL_TIMEVAL_SEC(_kern_ipc, OID_AUTO, sooverinterval, CTLFLAG_RW, 999 &overinterval, 1000 "Delay in seconds between warnings for listen socket overflows"); 1001 1002 /* 1003 * When an attempt at a new connection is noted on a socket which supports 1004 * accept(2), the protocol has two options: 1005 * 1) Call legacy sonewconn() function, which would call protocol attach 1006 * method, same as used for socket(2). 1007 * 2) Call solisten_clone(), do attach that is specific to a cloned connection, 1008 * and then call solisten_enqueue(). 1009 * 1010 * Note: the ref count on the socket is 0 on return. 1011 */ 1012 struct socket * 1013 solisten_clone(struct socket *head) 1014 { 1015 struct sbuf descrsb; 1016 struct socket *so; 1017 int len, overcount; 1018 u_int qlen; 1019 const char localprefix[] = "local:"; 1020 char descrbuf[SUNPATHLEN + sizeof(localprefix)]; 1021 #if defined(INET6) 1022 char addrbuf[INET6_ADDRSTRLEN]; 1023 #elif defined(INET) 1024 char addrbuf[INET_ADDRSTRLEN]; 1025 #endif 1026 bool dolog, over; 1027 1028 SOLISTEN_LOCK(head); 1029 over = (head->sol_qlen > 3 * head->sol_qlimit / 2); 1030 #ifdef REGRESSION 1031 if (regression_sonewconn_earlytest && over) { 1032 #else 1033 if (over) { 1034 #endif 1035 head->sol_overcount++; 1036 dolog = (sooverprio >= 0) && 1037 !!ratecheck(&head->sol_lastover, &overinterval); 1038 1039 /* 1040 * If we're going to log, copy the overflow count and queue 1041 * length from the listen socket before dropping the lock. 1042 * Also, reset the overflow count. 1043 */ 1044 if (dolog) { 1045 overcount = head->sol_overcount; 1046 head->sol_overcount = 0; 1047 qlen = head->sol_qlen; 1048 } 1049 SOLISTEN_UNLOCK(head); 1050 1051 if (dolog) { 1052 /* 1053 * Try to print something descriptive about the 1054 * socket for the error message. 1055 */ 1056 sbuf_new(&descrsb, descrbuf, sizeof(descrbuf), 1057 SBUF_FIXEDLEN); 1058 switch (head->so_proto->pr_domain->dom_family) { 1059 #if defined(INET) || defined(INET6) 1060 #ifdef INET 1061 case AF_INET: 1062 #endif 1063 #ifdef INET6 1064 case AF_INET6: 1065 if (head->so_proto->pr_domain->dom_family == 1066 AF_INET6 || 1067 (sotoinpcb(head)->inp_inc.inc_flags & 1068 INC_ISIPV6)) { 1069 ip6_sprintf(addrbuf, 1070 &sotoinpcb(head)->inp_inc.inc6_laddr); 1071 sbuf_printf(&descrsb, "[%s]", addrbuf); 1072 } else 1073 #endif 1074 { 1075 #ifdef INET 1076 inet_ntoa_r( 1077 sotoinpcb(head)->inp_inc.inc_laddr, 1078 addrbuf); 1079 sbuf_cat(&descrsb, addrbuf); 1080 #endif 1081 } 1082 sbuf_printf(&descrsb, ":%hu (proto %u)", 1083 ntohs(sotoinpcb(head)->inp_inc.inc_lport), 1084 head->so_proto->pr_protocol); 1085 break; 1086 #endif /* INET || INET6 */ 1087 case AF_UNIX: 1088 sbuf_cat(&descrsb, localprefix); 1089 if (sotounpcb(head)->unp_addr != NULL) 1090 len = 1091 sotounpcb(head)->unp_addr->sun_len - 1092 offsetof(struct sockaddr_un, 1093 sun_path); 1094 else 1095 len = 0; 1096 if (len > 0) 1097 sbuf_bcat(&descrsb, 1098 sotounpcb(head)->unp_addr->sun_path, 1099 len); 1100 else 1101 sbuf_cat(&descrsb, "(unknown)"); 1102 break; 1103 } 1104 1105 /* 1106 * If we can't print something more specific, at least 1107 * print the domain name. 1108 */ 1109 if (sbuf_finish(&descrsb) != 0 || 1110 sbuf_len(&descrsb) <= 0) { 1111 sbuf_clear(&descrsb); 1112 sbuf_cat(&descrsb, 1113 head->so_proto->pr_domain->dom_name ?: 1114 "unknown"); 1115 sbuf_finish(&descrsb); 1116 } 1117 KASSERT(sbuf_len(&descrsb) > 0, 1118 ("%s: sbuf creation failed", __func__)); 1119 /* 1120 * Preserve the historic listen queue overflow log 1121 * message, that starts with "sonewconn:". It has 1122 * been known to sysadmins for years and also test 1123 * sys/kern/sonewconn_overflow checks for it. 1124 */ 1125 if (head->so_cred == 0) { 1126 log(LOG_PRI(sooverprio), 1127 "sonewconn: pcb %p (%s): " 1128 "Listen queue overflow: %i already in " 1129 "queue awaiting acceptance (%d " 1130 "occurrences)\n", head->so_pcb, 1131 sbuf_data(&descrsb), 1132 qlen, overcount); 1133 } else { 1134 log(LOG_PRI(sooverprio), 1135 "sonewconn: pcb %p (%s): " 1136 "Listen queue overflow: " 1137 "%i already in queue awaiting acceptance " 1138 "(%d occurrences), euid %d, rgid %d, jail %s\n", 1139 head->so_pcb, sbuf_data(&descrsb), qlen, 1140 overcount, head->so_cred->cr_uid, 1141 head->so_cred->cr_rgid, 1142 head->so_cred->cr_prison ? 1143 head->so_cred->cr_prison->pr_name : 1144 "not_jailed"); 1145 } 1146 sbuf_delete(&descrsb); 1147 1148 overcount = 0; 1149 } 1150 1151 return (NULL); 1152 } 1153 SOLISTEN_UNLOCK(head); 1154 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL", 1155 __func__, head)); 1156 so = soalloc(head->so_vnet); 1157 if (so == NULL) { 1158 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 1159 "limit reached or out of memory\n", 1160 __func__, head->so_pcb); 1161 return (NULL); 1162 } 1163 so->so_listen = head; 1164 so->so_type = head->so_type; 1165 /* 1166 * POSIX is ambiguous on what options an accept(2)ed socket should 1167 * inherit from the listener. Words "create a new socket" may be 1168 * interpreted as not inheriting anything. Best programming practice 1169 * for application developers is to not rely on such inheritance. 1170 * FreeBSD had historically inherited all so_options excluding 1171 * SO_ACCEPTCONN, which virtually means all SOL_SOCKET level options, 1172 * including those completely irrelevant to a new born socket. For 1173 * compatibility with older versions we will inherit a list of 1174 * meaningful options. 1175 * The crucial bit to inherit is SO_ACCEPTFILTER. We need it present 1176 * in the child socket for soisconnected() promoting socket from the 1177 * incomplete queue to complete. It will be cleared before the child 1178 * gets available to accept(2). 1179 */ 1180 so->so_options = head->so_options & (SO_ACCEPTFILTER | SO_KEEPALIVE | 1181 SO_DONTROUTE | SO_LINGER | SO_OOBINLINE | SO_NOSIGPIPE); 1182 so->so_linger = head->so_linger; 1183 so->so_state = head->so_state; 1184 so->so_fibnum = head->so_fibnum; 1185 so->so_proto = head->so_proto; 1186 so->so_cred = crhold(head->so_cred); 1187 #ifdef SOCKET_HHOOK 1188 if (V_socket_hhh[HHOOK_SOCKET_NEWCONN]->hhh_nhooks > 0) { 1189 if (hhook_run_socket(so, head, HHOOK_SOCKET_NEWCONN)) { 1190 sodealloc(so); 1191 log(LOG_DEBUG, "%s: hhook run failed\n", __func__); 1192 return (NULL); 1193 } 1194 } 1195 #endif 1196 #ifdef MAC 1197 mac_socket_newconn(head, so); 1198 #endif 1199 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1200 so_rdknl_assert_lock); 1201 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1202 so_wrknl_assert_lock); 1203 VNET_SO_ASSERT(head); 1204 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) { 1205 sodealloc(so); 1206 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 1207 __func__, head->so_pcb); 1208 return (NULL); 1209 } 1210 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; 1211 so->so_snd.sb_lowat = head->sol_sbsnd_lowat; 1212 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; 1213 so->so_snd.sb_timeo = head->sol_sbsnd_timeo; 1214 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE; 1215 so->so_snd.sb_flags = head->sol_sbsnd_flags & SB_AUTOSIZE; 1216 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { 1217 so->so_snd.sb_mtx = &so->so_snd_mtx; 1218 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 1219 } 1220 1221 return (so); 1222 } 1223 1224 /* Connstatus may be 0 or SS_ISCONNECTED. */ 1225 struct socket * 1226 sonewconn(struct socket *head, int connstatus) 1227 { 1228 struct socket *so; 1229 1230 if ((so = solisten_clone(head)) == NULL) 1231 return (NULL); 1232 1233 if (so->so_proto->pr_attach(so, 0, NULL) != 0) { 1234 sodealloc(so); 1235 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", 1236 __func__, head->so_pcb); 1237 return (NULL); 1238 } 1239 1240 (void)solisten_enqueue(so, connstatus); 1241 1242 return (so); 1243 } 1244 1245 /* 1246 * Enqueue socket cloned by solisten_clone() to the listen queue of the 1247 * listener it has been cloned from. 1248 * 1249 * Return 'true' if socket landed on complete queue, otherwise 'false'. 1250 */ 1251 bool 1252 solisten_enqueue(struct socket *so, int connstatus) 1253 { 1254 struct socket *head = so->so_listen; 1255 1256 MPASS(refcount_load(&so->so_count) == 0); 1257 refcount_init(&so->so_count, 1); 1258 1259 SOLISTEN_LOCK(head); 1260 if (head->sol_accept_filter != NULL) 1261 connstatus = 0; 1262 so->so_state |= connstatus; 1263 soref(head); /* A socket on (in)complete queue refs head. */ 1264 if (connstatus) { 1265 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 1266 so->so_qstate = SQ_COMP; 1267 head->sol_qlen++; 1268 solisten_wakeup(head); /* unlocks */ 1269 return (true); 1270 } else { 1271 /* 1272 * Keep removing sockets from the head until there's room for 1273 * us to insert on the tail. In pre-locking revisions, this 1274 * was a simple if(), but as we could be racing with other 1275 * threads and soabort() requires dropping locks, we must 1276 * loop waiting for the condition to be true. 1277 */ 1278 while (head->sol_incqlen > head->sol_qlimit) { 1279 struct socket *sp; 1280 1281 sp = TAILQ_FIRST(&head->sol_incomp); 1282 TAILQ_REMOVE(&head->sol_incomp, sp, so_list); 1283 head->sol_incqlen--; 1284 SOCK_LOCK(sp); 1285 sp->so_qstate = SQ_NONE; 1286 sp->so_listen = NULL; 1287 SOCK_UNLOCK(sp); 1288 sorele_locked(head); /* does SOLISTEN_UNLOCK, head stays */ 1289 soabort(sp); 1290 SOLISTEN_LOCK(head); 1291 } 1292 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list); 1293 so->so_qstate = SQ_INCOMP; 1294 head->sol_incqlen++; 1295 SOLISTEN_UNLOCK(head); 1296 return (false); 1297 } 1298 } 1299 1300 #if defined(SCTP) || defined(SCTP_SUPPORT) 1301 /* 1302 * Socket part of sctp_peeloff(). Detach a new socket from an 1303 * association. The new socket is returned with a reference. 1304 * 1305 * XXXGL: reduce copy-paste with solisten_clone(). 1306 */ 1307 struct socket * 1308 sopeeloff(struct socket *head) 1309 { 1310 struct socket *so; 1311 1312 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", 1313 __func__, __LINE__, head)); 1314 so = soalloc(head->so_vnet); 1315 if (so == NULL) { 1316 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 1317 "limit reached or out of memory\n", 1318 __func__, head->so_pcb); 1319 return (NULL); 1320 } 1321 so->so_type = head->so_type; 1322 so->so_options = head->so_options; 1323 so->so_linger = head->so_linger; 1324 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; 1325 so->so_fibnum = head->so_fibnum; 1326 so->so_proto = head->so_proto; 1327 so->so_cred = crhold(head->so_cred); 1328 #ifdef MAC 1329 mac_socket_newconn(head, so); 1330 #endif 1331 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1332 so_rdknl_assert_lock); 1333 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1334 so_wrknl_assert_lock); 1335 VNET_SO_ASSERT(head); 1336 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 1337 sodealloc(so); 1338 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 1339 __func__, head->so_pcb); 1340 return (NULL); 1341 } 1342 if ((*so->so_proto->pr_attach)(so, 0, NULL)) { 1343 sodealloc(so); 1344 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 1345 __func__, head->so_pcb); 1346 return (NULL); 1347 } 1348 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 1349 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 1350 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 1351 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 1352 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 1353 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 1354 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { 1355 so->so_snd.sb_mtx = &so->so_snd_mtx; 1356 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 1357 } 1358 1359 soref(so); 1360 1361 return (so); 1362 } 1363 #endif /* SCTP */ 1364 1365 int 1366 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) 1367 { 1368 int error; 1369 1370 CURVNET_SET(so->so_vnet); 1371 error = so->so_proto->pr_bind(so, nam, td); 1372 CURVNET_RESTORE(); 1373 return (error); 1374 } 1375 1376 int 1377 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 1378 { 1379 int error; 1380 1381 CURVNET_SET(so->so_vnet); 1382 error = so->so_proto->pr_bindat(fd, so, nam, td); 1383 CURVNET_RESTORE(); 1384 return (error); 1385 } 1386 1387 /* 1388 * solisten() transitions a socket from a non-listening state to a listening 1389 * state, but can also be used to update the listen queue depth on an 1390 * existing listen socket. The protocol will call back into the sockets 1391 * layer using solisten_proto_check() and solisten_proto() to check and set 1392 * socket-layer listen state. Call backs are used so that the protocol can 1393 * acquire both protocol and socket layer locks in whatever order is required 1394 * by the protocol. 1395 * 1396 * Protocol implementors are advised to hold the socket lock across the 1397 * socket-layer test and set to avoid races at the socket layer. 1398 */ 1399 int 1400 solisten(struct socket *so, int backlog, struct thread *td) 1401 { 1402 int error; 1403 1404 CURVNET_SET(so->so_vnet); 1405 error = so->so_proto->pr_listen(so, backlog, td); 1406 CURVNET_RESTORE(); 1407 return (error); 1408 } 1409 1410 /* 1411 * Prepare for a call to solisten_proto(). Acquire all socket buffer locks in 1412 * order to interlock with socket I/O. 1413 */ 1414 int 1415 solisten_proto_check(struct socket *so) 1416 { 1417 SOCK_LOCK_ASSERT(so); 1418 1419 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 1420 SS_ISDISCONNECTING)) != 0) 1421 return (EINVAL); 1422 1423 /* 1424 * Sleeping is not permitted here, so simply fail if userspace is 1425 * attempting to transmit or receive on the socket. This kind of 1426 * transient failure is not ideal, but it should occur only if userspace 1427 * is misusing the socket interfaces. 1428 */ 1429 if (!sx_try_xlock(&so->so_snd_sx)) 1430 return (EAGAIN); 1431 if (!sx_try_xlock(&so->so_rcv_sx)) { 1432 sx_xunlock(&so->so_snd_sx); 1433 return (EAGAIN); 1434 } 1435 mtx_lock(&so->so_snd_mtx); 1436 mtx_lock(&so->so_rcv_mtx); 1437 1438 /* Interlock with soo_aio_queue() and KTLS. */ 1439 if (!SOLISTENING(so)) { 1440 bool ktls; 1441 1442 #ifdef KERN_TLS 1443 ktls = so->so_snd.sb_tls_info != NULL || 1444 so->so_rcv.sb_tls_info != NULL; 1445 #else 1446 ktls = false; 1447 #endif 1448 if (ktls || 1449 (so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 || 1450 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0) { 1451 solisten_proto_abort(so); 1452 return (EINVAL); 1453 } 1454 } 1455 1456 return (0); 1457 } 1458 1459 /* 1460 * Undo the setup done by solisten_proto_check(). 1461 */ 1462 void 1463 solisten_proto_abort(struct socket *so) 1464 { 1465 mtx_unlock(&so->so_snd_mtx); 1466 mtx_unlock(&so->so_rcv_mtx); 1467 sx_xunlock(&so->so_snd_sx); 1468 sx_xunlock(&so->so_rcv_sx); 1469 } 1470 1471 void 1472 solisten_proto(struct socket *so, int backlog) 1473 { 1474 int sbrcv_lowat, sbsnd_lowat; 1475 u_int sbrcv_hiwat, sbsnd_hiwat; 1476 short sbrcv_flags, sbsnd_flags; 1477 sbintime_t sbrcv_timeo, sbsnd_timeo; 1478 1479 SOCK_LOCK_ASSERT(so); 1480 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 1481 SS_ISDISCONNECTING)) == 0, 1482 ("%s: bad socket state %p", __func__, so)); 1483 1484 if (SOLISTENING(so)) 1485 goto listening; 1486 1487 /* 1488 * Change this socket to listening state. 1489 */ 1490 sbrcv_lowat = so->so_rcv.sb_lowat; 1491 sbsnd_lowat = so->so_snd.sb_lowat; 1492 sbrcv_hiwat = so->so_rcv.sb_hiwat; 1493 sbsnd_hiwat = so->so_snd.sb_hiwat; 1494 sbrcv_flags = so->so_rcv.sb_flags; 1495 sbsnd_flags = so->so_snd.sb_flags; 1496 sbrcv_timeo = so->so_rcv.sb_timeo; 1497 sbsnd_timeo = so->so_snd.sb_timeo; 1498 1499 #ifdef MAC 1500 mac_socketpeer_label_free(so->so_peerlabel); 1501 #endif 1502 1503 if (!(so->so_proto->pr_flags & PR_SOCKBUF)) { 1504 sbdestroy(so, SO_SND); 1505 sbdestroy(so, SO_RCV); 1506 } 1507 1508 #ifdef INVARIANTS 1509 bzero(&so->so_rcv, 1510 sizeof(struct socket) - offsetof(struct socket, so_rcv)); 1511 #endif 1512 1513 so->sol_sbrcv_lowat = sbrcv_lowat; 1514 so->sol_sbsnd_lowat = sbsnd_lowat; 1515 so->sol_sbrcv_hiwat = sbrcv_hiwat; 1516 so->sol_sbsnd_hiwat = sbsnd_hiwat; 1517 so->sol_sbrcv_flags = sbrcv_flags; 1518 so->sol_sbsnd_flags = sbsnd_flags; 1519 so->sol_sbrcv_timeo = sbrcv_timeo; 1520 so->sol_sbsnd_timeo = sbsnd_timeo; 1521 1522 so->sol_qlen = so->sol_incqlen = 0; 1523 TAILQ_INIT(&so->sol_incomp); 1524 TAILQ_INIT(&so->sol_comp); 1525 1526 so->sol_accept_filter = NULL; 1527 so->sol_accept_filter_arg = NULL; 1528 so->sol_accept_filter_str = NULL; 1529 1530 so->sol_upcall = NULL; 1531 so->sol_upcallarg = NULL; 1532 1533 so->so_options |= SO_ACCEPTCONN; 1534 1535 listening: 1536 if (backlog < 0 || backlog > V_somaxconn) 1537 backlog = V_somaxconn; 1538 so->sol_qlimit = backlog; 1539 1540 mtx_unlock(&so->so_snd_mtx); 1541 mtx_unlock(&so->so_rcv_mtx); 1542 sx_xunlock(&so->so_snd_sx); 1543 sx_xunlock(&so->so_rcv_sx); 1544 } 1545 1546 /* 1547 * Wakeup listeners/subsystems once we have a complete connection. 1548 * Enters with lock, returns unlocked. 1549 */ 1550 void 1551 solisten_wakeup(struct socket *sol) 1552 { 1553 1554 if (sol->sol_upcall != NULL) 1555 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT); 1556 else { 1557 selwakeuppri(&sol->so_rdsel, PSOCK); 1558 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0); 1559 } 1560 SOLISTEN_UNLOCK(sol); 1561 wakeup_one(&sol->sol_comp); 1562 if ((sol->so_state & SS_ASYNC) && sol->so_sigio != NULL) 1563 pgsigio(&sol->so_sigio, SIGIO, 0); 1564 } 1565 1566 /* 1567 * Return single connection off a listening socket queue. Main consumer of 1568 * the function is kern_accept4(). Some modules, that do their own accept 1569 * management also use the function. The socket reference held by the 1570 * listen queue is handed to the caller. 1571 * 1572 * Listening socket must be locked on entry and is returned unlocked on 1573 * return. 1574 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT. 1575 */ 1576 int 1577 solisten_dequeue(struct socket *head, struct socket **ret, int flags) 1578 { 1579 struct socket *so; 1580 int error; 1581 1582 SOLISTEN_LOCK_ASSERT(head); 1583 1584 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) && 1585 head->so_error == 0) { 1586 error = msleep(&head->sol_comp, SOCK_MTX(head), PSOCK | PCATCH, 1587 "accept", 0); 1588 if (error != 0) { 1589 SOLISTEN_UNLOCK(head); 1590 return (error); 1591 } 1592 } 1593 if (head->so_error) { 1594 error = head->so_error; 1595 head->so_error = 0; 1596 } else if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp)) 1597 error = EWOULDBLOCK; 1598 else 1599 error = 0; 1600 if (error) { 1601 SOLISTEN_UNLOCK(head); 1602 return (error); 1603 } 1604 so = TAILQ_FIRST(&head->sol_comp); 1605 SOCK_LOCK(so); 1606 KASSERT(so->so_qstate == SQ_COMP, 1607 ("%s: so %p not SQ_COMP", __func__, so)); 1608 head->sol_qlen--; 1609 so->so_qstate = SQ_NONE; 1610 so->so_listen = NULL; 1611 TAILQ_REMOVE(&head->sol_comp, so, so_list); 1612 if (flags & ACCEPT4_INHERIT) 1613 so->so_state |= (head->so_state & SS_NBIO); 1614 else 1615 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 1616 SOCK_UNLOCK(so); 1617 sorele_locked(head); 1618 1619 *ret = so; 1620 return (0); 1621 } 1622 1623 static struct so_splice * 1624 so_splice_alloc(off_t max) 1625 { 1626 struct so_splice *sp; 1627 1628 sp = uma_zalloc(splice_zone, M_WAITOK); 1629 sp->src = NULL; 1630 sp->dst = NULL; 1631 sp->max = max > 0 ? max : -1; 1632 do { 1633 sp->wq_index = atomic_fetchadd_32(&splice_index, 1) % 1634 (mp_maxid + 1); 1635 } while (CPU_ABSENT(sp->wq_index)); 1636 sp->state = SPLICE_INIT; 1637 TIMEOUT_TASK_INIT(taskqueue_thread, &sp->timeout, 0, so_splice_timeout, 1638 sp); 1639 return (sp); 1640 } 1641 1642 static void 1643 so_splice_free(struct so_splice *sp) 1644 { 1645 KASSERT(sp->state == SPLICE_CLOSED, 1646 ("so_splice_free: sp %p not closed", sp)); 1647 uma_zfree(splice_zone, sp); 1648 } 1649 1650 static void 1651 so_splice_timeout(void *arg, int pending __unused) 1652 { 1653 struct so_splice *sp; 1654 1655 sp = arg; 1656 (void)so_unsplice(sp->src, true); 1657 } 1658 1659 /* 1660 * Splice the output from so to the input of so2. 1661 */ 1662 static int 1663 so_splice(struct socket *so, struct socket *so2, struct splice *splice) 1664 { 1665 struct so_splice *sp; 1666 int error; 1667 1668 if (splice->sp_max < 0) 1669 return (EINVAL); 1670 /* Handle only TCP for now; TODO: other streaming protos */ 1671 if (so->so_proto->pr_protocol != IPPROTO_TCP || 1672 so2->so_proto->pr_protocol != IPPROTO_TCP) 1673 return (EPROTONOSUPPORT); 1674 if (so->so_vnet != so2->so_vnet) 1675 return (EINVAL); 1676 1677 /* so_splice_xfer() assumes that we're using these implementations. */ 1678 KASSERT(so->so_proto->pr_sosend == sosend_generic, 1679 ("so_splice: sosend not sosend_generic")); 1680 KASSERT(so2->so_proto->pr_soreceive == soreceive_generic || 1681 so2->so_proto->pr_soreceive == soreceive_stream, 1682 ("so_splice: soreceive not soreceive_generic/stream")); 1683 1684 sp = so_splice_alloc(splice->sp_max); 1685 so->so_splice_sent = 0; 1686 sp->src = so; 1687 sp->dst = so2; 1688 1689 error = 0; 1690 SOCK_LOCK(so); 1691 if (SOLISTENING(so)) 1692 error = EINVAL; 1693 else if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0) 1694 error = ENOTCONN; 1695 else if (so->so_splice != NULL) 1696 error = EBUSY; 1697 if (error != 0) { 1698 SOCK_UNLOCK(so); 1699 uma_zfree(splice_zone, sp); 1700 return (error); 1701 } 1702 soref(so); 1703 so->so_splice = sp; 1704 SOCK_RECVBUF_LOCK(so); 1705 so->so_rcv.sb_flags |= SB_SPLICED; 1706 SOCK_RECVBUF_UNLOCK(so); 1707 SOCK_UNLOCK(so); 1708 1709 error = 0; 1710 SOCK_LOCK(so2); 1711 if (SOLISTENING(so2)) 1712 error = EINVAL; 1713 else if ((so2->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0) 1714 error = ENOTCONN; 1715 else if (so2->so_splice_back != NULL) 1716 error = EBUSY; 1717 if (error != 0) { 1718 SOCK_UNLOCK(so2); 1719 SOCK_LOCK(so); 1720 so->so_splice = NULL; 1721 SOCK_RECVBUF_LOCK(so); 1722 so->so_rcv.sb_flags &= ~SB_SPLICED; 1723 SOCK_RECVBUF_UNLOCK(so); 1724 SOCK_UNLOCK(so); 1725 sorele(so); 1726 uma_zfree(splice_zone, sp); 1727 return (error); 1728 } 1729 soref(so2); 1730 so2->so_splice_back = sp; 1731 SOCK_SENDBUF_LOCK(so2); 1732 so2->so_snd.sb_flags |= SB_SPLICED; 1733 mtx_lock(&sp->mtx); 1734 SOCK_SENDBUF_UNLOCK(so2); 1735 SOCK_UNLOCK(so2); 1736 1737 if (splice->sp_idle.tv_sec != 0 || splice->sp_idle.tv_usec != 0) { 1738 taskqueue_enqueue_timeout_sbt(taskqueue_thread, &sp->timeout, 1739 tvtosbt(splice->sp_idle), 0, C_PREL(4)); 1740 } 1741 1742 /* 1743 * Transfer any data already present in the socket buffer. 1744 */ 1745 sp->state = SPLICE_QUEUED; 1746 so_splice_xfer(sp); 1747 return (0); 1748 } 1749 1750 static int 1751 so_unsplice(struct socket *so, bool timeout) 1752 { 1753 struct socket *so2; 1754 struct so_splice *sp; 1755 bool drain; 1756 1757 /* 1758 * First unset SB_SPLICED and hide the splice structure so that 1759 * wakeup routines will stop enqueuing work. This also ensures that 1760 * a only a single thread will proceed with the unsplice. 1761 */ 1762 SOCK_LOCK(so); 1763 if (SOLISTENING(so)) { 1764 SOCK_UNLOCK(so); 1765 return (EINVAL); 1766 } 1767 SOCK_RECVBUF_LOCK(so); 1768 if ((so->so_rcv.sb_flags & SB_SPLICED) == 0) { 1769 SOCK_RECVBUF_UNLOCK(so); 1770 SOCK_UNLOCK(so); 1771 return (ENOTCONN); 1772 } 1773 so->so_rcv.sb_flags &= ~SB_SPLICED; 1774 sp = so->so_splice; 1775 so->so_splice = NULL; 1776 SOCK_RECVBUF_UNLOCK(so); 1777 SOCK_UNLOCK(so); 1778 1779 so2 = sp->dst; 1780 SOCK_LOCK(so2); 1781 KASSERT(!SOLISTENING(so2), ("%s: so2 is listening", __func__)); 1782 SOCK_SENDBUF_LOCK(so2); 1783 KASSERT((so2->so_snd.sb_flags & SB_SPLICED) != 0, 1784 ("%s: so2 is not spliced", __func__)); 1785 KASSERT(so2->so_splice_back == sp, 1786 ("%s: so_splice_back != sp", __func__)); 1787 so2->so_snd.sb_flags &= ~SB_SPLICED; 1788 so2->so_splice_back = NULL; 1789 SOCK_SENDBUF_UNLOCK(so2); 1790 SOCK_UNLOCK(so2); 1791 1792 /* 1793 * No new work is being enqueued. The worker thread might be 1794 * splicing data right now, in which case we want to wait for it to 1795 * finish before proceeding. 1796 */ 1797 mtx_lock(&sp->mtx); 1798 switch (sp->state) { 1799 case SPLICE_QUEUED: 1800 case SPLICE_RUNNING: 1801 sp->state = SPLICE_CLOSING; 1802 while (sp->state == SPLICE_CLOSING) 1803 msleep(sp, &sp->mtx, PSOCK, "unsplice", 0); 1804 break; 1805 case SPLICE_IDLE: 1806 case SPLICE_EXCEPTION: 1807 sp->state = SPLICE_CLOSED; 1808 break; 1809 default: 1810 __assert_unreachable(); 1811 } 1812 if (!timeout) { 1813 drain = taskqueue_cancel_timeout(taskqueue_thread, &sp->timeout, 1814 NULL) != 0; 1815 } else { 1816 drain = false; 1817 } 1818 mtx_unlock(&sp->mtx); 1819 if (drain) 1820 taskqueue_drain_timeout(taskqueue_thread, &sp->timeout); 1821 1822 /* 1823 * Now we hold the sole reference to the splice structure. 1824 * Clean up: signal userspace and release socket references. 1825 */ 1826 sorwakeup(so); 1827 CURVNET_SET(so->so_vnet); 1828 sorele(so); 1829 sowwakeup(so2); 1830 sorele(so2); 1831 CURVNET_RESTORE(); 1832 so_splice_free(sp); 1833 return (0); 1834 } 1835 1836 /* 1837 * Free socket upon release of the very last reference. 1838 */ 1839 static void 1840 sofree(struct socket *so) 1841 { 1842 struct protosw *pr = so->so_proto; 1843 1844 SOCK_LOCK_ASSERT(so); 1845 KASSERT(refcount_load(&so->so_count) == 0, 1846 ("%s: so %p has references", __func__, so)); 1847 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE, 1848 ("%s: so %p is on listen queue", __func__, so)); 1849 KASSERT(SOLISTENING(so) || (so->so_rcv.sb_flags & SB_SPLICED) == 0, 1850 ("%s: so %p rcvbuf is spliced", __func__, so)); 1851 KASSERT(SOLISTENING(so) || (so->so_snd.sb_flags & SB_SPLICED) == 0, 1852 ("%s: so %p sndbuf is spliced", __func__, so)); 1853 KASSERT(so->so_splice == NULL && so->so_splice_back == NULL, 1854 ("%s: so %p has spliced data", __func__, so)); 1855 1856 SOCK_UNLOCK(so); 1857 1858 if (so->so_dtor != NULL) 1859 so->so_dtor(so); 1860 1861 VNET_SO_ASSERT(so); 1862 if (pr->pr_detach != NULL) 1863 pr->pr_detach(so); 1864 1865 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) { 1866 /* 1867 * From this point on, we assume that no other references to 1868 * this socket exist anywhere else in the stack. Therefore, 1869 * no locks need to be acquired or held. 1870 */ 1871 #ifdef INVARIANTS 1872 SOCK_SENDBUF_LOCK(so); 1873 SOCK_RECVBUF_LOCK(so); 1874 #endif 1875 sbdestroy(so, SO_SND); 1876 sbdestroy(so, SO_RCV); 1877 #ifdef INVARIANTS 1878 SOCK_SENDBUF_UNLOCK(so); 1879 SOCK_RECVBUF_UNLOCK(so); 1880 #endif 1881 } 1882 seldrain(&so->so_rdsel); 1883 seldrain(&so->so_wrsel); 1884 knlist_destroy(&so->so_rdsel.si_note); 1885 knlist_destroy(&so->so_wrsel.si_note); 1886 sodealloc(so); 1887 } 1888 1889 /* 1890 * Release a reference on a socket while holding the socket lock. 1891 * Unlocks the socket lock before returning. 1892 */ 1893 void 1894 sorele_locked(struct socket *so) 1895 { 1896 SOCK_LOCK_ASSERT(so); 1897 if (refcount_release(&so->so_count)) 1898 sofree(so); 1899 else 1900 SOCK_UNLOCK(so); 1901 } 1902 1903 /* 1904 * Close a socket on last file table reference removal. Initiate disconnect 1905 * if connected. Free socket when disconnect complete. 1906 * 1907 * This function will sorele() the socket. Note that soclose() may be called 1908 * prior to the ref count reaching zero. The actual socket structure will 1909 * not be freed until the ref count reaches zero. 1910 */ 1911 int 1912 soclose(struct socket *so) 1913 { 1914 struct accept_queue lqueue; 1915 int error = 0; 1916 bool listening, last __diagused; 1917 1918 CURVNET_SET(so->so_vnet); 1919 funsetown(&so->so_sigio); 1920 if (so->so_state & SS_ISCONNECTED) { 1921 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 1922 error = sodisconnect(so); 1923 if (error) { 1924 if (error == ENOTCONN) 1925 error = 0; 1926 goto drop; 1927 } 1928 } 1929 1930 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) { 1931 if ((so->so_state & SS_ISDISCONNECTING) && 1932 (so->so_state & SS_NBIO)) 1933 goto drop; 1934 while (so->so_state & SS_ISCONNECTED) { 1935 error = tsleep(&so->so_timeo, 1936 PSOCK | PCATCH, "soclos", 1937 so->so_linger * hz); 1938 if (error) 1939 break; 1940 } 1941 } 1942 } 1943 1944 drop: 1945 if (so->so_proto->pr_close != NULL) 1946 so->so_proto->pr_close(so); 1947 1948 SOCK_LOCK(so); 1949 if ((listening = SOLISTENING(so))) { 1950 struct socket *sp; 1951 1952 TAILQ_INIT(&lqueue); 1953 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list); 1954 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list); 1955 1956 so->sol_qlen = so->sol_incqlen = 0; 1957 1958 TAILQ_FOREACH(sp, &lqueue, so_list) { 1959 SOCK_LOCK(sp); 1960 sp->so_qstate = SQ_NONE; 1961 sp->so_listen = NULL; 1962 SOCK_UNLOCK(sp); 1963 last = refcount_release(&so->so_count); 1964 KASSERT(!last, ("%s: released last reference for %p", 1965 __func__, so)); 1966 } 1967 } 1968 sorele_locked(so); 1969 if (listening) { 1970 struct socket *sp, *tsp; 1971 1972 TAILQ_FOREACH_SAFE(sp, &lqueue, so_list, tsp) 1973 soabort(sp); 1974 } 1975 CURVNET_RESTORE(); 1976 return (error); 1977 } 1978 1979 /* 1980 * soabort() is used to abruptly tear down a connection, such as when a 1981 * resource limit is reached (listen queue depth exceeded), or if a listen 1982 * socket is closed while there are sockets waiting to be accepted. 1983 * 1984 * This interface is tricky, because it is called on an unreferenced socket, 1985 * and must be called only by a thread that has actually removed the socket 1986 * from the listen queue it was on. Likely this thread holds the last 1987 * reference on the socket and soabort() will proceed with sofree(). But 1988 * it might be not the last, as the sockets on the listen queues are seen 1989 * from the protocol side. 1990 * 1991 * This interface will call into the protocol code, so must not be called 1992 * with any socket locks held. Protocols do call it while holding their own 1993 * recursible protocol mutexes, but this is something that should be subject 1994 * to review in the future. 1995 * 1996 * Usually socket should have a single reference left, but this is not a 1997 * requirement. In the past, when we have had named references for file 1998 * descriptor and protocol, we asserted that none of them are being held. 1999 */ 2000 void 2001 soabort(struct socket *so) 2002 { 2003 2004 VNET_SO_ASSERT(so); 2005 2006 if (so->so_proto->pr_abort != NULL) 2007 so->so_proto->pr_abort(so); 2008 SOCK_LOCK(so); 2009 sorele_locked(so); 2010 } 2011 2012 int 2013 soaccept(struct socket *so, struct sockaddr *sa) 2014 { 2015 #ifdef INVARIANTS 2016 u_char len = sa->sa_len; 2017 #endif 2018 int error; 2019 2020 CURVNET_SET(so->so_vnet); 2021 error = so->so_proto->pr_accept(so, sa); 2022 KASSERT(sa->sa_len <= len, 2023 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2024 CURVNET_RESTORE(); 2025 return (error); 2026 } 2027 2028 int 2029 sopeeraddr(struct socket *so, struct sockaddr *sa) 2030 { 2031 #ifdef INVARIANTS 2032 u_char len = sa->sa_len; 2033 #endif 2034 int error; 2035 2036 CURVNET_ASSERT_SET(); 2037 2038 error = so->so_proto->pr_peeraddr(so, sa); 2039 KASSERT(sa->sa_len <= len, 2040 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2041 2042 return (error); 2043 } 2044 2045 int 2046 sosockaddr(struct socket *so, struct sockaddr *sa) 2047 { 2048 #ifdef INVARIANTS 2049 u_char len = sa->sa_len; 2050 #endif 2051 int error; 2052 2053 CURVNET_SET(so->so_vnet); 2054 error = so->so_proto->pr_sockaddr(so, sa); 2055 KASSERT(sa->sa_len <= len, 2056 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2057 CURVNET_RESTORE(); 2058 2059 return (error); 2060 } 2061 2062 int 2063 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) 2064 { 2065 2066 return (soconnectat(AT_FDCWD, so, nam, td)); 2067 } 2068 2069 int 2070 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 2071 { 2072 int error; 2073 2074 CURVNET_SET(so->so_vnet); 2075 2076 /* 2077 * If protocol is connection-based, can only connect once. 2078 * Otherwise, if connected, try to disconnect first. This allows 2079 * user to disconnect by connecting to, e.g., a null address. 2080 * 2081 * Note, this check is racy and may need to be re-evaluated at the 2082 * protocol layer. 2083 */ 2084 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 2085 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 2086 (error = sodisconnect(so)))) { 2087 error = EISCONN; 2088 } else { 2089 /* 2090 * Prevent accumulated error from previous connection from 2091 * biting us. 2092 */ 2093 so->so_error = 0; 2094 if (fd == AT_FDCWD) { 2095 error = so->so_proto->pr_connect(so, nam, td); 2096 } else { 2097 error = so->so_proto->pr_connectat(fd, so, nam, td); 2098 } 2099 } 2100 CURVNET_RESTORE(); 2101 2102 return (error); 2103 } 2104 2105 int 2106 soconnect2(struct socket *so1, struct socket *so2) 2107 { 2108 int error; 2109 2110 CURVNET_SET(so1->so_vnet); 2111 error = so1->so_proto->pr_connect2(so1, so2); 2112 CURVNET_RESTORE(); 2113 return (error); 2114 } 2115 2116 int 2117 sodisconnect(struct socket *so) 2118 { 2119 int error; 2120 2121 if ((so->so_state & SS_ISCONNECTED) == 0) 2122 return (ENOTCONN); 2123 if (so->so_state & SS_ISDISCONNECTING) 2124 return (EALREADY); 2125 VNET_SO_ASSERT(so); 2126 error = so->so_proto->pr_disconnect(so); 2127 return (error); 2128 } 2129 2130 int 2131 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 2132 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2133 { 2134 long space; 2135 ssize_t resid; 2136 int clen = 0, error, dontroute; 2137 2138 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM")); 2139 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 2140 ("sosend_dgram: !PR_ATOMIC")); 2141 2142 if (uio != NULL) 2143 resid = uio->uio_resid; 2144 else 2145 resid = top->m_pkthdr.len; 2146 /* 2147 * In theory resid should be unsigned. However, space must be 2148 * signed, as it might be less than 0 if we over-committed, and we 2149 * must use a signed comparison of space and resid. On the other 2150 * hand, a negative resid causes us to loop sending 0-length 2151 * segments to the protocol. 2152 */ 2153 if (resid < 0) { 2154 error = EINVAL; 2155 goto out; 2156 } 2157 2158 dontroute = 2159 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 2160 if (td != NULL) 2161 td->td_ru.ru_msgsnd++; 2162 if (control != NULL) 2163 clen = control->m_len; 2164 2165 SOCKBUF_LOCK(&so->so_snd); 2166 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2167 SOCKBUF_UNLOCK(&so->so_snd); 2168 error = EPIPE; 2169 goto out; 2170 } 2171 if (so->so_error) { 2172 error = so->so_error; 2173 so->so_error = 0; 2174 SOCKBUF_UNLOCK(&so->so_snd); 2175 goto out; 2176 } 2177 if ((so->so_state & SS_ISCONNECTED) == 0) { 2178 /* 2179 * `sendto' and `sendmsg' is allowed on a connection-based 2180 * socket if it supports implied connect. Return ENOTCONN if 2181 * not connected and no address is supplied. 2182 */ 2183 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 2184 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 2185 if (!(resid == 0 && clen != 0)) { 2186 SOCKBUF_UNLOCK(&so->so_snd); 2187 error = ENOTCONN; 2188 goto out; 2189 } 2190 } else if (addr == NULL) { 2191 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2192 error = ENOTCONN; 2193 else 2194 error = EDESTADDRREQ; 2195 SOCKBUF_UNLOCK(&so->so_snd); 2196 goto out; 2197 } 2198 } 2199 2200 /* 2201 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 2202 * problem and need fixing. 2203 */ 2204 space = sbspace(&so->so_snd); 2205 if (flags & MSG_OOB) 2206 space += 1024; 2207 space -= clen; 2208 SOCKBUF_UNLOCK(&so->so_snd); 2209 if (resid > space) { 2210 error = EMSGSIZE; 2211 goto out; 2212 } 2213 if (uio == NULL) { 2214 resid = 0; 2215 if (flags & MSG_EOR) 2216 top->m_flags |= M_EOR; 2217 } else { 2218 /* 2219 * Copy the data from userland into a mbuf chain. 2220 * If no data is to be copied in, a single empty mbuf 2221 * is returned. 2222 */ 2223 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 2224 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 2225 if (top == NULL) { 2226 error = EFAULT; /* only possible error */ 2227 goto out; 2228 } 2229 space -= resid - uio->uio_resid; 2230 resid = uio->uio_resid; 2231 } 2232 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 2233 /* 2234 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 2235 * than with. 2236 */ 2237 if (dontroute) { 2238 SOCK_LOCK(so); 2239 so->so_options |= SO_DONTROUTE; 2240 SOCK_UNLOCK(so); 2241 } 2242 /* 2243 * XXX all the SBS_CANTSENDMORE checks previously done could be out 2244 * of date. We could have received a reset packet in an interrupt or 2245 * maybe we slept while doing page faults in uiomove() etc. We could 2246 * probably recheck again inside the locking protection here, but 2247 * there are probably other places that this also happens. We must 2248 * rethink this. 2249 */ 2250 VNET_SO_ASSERT(so); 2251 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB : 2252 /* 2253 * If the user set MSG_EOF, the protocol understands this flag and 2254 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 2255 */ 2256 ((flags & MSG_EOF) && 2257 (so->so_proto->pr_flags & PR_IMPLOPCL) && 2258 (resid <= 0)) ? 2259 PRUS_EOF : 2260 /* If there is more to send set PRUS_MORETOCOME */ 2261 (flags & MSG_MORETOCOME) || 2262 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 2263 top, addr, control, td); 2264 if (dontroute) { 2265 SOCK_LOCK(so); 2266 so->so_options &= ~SO_DONTROUTE; 2267 SOCK_UNLOCK(so); 2268 } 2269 clen = 0; 2270 control = NULL; 2271 top = NULL; 2272 out: 2273 if (top != NULL) 2274 m_freem(top); 2275 if (control != NULL) 2276 m_freem(control); 2277 return (error); 2278 } 2279 2280 /* 2281 * Send on a socket. If send must go all at once and message is larger than 2282 * send buffering, then hard error. Lock against other senders. If must go 2283 * all at once and not enough room now, then inform user that this would 2284 * block and do nothing. Otherwise, if nonblocking, send as much as 2285 * possible. The data to be sent is described by "uio" if nonzero, otherwise 2286 * by the mbuf chain "top" (which must be null if uio is not). Data provided 2287 * in mbuf chain must be small enough to send all at once. 2288 * 2289 * Returns nonzero on error, timeout or signal; callers must check for short 2290 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 2291 * on return. 2292 */ 2293 static int 2294 sosend_generic_locked(struct socket *so, struct sockaddr *addr, struct uio *uio, 2295 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2296 { 2297 long space; 2298 ssize_t resid; 2299 int clen = 0, error, dontroute; 2300 int atomic = sosendallatonce(so) || top; 2301 int pr_send_flag; 2302 #ifdef KERN_TLS 2303 struct ktls_session *tls; 2304 int tls_enq_cnt, tls_send_flag; 2305 uint8_t tls_rtype; 2306 2307 tls = NULL; 2308 tls_rtype = TLS_RLTYPE_APP; 2309 #endif 2310 2311 SOCK_IO_SEND_ASSERT_LOCKED(so); 2312 2313 if (uio != NULL) 2314 resid = uio->uio_resid; 2315 else if ((top->m_flags & M_PKTHDR) != 0) 2316 resid = top->m_pkthdr.len; 2317 else 2318 resid = m_length(top, NULL); 2319 /* 2320 * In theory resid should be unsigned. However, space must be 2321 * signed, as it might be less than 0 if we over-committed, and we 2322 * must use a signed comparison of space and resid. On the other 2323 * hand, a negative resid causes us to loop sending 0-length 2324 * segments to the protocol. 2325 * 2326 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 2327 * type sockets since that's an error. 2328 */ 2329 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 2330 error = EINVAL; 2331 goto out; 2332 } 2333 2334 dontroute = 2335 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 2336 (so->so_proto->pr_flags & PR_ATOMIC); 2337 if (td != NULL) 2338 td->td_ru.ru_msgsnd++; 2339 if (control != NULL) 2340 clen = control->m_len; 2341 2342 #ifdef KERN_TLS 2343 tls_send_flag = 0; 2344 tls = ktls_hold(so->so_snd.sb_tls_info); 2345 if (tls != NULL) { 2346 if (tls->mode == TCP_TLS_MODE_SW) 2347 tls_send_flag = PRUS_NOTREADY; 2348 2349 if (control != NULL) { 2350 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 2351 2352 if (clen >= sizeof(*cm) && 2353 cm->cmsg_type == TLS_SET_RECORD_TYPE) { 2354 tls_rtype = *((uint8_t *)CMSG_DATA(cm)); 2355 clen = 0; 2356 m_freem(control); 2357 control = NULL; 2358 atomic = 1; 2359 } 2360 } 2361 2362 if (resid == 0 && !ktls_permit_empty_frames(tls)) { 2363 error = EINVAL; 2364 goto out; 2365 } 2366 } 2367 #endif 2368 2369 restart: 2370 do { 2371 SOCKBUF_LOCK(&so->so_snd); 2372 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2373 SOCKBUF_UNLOCK(&so->so_snd); 2374 error = EPIPE; 2375 goto out; 2376 } 2377 if (so->so_error) { 2378 error = so->so_error; 2379 so->so_error = 0; 2380 SOCKBUF_UNLOCK(&so->so_snd); 2381 goto out; 2382 } 2383 if ((so->so_state & SS_ISCONNECTED) == 0) { 2384 /* 2385 * `sendto' and `sendmsg' is allowed on a connection- 2386 * based socket if it supports implied connect. 2387 * Return ENOTCONN if not connected and no address is 2388 * supplied. 2389 */ 2390 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 2391 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 2392 if (!(resid == 0 && clen != 0)) { 2393 SOCKBUF_UNLOCK(&so->so_snd); 2394 error = ENOTCONN; 2395 goto out; 2396 } 2397 } else if (addr == NULL) { 2398 SOCKBUF_UNLOCK(&so->so_snd); 2399 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2400 error = ENOTCONN; 2401 else 2402 error = EDESTADDRREQ; 2403 goto out; 2404 } 2405 } 2406 space = sbspace(&so->so_snd); 2407 if (flags & MSG_OOB) 2408 space += 1024; 2409 if ((atomic && resid > so->so_snd.sb_hiwat) || 2410 clen > so->so_snd.sb_hiwat) { 2411 SOCKBUF_UNLOCK(&so->so_snd); 2412 error = EMSGSIZE; 2413 goto out; 2414 } 2415 if (space < resid + clen && 2416 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 2417 if ((so->so_state & SS_NBIO) || 2418 (flags & (MSG_NBIO | MSG_DONTWAIT)) != 0) { 2419 SOCKBUF_UNLOCK(&so->so_snd); 2420 error = EWOULDBLOCK; 2421 goto out; 2422 } 2423 error = sbwait(so, SO_SND); 2424 SOCKBUF_UNLOCK(&so->so_snd); 2425 if (error) 2426 goto out; 2427 goto restart; 2428 } 2429 SOCKBUF_UNLOCK(&so->so_snd); 2430 space -= clen; 2431 do { 2432 if (uio == NULL) { 2433 resid = 0; 2434 if (flags & MSG_EOR) 2435 top->m_flags |= M_EOR; 2436 #ifdef KERN_TLS 2437 if (tls != NULL) { 2438 ktls_frame(top, tls, &tls_enq_cnt, 2439 tls_rtype); 2440 tls_rtype = TLS_RLTYPE_APP; 2441 } 2442 #endif 2443 } else { 2444 /* 2445 * Copy the data from userland into a mbuf 2446 * chain. If resid is 0, which can happen 2447 * only if we have control to send, then 2448 * a single empty mbuf is returned. This 2449 * is a workaround to prevent protocol send 2450 * methods to panic. 2451 */ 2452 #ifdef KERN_TLS 2453 if (tls != NULL) { 2454 top = m_uiotombuf(uio, M_WAITOK, space, 2455 tls->params.max_frame_len, 2456 M_EXTPG | 2457 ((flags & MSG_EOR) ? M_EOR : 0)); 2458 if (top != NULL) { 2459 ktls_frame(top, tls, 2460 &tls_enq_cnt, tls_rtype); 2461 } 2462 tls_rtype = TLS_RLTYPE_APP; 2463 } else 2464 #endif 2465 top = m_uiotombuf(uio, M_WAITOK, space, 2466 (atomic ? max_hdr : 0), 2467 (atomic ? M_PKTHDR : 0) | 2468 ((flags & MSG_EOR) ? M_EOR : 0)); 2469 if (top == NULL) { 2470 error = EFAULT; /* only possible error */ 2471 goto out; 2472 } 2473 space -= resid - uio->uio_resid; 2474 resid = uio->uio_resid; 2475 } 2476 if (dontroute) { 2477 SOCK_LOCK(so); 2478 so->so_options |= SO_DONTROUTE; 2479 SOCK_UNLOCK(so); 2480 } 2481 /* 2482 * XXX all the SBS_CANTSENDMORE checks previously 2483 * done could be out of date. We could have received 2484 * a reset packet in an interrupt or maybe we slept 2485 * while doing page faults in uiomove() etc. We 2486 * could probably recheck again inside the locking 2487 * protection here, but there are probably other 2488 * places that this also happens. We must rethink 2489 * this. 2490 */ 2491 VNET_SO_ASSERT(so); 2492 2493 pr_send_flag = (flags & MSG_OOB) ? PRUS_OOB : 2494 /* 2495 * If the user set MSG_EOF, the protocol understands 2496 * this flag and nothing left to send then use 2497 * PRU_SEND_EOF instead of PRU_SEND. 2498 */ 2499 ((flags & MSG_EOF) && 2500 (so->so_proto->pr_flags & PR_IMPLOPCL) && 2501 (resid <= 0)) ? 2502 PRUS_EOF : 2503 /* If there is more to send set PRUS_MORETOCOME. */ 2504 (flags & MSG_MORETOCOME) || 2505 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0; 2506 2507 #ifdef KERN_TLS 2508 pr_send_flag |= tls_send_flag; 2509 #endif 2510 2511 error = so->so_proto->pr_send(so, pr_send_flag, top, 2512 addr, control, td); 2513 2514 if (dontroute) { 2515 SOCK_LOCK(so); 2516 so->so_options &= ~SO_DONTROUTE; 2517 SOCK_UNLOCK(so); 2518 } 2519 2520 #ifdef KERN_TLS 2521 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { 2522 if (error != 0) { 2523 m_freem(top); 2524 top = NULL; 2525 } else { 2526 soref(so); 2527 ktls_enqueue(top, so, tls_enq_cnt); 2528 } 2529 } 2530 #endif 2531 clen = 0; 2532 control = NULL; 2533 top = NULL; 2534 if (error) 2535 goto out; 2536 } while (resid && space > 0); 2537 } while (resid); 2538 2539 out: 2540 #ifdef KERN_TLS 2541 if (tls != NULL) 2542 ktls_free(tls); 2543 #endif 2544 if (top != NULL) 2545 m_freem(top); 2546 if (control != NULL) 2547 m_freem(control); 2548 return (error); 2549 } 2550 2551 int 2552 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, 2553 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2554 { 2555 int error; 2556 2557 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2558 if (error) 2559 return (error); 2560 error = sosend_generic_locked(so, addr, uio, top, control, flags, td); 2561 SOCK_IO_SEND_UNLOCK(so); 2562 return (error); 2563 } 2564 2565 /* 2566 * Send to a socket from a kernel thread. 2567 * 2568 * XXXGL: in almost all cases uio is NULL and the mbuf is supplied. 2569 * Exception is nfs/bootp_subr.c. It is arguable that the VNET context needs 2570 * to be set at all. This function should just boil down to a static inline 2571 * calling the protocol method. 2572 */ 2573 int 2574 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, 2575 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2576 { 2577 int error; 2578 2579 CURVNET_SET(so->so_vnet); 2580 error = so->so_proto->pr_sosend(so, addr, uio, 2581 top, control, flags, td); 2582 CURVNET_RESTORE(); 2583 return (error); 2584 } 2585 2586 /* 2587 * send(2), write(2) or aio_write(2) on a socket. 2588 */ 2589 int 2590 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio, 2591 struct mbuf *control, int flags, struct proc *userproc) 2592 { 2593 struct thread *td; 2594 ssize_t len; 2595 int error; 2596 2597 td = uio->uio_td; 2598 len = uio->uio_resid; 2599 CURVNET_SET(so->so_vnet); 2600 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags, 2601 td); 2602 CURVNET_RESTORE(); 2603 if (error != 0) { 2604 /* 2605 * Clear transient errors for stream protocols if they made 2606 * some progress. Make exclusion for aio(4) that would 2607 * schedule a new write in case of EWOULDBLOCK and clear 2608 * error itself. See soaio_process_job(). 2609 */ 2610 if (uio->uio_resid != len && 2611 (so->so_proto->pr_flags & PR_ATOMIC) == 0 && 2612 userproc == NULL && 2613 (error == ERESTART || error == EINTR || 2614 error == EWOULDBLOCK)) 2615 error = 0; 2616 /* Generation of SIGPIPE can be controlled per socket. */ 2617 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 && 2618 (flags & MSG_NOSIGNAL) == 0) { 2619 if (userproc != NULL) { 2620 /* aio(4) job */ 2621 PROC_LOCK(userproc); 2622 kern_psignal(userproc, SIGPIPE); 2623 PROC_UNLOCK(userproc); 2624 } else { 2625 PROC_LOCK(td->td_proc); 2626 tdsignal(td, SIGPIPE); 2627 PROC_UNLOCK(td->td_proc); 2628 } 2629 } 2630 } 2631 return (error); 2632 } 2633 2634 /* 2635 * The part of soreceive() that implements reading non-inline out-of-band 2636 * data from a socket. For more complete comments, see soreceive(), from 2637 * which this code originated. 2638 * 2639 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 2640 * unable to return an mbuf chain to the caller. 2641 */ 2642 static int 2643 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) 2644 { 2645 struct protosw *pr = so->so_proto; 2646 struct mbuf *m; 2647 int error; 2648 2649 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 2650 VNET_SO_ASSERT(so); 2651 2652 m = m_get(M_WAITOK, MT_DATA); 2653 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK); 2654 if (error) 2655 goto bad; 2656 do { 2657 error = uiomove(mtod(m, void *), 2658 (int) min(uio->uio_resid, m->m_len), uio); 2659 m = m_free(m); 2660 } while (uio->uio_resid && error == 0 && m); 2661 bad: 2662 if (m != NULL) 2663 m_freem(m); 2664 return (error); 2665 } 2666 2667 /* 2668 * Following replacement or removal of the first mbuf on the first mbuf chain 2669 * of a socket buffer, push necessary state changes back into the socket 2670 * buffer so that other consumers see the values consistently. 'nextrecord' 2671 * is the callers locally stored value of the original value of 2672 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 2673 * NOTE: 'nextrecord' may be NULL. 2674 */ 2675 static __inline void 2676 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 2677 { 2678 2679 SOCKBUF_LOCK_ASSERT(sb); 2680 /* 2681 * First, update for the new value of nextrecord. If necessary, make 2682 * it the first record. 2683 */ 2684 if (sb->sb_mb != NULL) 2685 sb->sb_mb->m_nextpkt = nextrecord; 2686 else 2687 sb->sb_mb = nextrecord; 2688 2689 /* 2690 * Now update any dependent socket buffer fields to reflect the new 2691 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 2692 * addition of a second clause that takes care of the case where 2693 * sb_mb has been updated, but remains the last record. 2694 */ 2695 if (sb->sb_mb == NULL) { 2696 sb->sb_mbtail = NULL; 2697 sb->sb_lastrecord = NULL; 2698 } else if (sb->sb_mb->m_nextpkt == NULL) 2699 sb->sb_lastrecord = sb->sb_mb; 2700 } 2701 2702 /* 2703 * Implement receive operations on a socket. We depend on the way that 2704 * records are added to the sockbuf by sbappend. In particular, each record 2705 * (mbufs linked through m_next) must begin with an address if the protocol 2706 * so specifies, followed by an optional mbuf or mbufs containing ancillary 2707 * data, and then zero or more mbufs of data. In order to allow parallelism 2708 * between network receive and copying to user space, as well as avoid 2709 * sleeping with a mutex held, we release the socket buffer mutex during the 2710 * user space copy. Although the sockbuf is locked, new data may still be 2711 * appended, and thus we must maintain consistency of the sockbuf during that 2712 * time. 2713 * 2714 * The caller may receive the data as a single mbuf chain by supplying an 2715 * mbuf **mp0 for use in returning the chain. The uio is then used only for 2716 * the count in uio_resid. 2717 */ 2718 static int 2719 soreceive_generic_locked(struct socket *so, struct sockaddr **psa, 2720 struct uio *uio, struct mbuf **mp, struct mbuf **controlp, int *flagsp) 2721 { 2722 struct mbuf *m; 2723 int flags, error, offset; 2724 ssize_t len; 2725 struct protosw *pr = so->so_proto; 2726 struct mbuf *nextrecord; 2727 int moff, type = 0; 2728 ssize_t orig_resid = uio->uio_resid; 2729 bool report_real_len = false; 2730 2731 SOCK_IO_RECV_ASSERT_LOCKED(so); 2732 2733 error = 0; 2734 if (flagsp != NULL) { 2735 report_real_len = *flagsp & MSG_TRUNC; 2736 *flagsp &= ~MSG_TRUNC; 2737 flags = *flagsp &~ MSG_EOR; 2738 } else 2739 flags = 0; 2740 2741 restart: 2742 SOCKBUF_LOCK(&so->so_rcv); 2743 m = so->so_rcv.sb_mb; 2744 /* 2745 * If we have less data than requested, block awaiting more (subject 2746 * to any timeout) if: 2747 * 1. the current count is less than the low water mark, or 2748 * 2. MSG_DONTWAIT is not set 2749 */ 2750 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 2751 sbavail(&so->so_rcv) < uio->uio_resid) && 2752 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat && 2753 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 2754 KASSERT(m != NULL || !sbavail(&so->so_rcv), 2755 ("receive: m == %p sbavail == %u", 2756 m, sbavail(&so->so_rcv))); 2757 if (so->so_error || so->so_rerror) { 2758 if (m != NULL) 2759 goto dontblock; 2760 if (so->so_error) 2761 error = so->so_error; 2762 else 2763 error = so->so_rerror; 2764 if ((flags & MSG_PEEK) == 0) { 2765 if (so->so_error) 2766 so->so_error = 0; 2767 else 2768 so->so_rerror = 0; 2769 } 2770 SOCKBUF_UNLOCK(&so->so_rcv); 2771 goto release; 2772 } 2773 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2774 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 2775 if (m != NULL) 2776 goto dontblock; 2777 #ifdef KERN_TLS 2778 else if (so->so_rcv.sb_tlsdcc == 0 && 2779 so->so_rcv.sb_tlscc == 0) { 2780 #else 2781 else { 2782 #endif 2783 SOCKBUF_UNLOCK(&so->so_rcv); 2784 goto release; 2785 } 2786 } 2787 for (; m != NULL; m = m->m_next) 2788 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 2789 m = so->so_rcv.sb_mb; 2790 goto dontblock; 2791 } 2792 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED | 2793 SS_ISDISCONNECTING | SS_ISDISCONNECTED)) == 0 && 2794 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) { 2795 SOCKBUF_UNLOCK(&so->so_rcv); 2796 error = ENOTCONN; 2797 goto release; 2798 } 2799 if (uio->uio_resid == 0 && !report_real_len) { 2800 SOCKBUF_UNLOCK(&so->so_rcv); 2801 goto release; 2802 } 2803 if ((so->so_state & SS_NBIO) || 2804 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 2805 SOCKBUF_UNLOCK(&so->so_rcv); 2806 error = EWOULDBLOCK; 2807 goto release; 2808 } 2809 SBLASTRECORDCHK(&so->so_rcv); 2810 SBLASTMBUFCHK(&so->so_rcv); 2811 error = sbwait(so, SO_RCV); 2812 SOCKBUF_UNLOCK(&so->so_rcv); 2813 if (error) 2814 goto release; 2815 goto restart; 2816 } 2817 dontblock: 2818 /* 2819 * From this point onward, we maintain 'nextrecord' as a cache of the 2820 * pointer to the next record in the socket buffer. We must keep the 2821 * various socket buffer pointers and local stack versions of the 2822 * pointers in sync, pushing out modifications before dropping the 2823 * socket buffer mutex, and re-reading them when picking it up. 2824 * 2825 * Otherwise, we will race with the network stack appending new data 2826 * or records onto the socket buffer by using inconsistent/stale 2827 * versions of the field, possibly resulting in socket buffer 2828 * corruption. 2829 * 2830 * By holding the high-level sblock(), we prevent simultaneous 2831 * readers from pulling off the front of the socket buffer. 2832 */ 2833 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2834 if (uio->uio_td) 2835 uio->uio_td->td_ru.ru_msgrcv++; 2836 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 2837 SBLASTRECORDCHK(&so->so_rcv); 2838 SBLASTMBUFCHK(&so->so_rcv); 2839 nextrecord = m->m_nextpkt; 2840 if (pr->pr_flags & PR_ADDR) { 2841 KASSERT(m->m_type == MT_SONAME, 2842 ("m->m_type == %d", m->m_type)); 2843 orig_resid = 0; 2844 if (psa != NULL) 2845 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 2846 M_NOWAIT); 2847 if (flags & MSG_PEEK) { 2848 m = m->m_next; 2849 } else { 2850 sbfree(&so->so_rcv, m); 2851 so->so_rcv.sb_mb = m_free(m); 2852 m = so->so_rcv.sb_mb; 2853 sockbuf_pushsync(&so->so_rcv, nextrecord); 2854 } 2855 } 2856 2857 /* 2858 * Process one or more MT_CONTROL mbufs present before any data mbufs 2859 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 2860 * just copy the data; if !MSG_PEEK, we call into the protocol to 2861 * perform externalization (or freeing if controlp == NULL). 2862 */ 2863 if (m != NULL && m->m_type == MT_CONTROL) { 2864 struct mbuf *cm = NULL, *cmn; 2865 struct mbuf **cme = &cm; 2866 #ifdef KERN_TLS 2867 struct cmsghdr *cmsg; 2868 struct tls_get_record tgr; 2869 2870 /* 2871 * For MSG_TLSAPPDATA, check for an alert record. 2872 * If found, return ENXIO without removing 2873 * it from the receive queue. This allows a subsequent 2874 * call without MSG_TLSAPPDATA to receive it. 2875 * Note that, for TLS, there should only be a single 2876 * control mbuf with the TLS_GET_RECORD message in it. 2877 */ 2878 if (flags & MSG_TLSAPPDATA) { 2879 cmsg = mtod(m, struct cmsghdr *); 2880 if (cmsg->cmsg_type == TLS_GET_RECORD && 2881 cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) { 2882 memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr)); 2883 if (__predict_false(tgr.tls_type == 2884 TLS_RLTYPE_ALERT)) { 2885 SOCKBUF_UNLOCK(&so->so_rcv); 2886 error = ENXIO; 2887 goto release; 2888 } 2889 } 2890 } 2891 #endif 2892 2893 do { 2894 if (flags & MSG_PEEK) { 2895 if (controlp != NULL) { 2896 *controlp = m_copym(m, 0, m->m_len, 2897 M_NOWAIT); 2898 controlp = &(*controlp)->m_next; 2899 } 2900 m = m->m_next; 2901 } else { 2902 sbfree(&so->so_rcv, m); 2903 so->so_rcv.sb_mb = m->m_next; 2904 m->m_next = NULL; 2905 *cme = m; 2906 cme = &(*cme)->m_next; 2907 m = so->so_rcv.sb_mb; 2908 } 2909 } while (m != NULL && m->m_type == MT_CONTROL); 2910 if ((flags & MSG_PEEK) == 0) 2911 sockbuf_pushsync(&so->so_rcv, nextrecord); 2912 while (cm != NULL) { 2913 cmn = cm->m_next; 2914 cm->m_next = NULL; 2915 if (pr->pr_domain->dom_externalize != NULL) { 2916 SOCKBUF_UNLOCK(&so->so_rcv); 2917 VNET_SO_ASSERT(so); 2918 error = (*pr->pr_domain->dom_externalize) 2919 (cm, controlp, flags); 2920 SOCKBUF_LOCK(&so->so_rcv); 2921 } else if (controlp != NULL) 2922 *controlp = cm; 2923 else 2924 m_freem(cm); 2925 if (controlp != NULL) { 2926 while (*controlp != NULL) 2927 controlp = &(*controlp)->m_next; 2928 } 2929 cm = cmn; 2930 } 2931 if (m != NULL) 2932 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 2933 else 2934 nextrecord = so->so_rcv.sb_mb; 2935 orig_resid = 0; 2936 } 2937 if (m != NULL) { 2938 if ((flags & MSG_PEEK) == 0) { 2939 KASSERT(m->m_nextpkt == nextrecord, 2940 ("soreceive: post-control, nextrecord !sync")); 2941 if (nextrecord == NULL) { 2942 KASSERT(so->so_rcv.sb_mb == m, 2943 ("soreceive: post-control, sb_mb!=m")); 2944 KASSERT(so->so_rcv.sb_lastrecord == m, 2945 ("soreceive: post-control, lastrecord!=m")); 2946 } 2947 } 2948 type = m->m_type; 2949 if (type == MT_OOBDATA) 2950 flags |= MSG_OOB; 2951 } else { 2952 if ((flags & MSG_PEEK) == 0) { 2953 KASSERT(so->so_rcv.sb_mb == nextrecord, 2954 ("soreceive: sb_mb != nextrecord")); 2955 if (so->so_rcv.sb_mb == NULL) { 2956 KASSERT(so->so_rcv.sb_lastrecord == NULL, 2957 ("soreceive: sb_lastercord != NULL")); 2958 } 2959 } 2960 } 2961 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2962 SBLASTRECORDCHK(&so->so_rcv); 2963 SBLASTMBUFCHK(&so->so_rcv); 2964 2965 /* 2966 * Now continue to read any data mbufs off of the head of the socket 2967 * buffer until the read request is satisfied. Note that 'type' is 2968 * used to store the type of any mbuf reads that have happened so far 2969 * such that soreceive() can stop reading if the type changes, which 2970 * causes soreceive() to return only one of regular data and inline 2971 * out-of-band data in a single socket receive operation. 2972 */ 2973 moff = 0; 2974 offset = 0; 2975 while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0 2976 && error == 0) { 2977 /* 2978 * If the type of mbuf has changed since the last mbuf 2979 * examined ('type'), end the receive operation. 2980 */ 2981 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2982 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) { 2983 if (type != m->m_type) 2984 break; 2985 } else if (type == MT_OOBDATA) 2986 break; 2987 else 2988 KASSERT(m->m_type == MT_DATA, 2989 ("m->m_type == %d", m->m_type)); 2990 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 2991 len = uio->uio_resid; 2992 if (so->so_oobmark && len > so->so_oobmark - offset) 2993 len = so->so_oobmark - offset; 2994 if (len > m->m_len - moff) 2995 len = m->m_len - moff; 2996 /* 2997 * If mp is set, just pass back the mbufs. Otherwise copy 2998 * them out via the uio, then free. Sockbuf must be 2999 * consistent here (points to current mbuf, it points to next 3000 * record) when we drop priority; we must note any additions 3001 * to the sockbuf when we block interrupts again. 3002 */ 3003 if (mp == NULL) { 3004 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3005 SBLASTRECORDCHK(&so->so_rcv); 3006 SBLASTMBUFCHK(&so->so_rcv); 3007 SOCKBUF_UNLOCK(&so->so_rcv); 3008 if ((m->m_flags & M_EXTPG) != 0) 3009 error = m_unmapped_uiomove(m, moff, uio, 3010 (int)len); 3011 else 3012 error = uiomove(mtod(m, char *) + moff, 3013 (int)len, uio); 3014 SOCKBUF_LOCK(&so->so_rcv); 3015 if (error) { 3016 /* 3017 * The MT_SONAME mbuf has already been removed 3018 * from the record, so it is necessary to 3019 * remove the data mbufs, if any, to preserve 3020 * the invariant in the case of PR_ADDR that 3021 * requires MT_SONAME mbufs at the head of 3022 * each record. 3023 */ 3024 if (pr->pr_flags & PR_ATOMIC && 3025 ((flags & MSG_PEEK) == 0)) 3026 (void)sbdroprecord_locked(&so->so_rcv); 3027 SOCKBUF_UNLOCK(&so->so_rcv); 3028 goto release; 3029 } 3030 } else 3031 uio->uio_resid -= len; 3032 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3033 if (len == m->m_len - moff) { 3034 if (m->m_flags & M_EOR) 3035 flags |= MSG_EOR; 3036 if (flags & MSG_PEEK) { 3037 m = m->m_next; 3038 moff = 0; 3039 } else { 3040 nextrecord = m->m_nextpkt; 3041 sbfree(&so->so_rcv, m); 3042 if (mp != NULL) { 3043 m->m_nextpkt = NULL; 3044 *mp = m; 3045 mp = &m->m_next; 3046 so->so_rcv.sb_mb = m = m->m_next; 3047 *mp = NULL; 3048 } else { 3049 so->so_rcv.sb_mb = m_free(m); 3050 m = so->so_rcv.sb_mb; 3051 } 3052 sockbuf_pushsync(&so->so_rcv, nextrecord); 3053 SBLASTRECORDCHK(&so->so_rcv); 3054 SBLASTMBUFCHK(&so->so_rcv); 3055 } 3056 } else { 3057 if (flags & MSG_PEEK) 3058 moff += len; 3059 else { 3060 if (mp != NULL) { 3061 if (flags & MSG_DONTWAIT) { 3062 *mp = m_copym(m, 0, len, 3063 M_NOWAIT); 3064 if (*mp == NULL) { 3065 /* 3066 * m_copym() couldn't 3067 * allocate an mbuf. 3068 * Adjust uio_resid back 3069 * (it was adjusted 3070 * down by len bytes, 3071 * which we didn't end 3072 * up "copying" over). 3073 */ 3074 uio->uio_resid += len; 3075 break; 3076 } 3077 } else { 3078 SOCKBUF_UNLOCK(&so->so_rcv); 3079 *mp = m_copym(m, 0, len, 3080 M_WAITOK); 3081 SOCKBUF_LOCK(&so->so_rcv); 3082 } 3083 } 3084 sbcut_locked(&so->so_rcv, len); 3085 } 3086 } 3087 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3088 if (so->so_oobmark) { 3089 if ((flags & MSG_PEEK) == 0) { 3090 so->so_oobmark -= len; 3091 if (so->so_oobmark == 0) { 3092 so->so_rcv.sb_state |= SBS_RCVATMARK; 3093 break; 3094 } 3095 } else { 3096 offset += len; 3097 if (offset == so->so_oobmark) 3098 break; 3099 } 3100 } 3101 if (flags & MSG_EOR) 3102 break; 3103 /* 3104 * If the MSG_WAITALL flag is set (for non-atomic socket), we 3105 * must not quit until "uio->uio_resid == 0" or an error 3106 * termination. If a signal/timeout occurs, return with a 3107 * short count but without error. Keep sockbuf locked 3108 * against other readers. 3109 */ 3110 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 3111 !sosendallatonce(so) && nextrecord == NULL) { 3112 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3113 if (so->so_error || so->so_rerror || 3114 so->so_rcv.sb_state & SBS_CANTRCVMORE) 3115 break; 3116 /* 3117 * Notify the protocol that some data has been 3118 * drained before blocking. 3119 */ 3120 if (pr->pr_flags & PR_WANTRCVD) { 3121 SOCKBUF_UNLOCK(&so->so_rcv); 3122 VNET_SO_ASSERT(so); 3123 pr->pr_rcvd(so, flags); 3124 SOCKBUF_LOCK(&so->so_rcv); 3125 if (__predict_false(so->so_rcv.sb_mb == NULL && 3126 (so->so_error || so->so_rerror || 3127 so->so_rcv.sb_state & SBS_CANTRCVMORE))) 3128 break; 3129 } 3130 SBLASTRECORDCHK(&so->so_rcv); 3131 SBLASTMBUFCHK(&so->so_rcv); 3132 /* 3133 * We could receive some data while was notifying 3134 * the protocol. Skip blocking in this case. 3135 */ 3136 if (so->so_rcv.sb_mb == NULL) { 3137 error = sbwait(so, SO_RCV); 3138 if (error) { 3139 SOCKBUF_UNLOCK(&so->so_rcv); 3140 goto release; 3141 } 3142 } 3143 m = so->so_rcv.sb_mb; 3144 if (m != NULL) 3145 nextrecord = m->m_nextpkt; 3146 } 3147 } 3148 3149 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3150 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 3151 if (report_real_len) 3152 uio->uio_resid -= m_length(m, NULL) - moff; 3153 flags |= MSG_TRUNC; 3154 if ((flags & MSG_PEEK) == 0) 3155 (void) sbdroprecord_locked(&so->so_rcv); 3156 } 3157 if ((flags & MSG_PEEK) == 0) { 3158 if (m == NULL) { 3159 /* 3160 * First part is an inline SB_EMPTY_FIXUP(). Second 3161 * part makes sure sb_lastrecord is up-to-date if 3162 * there is still data in the socket buffer. 3163 */ 3164 so->so_rcv.sb_mb = nextrecord; 3165 if (so->so_rcv.sb_mb == NULL) { 3166 so->so_rcv.sb_mbtail = NULL; 3167 so->so_rcv.sb_lastrecord = NULL; 3168 } else if (nextrecord->m_nextpkt == NULL) 3169 so->so_rcv.sb_lastrecord = nextrecord; 3170 } 3171 SBLASTRECORDCHK(&so->so_rcv); 3172 SBLASTMBUFCHK(&so->so_rcv); 3173 /* 3174 * If soreceive() is being done from the socket callback, 3175 * then don't need to generate ACK to peer to update window, 3176 * since ACK will be generated on return to TCP. 3177 */ 3178 if (!(flags & MSG_SOCALLBCK) && 3179 (pr->pr_flags & PR_WANTRCVD)) { 3180 SOCKBUF_UNLOCK(&so->so_rcv); 3181 VNET_SO_ASSERT(so); 3182 pr->pr_rcvd(so, flags); 3183 SOCKBUF_LOCK(&so->so_rcv); 3184 } 3185 } 3186 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3187 if (orig_resid == uio->uio_resid && orig_resid && 3188 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 3189 SOCKBUF_UNLOCK(&so->so_rcv); 3190 goto restart; 3191 } 3192 SOCKBUF_UNLOCK(&so->so_rcv); 3193 3194 if (flagsp != NULL) 3195 *flagsp |= flags; 3196 release: 3197 return (error); 3198 } 3199 3200 int 3201 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, 3202 struct mbuf **mp, struct mbuf **controlp, int *flagsp) 3203 { 3204 int error, flags; 3205 3206 if (psa != NULL) 3207 *psa = NULL; 3208 if (controlp != NULL) 3209 *controlp = NULL; 3210 if (flagsp != NULL) { 3211 flags = *flagsp; 3212 if ((flags & MSG_OOB) != 0) 3213 return (soreceive_rcvoob(so, uio, flags)); 3214 } else { 3215 flags = 0; 3216 } 3217 if (mp != NULL) 3218 *mp = NULL; 3219 3220 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 3221 if (error) 3222 return (error); 3223 error = soreceive_generic_locked(so, psa, uio, mp, controlp, flagsp); 3224 SOCK_IO_RECV_UNLOCK(so); 3225 return (error); 3226 } 3227 3228 /* 3229 * Optimized version of soreceive() for stream (TCP) sockets. 3230 */ 3231 static int 3232 soreceive_stream_locked(struct socket *so, struct sockbuf *sb, 3233 struct sockaddr **psa, struct uio *uio, struct mbuf **mp0, 3234 struct mbuf **controlp, int flags) 3235 { 3236 int len = 0, error = 0, oresid; 3237 struct mbuf *m, *n = NULL; 3238 3239 SOCK_IO_RECV_ASSERT_LOCKED(so); 3240 3241 /* Easy one, no space to copyout anything. */ 3242 if (uio->uio_resid == 0) 3243 return (EINVAL); 3244 oresid = uio->uio_resid; 3245 3246 SOCKBUF_LOCK(sb); 3247 /* We will never ever get anything unless we are or were connected. */ 3248 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { 3249 error = ENOTCONN; 3250 goto out; 3251 } 3252 3253 restart: 3254 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3255 3256 /* Abort if socket has reported problems. */ 3257 if (so->so_error) { 3258 if (sbavail(sb) > 0) 3259 goto deliver; 3260 if (oresid > uio->uio_resid) 3261 goto out; 3262 error = so->so_error; 3263 if (!(flags & MSG_PEEK)) 3264 so->so_error = 0; 3265 goto out; 3266 } 3267 3268 /* Door is closed. Deliver what is left, if any. */ 3269 if (sb->sb_state & SBS_CANTRCVMORE) { 3270 if (sbavail(sb) > 0) 3271 goto deliver; 3272 else 3273 goto out; 3274 } 3275 3276 /* Socket buffer is empty and we shall not block. */ 3277 if (sbavail(sb) == 0 && 3278 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 3279 error = EAGAIN; 3280 goto out; 3281 } 3282 3283 /* Socket buffer got some data that we shall deliver now. */ 3284 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) && 3285 ((so->so_state & SS_NBIO) || 3286 (flags & (MSG_DONTWAIT|MSG_NBIO)) || 3287 sbavail(sb) >= sb->sb_lowat || 3288 sbavail(sb) >= uio->uio_resid || 3289 sbavail(sb) >= sb->sb_hiwat) ) { 3290 goto deliver; 3291 } 3292 3293 /* On MSG_WAITALL we must wait until all data or error arrives. */ 3294 if ((flags & MSG_WAITALL) && 3295 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat)) 3296 goto deliver; 3297 3298 /* 3299 * Wait and block until (more) data comes in. 3300 * NB: Drops the sockbuf lock during wait. 3301 */ 3302 error = sbwait(so, SO_RCV); 3303 if (error) 3304 goto out; 3305 goto restart; 3306 3307 deliver: 3308 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3309 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__)); 3310 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 3311 3312 /* Statistics. */ 3313 if (uio->uio_td) 3314 uio->uio_td->td_ru.ru_msgrcv++; 3315 3316 /* Fill uio until full or current end of socket buffer is reached. */ 3317 len = min(uio->uio_resid, sbavail(sb)); 3318 if (mp0 != NULL) { 3319 /* Dequeue as many mbufs as possible. */ 3320 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 3321 if (*mp0 == NULL) 3322 *mp0 = sb->sb_mb; 3323 else 3324 m_cat(*mp0, sb->sb_mb); 3325 for (m = sb->sb_mb; 3326 m != NULL && m->m_len <= len; 3327 m = m->m_next) { 3328 KASSERT(!(m->m_flags & M_NOTAVAIL), 3329 ("%s: m %p not available", __func__, m)); 3330 len -= m->m_len; 3331 uio->uio_resid -= m->m_len; 3332 sbfree(sb, m); 3333 n = m; 3334 } 3335 n->m_next = NULL; 3336 sb->sb_mb = m; 3337 sb->sb_lastrecord = sb->sb_mb; 3338 if (sb->sb_mb == NULL) 3339 SB_EMPTY_FIXUP(sb); 3340 } 3341 /* Copy the remainder. */ 3342 if (len > 0) { 3343 KASSERT(sb->sb_mb != NULL, 3344 ("%s: len > 0 && sb->sb_mb empty", __func__)); 3345 3346 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT); 3347 if (m == NULL) 3348 len = 0; /* Don't flush data from sockbuf. */ 3349 else 3350 uio->uio_resid -= len; 3351 if (*mp0 != NULL) 3352 m_cat(*mp0, m); 3353 else 3354 *mp0 = m; 3355 if (*mp0 == NULL) { 3356 error = ENOBUFS; 3357 goto out; 3358 } 3359 } 3360 } else { 3361 /* NB: Must unlock socket buffer as uiomove may sleep. */ 3362 SOCKBUF_UNLOCK(sb); 3363 error = m_mbuftouio(uio, sb->sb_mb, len); 3364 SOCKBUF_LOCK(sb); 3365 if (error) 3366 goto out; 3367 } 3368 SBLASTRECORDCHK(sb); 3369 SBLASTMBUFCHK(sb); 3370 3371 /* 3372 * Remove the delivered data from the socket buffer unless we 3373 * were only peeking. 3374 */ 3375 if (!(flags & MSG_PEEK)) { 3376 if (len > 0) 3377 sbdrop_locked(sb, len); 3378 3379 /* Notify protocol that we drained some data. */ 3380 if ((so->so_proto->pr_flags & PR_WANTRCVD) && 3381 (((flags & MSG_WAITALL) && uio->uio_resid > 0) || 3382 !(flags & MSG_SOCALLBCK))) { 3383 SOCKBUF_UNLOCK(sb); 3384 VNET_SO_ASSERT(so); 3385 so->so_proto->pr_rcvd(so, flags); 3386 SOCKBUF_LOCK(sb); 3387 } 3388 } 3389 3390 /* 3391 * For MSG_WAITALL we may have to loop again and wait for 3392 * more data to come in. 3393 */ 3394 if ((flags & MSG_WAITALL) && uio->uio_resid > 0) 3395 goto restart; 3396 out: 3397 SBLASTRECORDCHK(sb); 3398 SBLASTMBUFCHK(sb); 3399 SOCKBUF_UNLOCK(sb); 3400 return (error); 3401 } 3402 3403 int 3404 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, 3405 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3406 { 3407 struct sockbuf *sb; 3408 int error, flags; 3409 3410 sb = &so->so_rcv; 3411 3412 /* We only do stream sockets. */ 3413 if (so->so_type != SOCK_STREAM) 3414 return (EINVAL); 3415 if (psa != NULL) 3416 *psa = NULL; 3417 if (flagsp != NULL) 3418 flags = *flagsp & ~MSG_EOR; 3419 else 3420 flags = 0; 3421 if (controlp != NULL) 3422 *controlp = NULL; 3423 if (flags & MSG_OOB) 3424 return (soreceive_rcvoob(so, uio, flags)); 3425 if (mp0 != NULL) 3426 *mp0 = NULL; 3427 3428 #ifdef KERN_TLS 3429 /* 3430 * KTLS store TLS records as records with a control message to 3431 * describe the framing. 3432 * 3433 * We check once here before acquiring locks to optimize the 3434 * common case. 3435 */ 3436 if (sb->sb_tls_info != NULL) 3437 return (soreceive_generic(so, psa, uio, mp0, controlp, 3438 flagsp)); 3439 #endif 3440 3441 /* 3442 * Prevent other threads from reading from the socket. This lock may be 3443 * dropped in order to sleep waiting for data to arrive. 3444 */ 3445 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 3446 if (error) 3447 return (error); 3448 #ifdef KERN_TLS 3449 if (__predict_false(sb->sb_tls_info != NULL)) { 3450 SOCK_IO_RECV_UNLOCK(so); 3451 return (soreceive_generic(so, psa, uio, mp0, controlp, 3452 flagsp)); 3453 } 3454 #endif 3455 error = soreceive_stream_locked(so, sb, psa, uio, mp0, controlp, flags); 3456 SOCK_IO_RECV_UNLOCK(so); 3457 return (error); 3458 } 3459 3460 /* 3461 * Optimized version of soreceive() for simple datagram cases from userspace. 3462 * Unlike in the stream case, we're able to drop a datagram if copyout() 3463 * fails, and because we handle datagrams atomically, we don't need to use a 3464 * sleep lock to prevent I/O interlacing. 3465 */ 3466 int 3467 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 3468 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3469 { 3470 struct mbuf *m, *m2; 3471 int flags, error; 3472 ssize_t len; 3473 struct protosw *pr = so->so_proto; 3474 struct mbuf *nextrecord; 3475 3476 if (psa != NULL) 3477 *psa = NULL; 3478 if (controlp != NULL) 3479 *controlp = NULL; 3480 if (flagsp != NULL) 3481 flags = *flagsp &~ MSG_EOR; 3482 else 3483 flags = 0; 3484 3485 /* 3486 * For any complicated cases, fall back to the full 3487 * soreceive_generic(). 3488 */ 3489 if (mp0 != NULL || (flags & (MSG_PEEK | MSG_OOB | MSG_TRUNC))) 3490 return (soreceive_generic(so, psa, uio, mp0, controlp, 3491 flagsp)); 3492 3493 /* 3494 * Enforce restrictions on use. 3495 */ 3496 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0, 3497 ("soreceive_dgram: wantrcvd")); 3498 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic")); 3499 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0, 3500 ("soreceive_dgram: SBS_RCVATMARK")); 3501 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0, 3502 ("soreceive_dgram: P_CONNREQUIRED")); 3503 3504 /* 3505 * Loop blocking while waiting for a datagram. 3506 */ 3507 SOCKBUF_LOCK(&so->so_rcv); 3508 while ((m = so->so_rcv.sb_mb) == NULL) { 3509 KASSERT(sbavail(&so->so_rcv) == 0, 3510 ("soreceive_dgram: sb_mb NULL but sbavail %u", 3511 sbavail(&so->so_rcv))); 3512 if (so->so_error) { 3513 error = so->so_error; 3514 so->so_error = 0; 3515 SOCKBUF_UNLOCK(&so->so_rcv); 3516 return (error); 3517 } 3518 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 3519 uio->uio_resid == 0) { 3520 SOCKBUF_UNLOCK(&so->so_rcv); 3521 return (0); 3522 } 3523 if ((so->so_state & SS_NBIO) || 3524 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 3525 SOCKBUF_UNLOCK(&so->so_rcv); 3526 return (EWOULDBLOCK); 3527 } 3528 SBLASTRECORDCHK(&so->so_rcv); 3529 SBLASTMBUFCHK(&so->so_rcv); 3530 error = sbwait(so, SO_RCV); 3531 if (error) { 3532 SOCKBUF_UNLOCK(&so->so_rcv); 3533 return (error); 3534 } 3535 } 3536 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3537 3538 if (uio->uio_td) 3539 uio->uio_td->td_ru.ru_msgrcv++; 3540 SBLASTRECORDCHK(&so->so_rcv); 3541 SBLASTMBUFCHK(&so->so_rcv); 3542 nextrecord = m->m_nextpkt; 3543 if (nextrecord == NULL) { 3544 KASSERT(so->so_rcv.sb_lastrecord == m, 3545 ("soreceive_dgram: lastrecord != m")); 3546 } 3547 3548 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord, 3549 ("soreceive_dgram: m_nextpkt != nextrecord")); 3550 3551 /* 3552 * Pull 'm' and its chain off the front of the packet queue. 3553 */ 3554 so->so_rcv.sb_mb = NULL; 3555 sockbuf_pushsync(&so->so_rcv, nextrecord); 3556 3557 /* 3558 * Walk 'm's chain and free that many bytes from the socket buffer. 3559 */ 3560 for (m2 = m; m2 != NULL; m2 = m2->m_next) 3561 sbfree(&so->so_rcv, m2); 3562 3563 /* 3564 * Do a few last checks before we let go of the lock. 3565 */ 3566 SBLASTRECORDCHK(&so->so_rcv); 3567 SBLASTMBUFCHK(&so->so_rcv); 3568 SOCKBUF_UNLOCK(&so->so_rcv); 3569 3570 if (pr->pr_flags & PR_ADDR) { 3571 KASSERT(m->m_type == MT_SONAME, 3572 ("m->m_type == %d", m->m_type)); 3573 if (psa != NULL) 3574 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 3575 M_WAITOK); 3576 m = m_free(m); 3577 } 3578 KASSERT(m, ("%s: no data or control after soname", __func__)); 3579 3580 /* 3581 * Packet to copyout() is now in 'm' and it is disconnected from the 3582 * queue. 3583 * 3584 * Process one or more MT_CONTROL mbufs present before any data mbufs 3585 * in the first mbuf chain on the socket buffer. We call into the 3586 * protocol to perform externalization (or freeing if controlp == 3587 * NULL). In some cases there can be only MT_CONTROL mbufs without 3588 * MT_DATA mbufs. 3589 */ 3590 if (m->m_type == MT_CONTROL) { 3591 struct mbuf *cm = NULL, *cmn; 3592 struct mbuf **cme = &cm; 3593 3594 do { 3595 m2 = m->m_next; 3596 m->m_next = NULL; 3597 *cme = m; 3598 cme = &(*cme)->m_next; 3599 m = m2; 3600 } while (m != NULL && m->m_type == MT_CONTROL); 3601 while (cm != NULL) { 3602 cmn = cm->m_next; 3603 cm->m_next = NULL; 3604 if (pr->pr_domain->dom_externalize != NULL) { 3605 error = (*pr->pr_domain->dom_externalize) 3606 (cm, controlp, flags); 3607 } else if (controlp != NULL) 3608 *controlp = cm; 3609 else 3610 m_freem(cm); 3611 if (controlp != NULL) { 3612 while (*controlp != NULL) 3613 controlp = &(*controlp)->m_next; 3614 } 3615 cm = cmn; 3616 } 3617 } 3618 KASSERT(m == NULL || m->m_type == MT_DATA, 3619 ("soreceive_dgram: !data")); 3620 while (m != NULL && uio->uio_resid > 0) { 3621 len = uio->uio_resid; 3622 if (len > m->m_len) 3623 len = m->m_len; 3624 error = uiomove(mtod(m, char *), (int)len, uio); 3625 if (error) { 3626 m_freem(m); 3627 return (error); 3628 } 3629 if (len == m->m_len) 3630 m = m_free(m); 3631 else { 3632 m->m_data += len; 3633 m->m_len -= len; 3634 } 3635 } 3636 if (m != NULL) { 3637 flags |= MSG_TRUNC; 3638 m_freem(m); 3639 } 3640 if (flagsp != NULL) 3641 *flagsp |= flags; 3642 return (0); 3643 } 3644 3645 int 3646 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, 3647 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3648 { 3649 int error; 3650 3651 CURVNET_SET(so->so_vnet); 3652 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp); 3653 CURVNET_RESTORE(); 3654 return (error); 3655 } 3656 3657 int 3658 soshutdown(struct socket *so, enum shutdown_how how) 3659 { 3660 int error; 3661 3662 CURVNET_SET(so->so_vnet); 3663 error = so->so_proto->pr_shutdown(so, how); 3664 CURVNET_RESTORE(); 3665 3666 return (error); 3667 } 3668 3669 /* 3670 * Used by several pr_shutdown implementations that use generic socket buffers. 3671 */ 3672 void 3673 sorflush(struct socket *so) 3674 { 3675 int error; 3676 3677 VNET_SO_ASSERT(so); 3678 3679 /* 3680 * Dislodge threads currently blocked in receive and wait to acquire 3681 * a lock against other simultaneous readers before clearing the 3682 * socket buffer. Don't let our acquire be interrupted by a signal 3683 * despite any existing socket disposition on interruptable waiting. 3684 * 3685 * The SOCK_IO_RECV_LOCK() is important here as there some pr_soreceive 3686 * methods that read the top of the socket buffer without acquisition 3687 * of the socket buffer mutex, assuming that top of the buffer 3688 * exclusively belongs to the read(2) syscall. This is handy when 3689 * performing MSG_PEEK. 3690 */ 3691 socantrcvmore(so); 3692 3693 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 3694 if (error != 0) { 3695 KASSERT(SOLISTENING(so), 3696 ("%s: soiolock(%p) failed", __func__, so)); 3697 return; 3698 } 3699 3700 sbrelease(so, SO_RCV); 3701 SOCK_IO_RECV_UNLOCK(so); 3702 3703 } 3704 3705 int 3706 sosetfib(struct socket *so, int fibnum) 3707 { 3708 if (fibnum < 0 || fibnum >= rt_numfibs) 3709 return (EINVAL); 3710 3711 SOCK_LOCK(so); 3712 so->so_fibnum = fibnum; 3713 SOCK_UNLOCK(so); 3714 3715 return (0); 3716 } 3717 3718 #ifdef SOCKET_HHOOK 3719 /* 3720 * Wrapper for Socket established helper hook. 3721 * Parameters: socket, context of the hook point, hook id. 3722 */ 3723 static inline int 3724 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) 3725 { 3726 struct socket_hhook_data hhook_data = { 3727 .so = so, 3728 .hctx = hctx, 3729 .m = NULL, 3730 .status = 0 3731 }; 3732 3733 CURVNET_SET(so->so_vnet); 3734 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd); 3735 CURVNET_RESTORE(); 3736 3737 /* Ugly but needed, since hhooks return void for now */ 3738 return (hhook_data.status); 3739 } 3740 #endif 3741 3742 /* 3743 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 3744 * additional variant to handle the case where the option value needs to be 3745 * some kind of integer, but not a specific size. In addition to their use 3746 * here, these functions are also called by the protocol-level pr_ctloutput() 3747 * routines. 3748 */ 3749 int 3750 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 3751 { 3752 size_t valsize; 3753 3754 /* 3755 * If the user gives us more than we wanted, we ignore it, but if we 3756 * don't get the minimum length the caller wants, we return EINVAL. 3757 * On success, sopt->sopt_valsize is set to however much we actually 3758 * retrieved. 3759 */ 3760 if ((valsize = sopt->sopt_valsize) < minlen) 3761 return EINVAL; 3762 if (valsize > len) 3763 sopt->sopt_valsize = valsize = len; 3764 3765 if (sopt->sopt_td != NULL) 3766 return (copyin(sopt->sopt_val, buf, valsize)); 3767 3768 bcopy(sopt->sopt_val, buf, valsize); 3769 return (0); 3770 } 3771 3772 /* 3773 * Kernel version of setsockopt(2). 3774 * 3775 * XXX: optlen is size_t, not socklen_t 3776 */ 3777 int 3778 so_setsockopt(struct socket *so, int level, int optname, void *optval, 3779 size_t optlen) 3780 { 3781 struct sockopt sopt; 3782 3783 sopt.sopt_level = level; 3784 sopt.sopt_name = optname; 3785 sopt.sopt_dir = SOPT_SET; 3786 sopt.sopt_val = optval; 3787 sopt.sopt_valsize = optlen; 3788 sopt.sopt_td = NULL; 3789 return (sosetopt(so, &sopt)); 3790 } 3791 3792 int 3793 sosetopt(struct socket *so, struct sockopt *sopt) 3794 { 3795 int error, optval; 3796 struct linger l; 3797 struct timeval tv; 3798 sbintime_t val, *valp; 3799 uint32_t val32; 3800 #ifdef MAC 3801 struct mac extmac; 3802 #endif 3803 3804 CURVNET_SET(so->so_vnet); 3805 error = 0; 3806 if (sopt->sopt_level != SOL_SOCKET) { 3807 if (so->so_proto->pr_ctloutput != NULL) 3808 error = (*so->so_proto->pr_ctloutput)(so, sopt); 3809 else 3810 error = ENOPROTOOPT; 3811 } else { 3812 switch (sopt->sopt_name) { 3813 case SO_ACCEPTFILTER: 3814 error = accept_filt_setopt(so, sopt); 3815 if (error) 3816 goto bad; 3817 break; 3818 3819 case SO_LINGER: 3820 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 3821 if (error) 3822 goto bad; 3823 if (l.l_linger < 0 || 3824 l.l_linger > USHRT_MAX || 3825 l.l_linger > (INT_MAX / hz)) { 3826 error = EDOM; 3827 goto bad; 3828 } 3829 SOCK_LOCK(so); 3830 so->so_linger = l.l_linger; 3831 if (l.l_onoff) 3832 so->so_options |= SO_LINGER; 3833 else 3834 so->so_options &= ~SO_LINGER; 3835 SOCK_UNLOCK(so); 3836 break; 3837 3838 case SO_DEBUG: 3839 case SO_KEEPALIVE: 3840 case SO_DONTROUTE: 3841 case SO_USELOOPBACK: 3842 case SO_BROADCAST: 3843 case SO_REUSEADDR: 3844 case SO_REUSEPORT: 3845 case SO_REUSEPORT_LB: 3846 case SO_OOBINLINE: 3847 case SO_TIMESTAMP: 3848 case SO_BINTIME: 3849 case SO_NOSIGPIPE: 3850 case SO_NO_DDP: 3851 case SO_NO_OFFLOAD: 3852 case SO_RERROR: 3853 error = sooptcopyin(sopt, &optval, sizeof optval, 3854 sizeof optval); 3855 if (error) 3856 goto bad; 3857 SOCK_LOCK(so); 3858 if (optval) 3859 so->so_options |= sopt->sopt_name; 3860 else 3861 so->so_options &= ~sopt->sopt_name; 3862 SOCK_UNLOCK(so); 3863 break; 3864 3865 case SO_SETFIB: 3866 error = so->so_proto->pr_ctloutput(so, sopt); 3867 break; 3868 3869 case SO_USER_COOKIE: 3870 error = sooptcopyin(sopt, &val32, sizeof val32, 3871 sizeof val32); 3872 if (error) 3873 goto bad; 3874 so->so_user_cookie = val32; 3875 break; 3876 3877 case SO_SNDBUF: 3878 case SO_RCVBUF: 3879 case SO_SNDLOWAT: 3880 case SO_RCVLOWAT: 3881 error = so->so_proto->pr_setsbopt(so, sopt); 3882 if (error) 3883 goto bad; 3884 break; 3885 3886 case SO_SNDTIMEO: 3887 case SO_RCVTIMEO: 3888 #ifdef COMPAT_FREEBSD32 3889 if (SV_CURPROC_FLAG(SV_ILP32)) { 3890 struct timeval32 tv32; 3891 3892 error = sooptcopyin(sopt, &tv32, sizeof tv32, 3893 sizeof tv32); 3894 CP(tv32, tv, tv_sec); 3895 CP(tv32, tv, tv_usec); 3896 } else 3897 #endif 3898 error = sooptcopyin(sopt, &tv, sizeof tv, 3899 sizeof tv); 3900 if (error) 3901 goto bad; 3902 if (tv.tv_sec < 0 || tv.tv_usec < 0 || 3903 tv.tv_usec >= 1000000) { 3904 error = EDOM; 3905 goto bad; 3906 } 3907 if (tv.tv_sec > INT32_MAX) 3908 val = SBT_MAX; 3909 else 3910 val = tvtosbt(tv); 3911 SOCK_LOCK(so); 3912 valp = sopt->sopt_name == SO_SNDTIMEO ? 3913 (SOLISTENING(so) ? &so->sol_sbsnd_timeo : 3914 &so->so_snd.sb_timeo) : 3915 (SOLISTENING(so) ? &so->sol_sbrcv_timeo : 3916 &so->so_rcv.sb_timeo); 3917 *valp = val; 3918 SOCK_UNLOCK(so); 3919 break; 3920 3921 case SO_LABEL: 3922 #ifdef MAC 3923 error = sooptcopyin(sopt, &extmac, sizeof extmac, 3924 sizeof extmac); 3925 if (error) 3926 goto bad; 3927 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 3928 so, &extmac); 3929 #else 3930 error = EOPNOTSUPP; 3931 #endif 3932 break; 3933 3934 case SO_TS_CLOCK: 3935 error = sooptcopyin(sopt, &optval, sizeof optval, 3936 sizeof optval); 3937 if (error) 3938 goto bad; 3939 if (optval < 0 || optval > SO_TS_CLOCK_MAX) { 3940 error = EINVAL; 3941 goto bad; 3942 } 3943 so->so_ts_clock = optval; 3944 break; 3945 3946 case SO_MAX_PACING_RATE: 3947 error = sooptcopyin(sopt, &val32, sizeof(val32), 3948 sizeof(val32)); 3949 if (error) 3950 goto bad; 3951 so->so_max_pacing_rate = val32; 3952 break; 3953 3954 case SO_SPLICE: { 3955 struct splice splice; 3956 3957 #ifdef COMPAT_FREEBSD32 3958 if (SV_CURPROC_FLAG(SV_ILP32)) { 3959 struct splice32 splice32; 3960 3961 error = sooptcopyin(sopt, &splice32, 3962 sizeof(splice32), sizeof(splice32)); 3963 if (error == 0) { 3964 splice.sp_fd = splice32.sp_fd; 3965 splice.sp_max = splice32.sp_max; 3966 CP(splice32.sp_idle, splice.sp_idle, 3967 tv_sec); 3968 CP(splice32.sp_idle, splice.sp_idle, 3969 tv_usec); 3970 } 3971 } else 3972 #endif 3973 { 3974 error = sooptcopyin(sopt, &splice, 3975 sizeof(splice), sizeof(splice)); 3976 } 3977 if (error) 3978 goto bad; 3979 #ifdef KTRACE 3980 if (KTRPOINT(curthread, KTR_STRUCT)) 3981 ktrsplice(&splice); 3982 #endif 3983 3984 error = splice_init(); 3985 if (error != 0) 3986 goto bad; 3987 3988 if (splice.sp_fd >= 0) { 3989 struct file *fp; 3990 struct socket *so2; 3991 3992 if (!cap_rights_contains(sopt->sopt_rights, 3993 &cap_recv_rights)) { 3994 error = ENOTCAPABLE; 3995 goto bad; 3996 } 3997 error = getsock(sopt->sopt_td, splice.sp_fd, 3998 &cap_send_rights, &fp); 3999 if (error != 0) 4000 goto bad; 4001 so2 = fp->f_data; 4002 4003 error = so_splice(so, so2, &splice); 4004 fdrop(fp, sopt->sopt_td); 4005 } else { 4006 error = so_unsplice(so, false); 4007 } 4008 break; 4009 } 4010 default: 4011 #ifdef SOCKET_HHOOK 4012 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 4013 error = hhook_run_socket(so, sopt, 4014 HHOOK_SOCKET_OPT); 4015 else 4016 #endif 4017 error = ENOPROTOOPT; 4018 break; 4019 } 4020 if (error == 0 && so->so_proto->pr_ctloutput != NULL) 4021 (void)(*so->so_proto->pr_ctloutput)(so, sopt); 4022 } 4023 bad: 4024 CURVNET_RESTORE(); 4025 return (error); 4026 } 4027 4028 /* 4029 * Helper routine for getsockopt. 4030 */ 4031 int 4032 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 4033 { 4034 int error; 4035 size_t valsize; 4036 4037 error = 0; 4038 4039 /* 4040 * Documented get behavior is that we always return a value, possibly 4041 * truncated to fit in the user's buffer. Traditional behavior is 4042 * that we always tell the user precisely how much we copied, rather 4043 * than something useful like the total amount we had available for 4044 * her. Note that this interface is not idempotent; the entire 4045 * answer must be generated ahead of time. 4046 */ 4047 valsize = min(len, sopt->sopt_valsize); 4048 sopt->sopt_valsize = valsize; 4049 if (sopt->sopt_val != NULL) { 4050 if (sopt->sopt_td != NULL) 4051 error = copyout(buf, sopt->sopt_val, valsize); 4052 else 4053 bcopy(buf, sopt->sopt_val, valsize); 4054 } 4055 return (error); 4056 } 4057 4058 int 4059 sogetopt(struct socket *so, struct sockopt *sopt) 4060 { 4061 int error, optval; 4062 struct linger l; 4063 struct timeval tv; 4064 #ifdef MAC 4065 struct mac extmac; 4066 #endif 4067 4068 CURVNET_SET(so->so_vnet); 4069 error = 0; 4070 if (sopt->sopt_level != SOL_SOCKET) { 4071 if (so->so_proto->pr_ctloutput != NULL) 4072 error = (*so->so_proto->pr_ctloutput)(so, sopt); 4073 else 4074 error = ENOPROTOOPT; 4075 CURVNET_RESTORE(); 4076 return (error); 4077 } else { 4078 switch (sopt->sopt_name) { 4079 case SO_ACCEPTFILTER: 4080 error = accept_filt_getopt(so, sopt); 4081 break; 4082 4083 case SO_LINGER: 4084 SOCK_LOCK(so); 4085 l.l_onoff = so->so_options & SO_LINGER; 4086 l.l_linger = so->so_linger; 4087 SOCK_UNLOCK(so); 4088 error = sooptcopyout(sopt, &l, sizeof l); 4089 break; 4090 4091 case SO_USELOOPBACK: 4092 case SO_DONTROUTE: 4093 case SO_DEBUG: 4094 case SO_KEEPALIVE: 4095 case SO_REUSEADDR: 4096 case SO_REUSEPORT: 4097 case SO_REUSEPORT_LB: 4098 case SO_BROADCAST: 4099 case SO_OOBINLINE: 4100 case SO_ACCEPTCONN: 4101 case SO_TIMESTAMP: 4102 case SO_BINTIME: 4103 case SO_NOSIGPIPE: 4104 case SO_NO_DDP: 4105 case SO_NO_OFFLOAD: 4106 case SO_RERROR: 4107 optval = so->so_options & sopt->sopt_name; 4108 integer: 4109 error = sooptcopyout(sopt, &optval, sizeof optval); 4110 break; 4111 4112 case SO_FIB: 4113 SOCK_LOCK(so); 4114 optval = so->so_fibnum; 4115 SOCK_UNLOCK(so); 4116 goto integer; 4117 4118 case SO_DOMAIN: 4119 optval = so->so_proto->pr_domain->dom_family; 4120 goto integer; 4121 4122 case SO_TYPE: 4123 optval = so->so_type; 4124 goto integer; 4125 4126 case SO_PROTOCOL: 4127 optval = so->so_proto->pr_protocol; 4128 goto integer; 4129 4130 case SO_ERROR: 4131 SOCK_LOCK(so); 4132 if (so->so_error) { 4133 optval = so->so_error; 4134 so->so_error = 0; 4135 } else { 4136 optval = so->so_rerror; 4137 so->so_rerror = 0; 4138 } 4139 SOCK_UNLOCK(so); 4140 goto integer; 4141 4142 case SO_SNDBUF: 4143 SOCK_LOCK(so); 4144 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat : 4145 so->so_snd.sb_hiwat; 4146 SOCK_UNLOCK(so); 4147 goto integer; 4148 4149 case SO_RCVBUF: 4150 SOCK_LOCK(so); 4151 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat : 4152 so->so_rcv.sb_hiwat; 4153 SOCK_UNLOCK(so); 4154 goto integer; 4155 4156 case SO_SNDLOWAT: 4157 SOCK_LOCK(so); 4158 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat : 4159 so->so_snd.sb_lowat; 4160 SOCK_UNLOCK(so); 4161 goto integer; 4162 4163 case SO_RCVLOWAT: 4164 SOCK_LOCK(so); 4165 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat : 4166 so->so_rcv.sb_lowat; 4167 SOCK_UNLOCK(so); 4168 goto integer; 4169 4170 case SO_SNDTIMEO: 4171 case SO_RCVTIMEO: 4172 SOCK_LOCK(so); 4173 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ? 4174 (SOLISTENING(so) ? so->sol_sbsnd_timeo : 4175 so->so_snd.sb_timeo) : 4176 (SOLISTENING(so) ? so->sol_sbrcv_timeo : 4177 so->so_rcv.sb_timeo)); 4178 SOCK_UNLOCK(so); 4179 #ifdef COMPAT_FREEBSD32 4180 if (SV_CURPROC_FLAG(SV_ILP32)) { 4181 struct timeval32 tv32; 4182 4183 CP(tv, tv32, tv_sec); 4184 CP(tv, tv32, tv_usec); 4185 error = sooptcopyout(sopt, &tv32, sizeof tv32); 4186 } else 4187 #endif 4188 error = sooptcopyout(sopt, &tv, sizeof tv); 4189 break; 4190 4191 case SO_LABEL: 4192 #ifdef MAC 4193 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 4194 sizeof(extmac)); 4195 if (error) 4196 goto bad; 4197 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 4198 so, &extmac); 4199 if (error) 4200 goto bad; 4201 /* Don't copy out extmac, it is unchanged. */ 4202 #else 4203 error = EOPNOTSUPP; 4204 #endif 4205 break; 4206 4207 case SO_PEERLABEL: 4208 #ifdef MAC 4209 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 4210 sizeof(extmac)); 4211 if (error) 4212 goto bad; 4213 error = mac_getsockopt_peerlabel( 4214 sopt->sopt_td->td_ucred, so, &extmac); 4215 if (error) 4216 goto bad; 4217 /* Don't copy out extmac, it is unchanged. */ 4218 #else 4219 error = EOPNOTSUPP; 4220 #endif 4221 break; 4222 4223 case SO_LISTENQLIMIT: 4224 SOCK_LOCK(so); 4225 optval = SOLISTENING(so) ? so->sol_qlimit : 0; 4226 SOCK_UNLOCK(so); 4227 goto integer; 4228 4229 case SO_LISTENQLEN: 4230 SOCK_LOCK(so); 4231 optval = SOLISTENING(so) ? so->sol_qlen : 0; 4232 SOCK_UNLOCK(so); 4233 goto integer; 4234 4235 case SO_LISTENINCQLEN: 4236 SOCK_LOCK(so); 4237 optval = SOLISTENING(so) ? so->sol_incqlen : 0; 4238 SOCK_UNLOCK(so); 4239 goto integer; 4240 4241 case SO_TS_CLOCK: 4242 optval = so->so_ts_clock; 4243 goto integer; 4244 4245 case SO_MAX_PACING_RATE: 4246 optval = so->so_max_pacing_rate; 4247 goto integer; 4248 4249 case SO_SPLICE: { 4250 off_t n; 4251 4252 /* 4253 * Acquire the I/O lock to serialize with 4254 * so_splice_xfer(). This is not required for 4255 * correctness, but makes testing simpler: once a byte 4256 * has been transmitted to the sink and observed (e.g., 4257 * by reading from the socket to which the sink is 4258 * connected), a subsequent getsockopt(SO_SPLICE) will 4259 * return an up-to-date value. 4260 */ 4261 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT); 4262 if (error != 0) 4263 goto bad; 4264 SOCK_LOCK(so); 4265 if (SOLISTENING(so)) { 4266 n = 0; 4267 } else { 4268 n = so->so_splice_sent; 4269 } 4270 SOCK_UNLOCK(so); 4271 SOCK_IO_RECV_UNLOCK(so); 4272 error = sooptcopyout(sopt, &n, sizeof(n)); 4273 break; 4274 } 4275 4276 default: 4277 #ifdef SOCKET_HHOOK 4278 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 4279 error = hhook_run_socket(so, sopt, 4280 HHOOK_SOCKET_OPT); 4281 else 4282 #endif 4283 error = ENOPROTOOPT; 4284 break; 4285 } 4286 } 4287 bad: 4288 CURVNET_RESTORE(); 4289 return (error); 4290 } 4291 4292 int 4293 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 4294 { 4295 struct mbuf *m, *m_prev; 4296 int sopt_size = sopt->sopt_valsize; 4297 4298 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 4299 if (m == NULL) 4300 return ENOBUFS; 4301 if (sopt_size > MLEN) { 4302 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT); 4303 if ((m->m_flags & M_EXT) == 0) { 4304 m_free(m); 4305 return ENOBUFS; 4306 } 4307 m->m_len = min(MCLBYTES, sopt_size); 4308 } else { 4309 m->m_len = min(MLEN, sopt_size); 4310 } 4311 sopt_size -= m->m_len; 4312 *mp = m; 4313 m_prev = m; 4314 4315 while (sopt_size) { 4316 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 4317 if (m == NULL) { 4318 m_freem(*mp); 4319 return ENOBUFS; 4320 } 4321 if (sopt_size > MLEN) { 4322 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK : 4323 M_NOWAIT); 4324 if ((m->m_flags & M_EXT) == 0) { 4325 m_freem(m); 4326 m_freem(*mp); 4327 return ENOBUFS; 4328 } 4329 m->m_len = min(MCLBYTES, sopt_size); 4330 } else { 4331 m->m_len = min(MLEN, sopt_size); 4332 } 4333 sopt_size -= m->m_len; 4334 m_prev->m_next = m; 4335 m_prev = m; 4336 } 4337 return (0); 4338 } 4339 4340 int 4341 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 4342 { 4343 struct mbuf *m0 = m; 4344 4345 if (sopt->sopt_val == NULL) 4346 return (0); 4347 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 4348 if (sopt->sopt_td != NULL) { 4349 int error; 4350 4351 error = copyin(sopt->sopt_val, mtod(m, char *), 4352 m->m_len); 4353 if (error != 0) { 4354 m_freem(m0); 4355 return(error); 4356 } 4357 } else 4358 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 4359 sopt->sopt_valsize -= m->m_len; 4360 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 4361 m = m->m_next; 4362 } 4363 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 4364 panic("ip6_sooptmcopyin"); 4365 return (0); 4366 } 4367 4368 int 4369 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 4370 { 4371 struct mbuf *m0 = m; 4372 size_t valsize = 0; 4373 4374 if (sopt->sopt_val == NULL) 4375 return (0); 4376 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 4377 if (sopt->sopt_td != NULL) { 4378 int error; 4379 4380 error = copyout(mtod(m, char *), sopt->sopt_val, 4381 m->m_len); 4382 if (error != 0) { 4383 m_freem(m0); 4384 return(error); 4385 } 4386 } else 4387 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 4388 sopt->sopt_valsize -= m->m_len; 4389 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 4390 valsize += m->m_len; 4391 m = m->m_next; 4392 } 4393 if (m != NULL) { 4394 /* enough soopt buffer should be given from user-land */ 4395 m_freem(m0); 4396 return(EINVAL); 4397 } 4398 sopt->sopt_valsize = valsize; 4399 return (0); 4400 } 4401 4402 /* 4403 * sohasoutofband(): protocol notifies socket layer of the arrival of new 4404 * out-of-band data, which will then notify socket consumers. 4405 */ 4406 void 4407 sohasoutofband(struct socket *so) 4408 { 4409 4410 if (so->so_sigio != NULL) 4411 pgsigio(&so->so_sigio, SIGURG, 0); 4412 selwakeuppri(&so->so_rdsel, PSOCK); 4413 } 4414 4415 int 4416 sopoll_generic(struct socket *so, int events, struct thread *td) 4417 { 4418 int revents; 4419 4420 SOCK_LOCK(so); 4421 if (SOLISTENING(so)) { 4422 if (!(events & (POLLIN | POLLRDNORM))) 4423 revents = 0; 4424 else if (!TAILQ_EMPTY(&so->sol_comp)) 4425 revents = events & (POLLIN | POLLRDNORM); 4426 else if ((events & POLLINIGNEOF) == 0 && so->so_error) 4427 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 4428 else { 4429 selrecord(td, &so->so_rdsel); 4430 revents = 0; 4431 } 4432 } else { 4433 revents = 0; 4434 SOCK_SENDBUF_LOCK(so); 4435 SOCK_RECVBUF_LOCK(so); 4436 if (events & (POLLIN | POLLRDNORM)) 4437 if (soreadabledata(so) && !isspliced(so)) 4438 revents |= events & (POLLIN | POLLRDNORM); 4439 if (events & (POLLOUT | POLLWRNORM)) 4440 if (sowriteable(so) && !issplicedback(so)) 4441 revents |= events & (POLLOUT | POLLWRNORM); 4442 if (events & (POLLPRI | POLLRDBAND)) 4443 if (so->so_oobmark || 4444 (so->so_rcv.sb_state & SBS_RCVATMARK)) 4445 revents |= events & (POLLPRI | POLLRDBAND); 4446 if ((events & POLLINIGNEOF) == 0) { 4447 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 4448 revents |= events & (POLLIN | POLLRDNORM); 4449 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 4450 revents |= POLLHUP; 4451 } 4452 } 4453 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 4454 revents |= events & POLLRDHUP; 4455 if (revents == 0) { 4456 if (events & 4457 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND | POLLRDHUP)) { 4458 selrecord(td, &so->so_rdsel); 4459 so->so_rcv.sb_flags |= SB_SEL; 4460 } 4461 if (events & (POLLOUT | POLLWRNORM)) { 4462 selrecord(td, &so->so_wrsel); 4463 so->so_snd.sb_flags |= SB_SEL; 4464 } 4465 } 4466 SOCK_RECVBUF_UNLOCK(so); 4467 SOCK_SENDBUF_UNLOCK(so); 4468 } 4469 SOCK_UNLOCK(so); 4470 return (revents); 4471 } 4472 4473 int 4474 soo_kqfilter(struct file *fp, struct knote *kn) 4475 { 4476 struct socket *so = kn->kn_fp->f_data; 4477 struct sockbuf *sb; 4478 sb_which which; 4479 struct knlist *knl; 4480 4481 switch (kn->kn_filter) { 4482 case EVFILT_READ: 4483 kn->kn_fop = &soread_filtops; 4484 knl = &so->so_rdsel.si_note; 4485 sb = &so->so_rcv; 4486 which = SO_RCV; 4487 break; 4488 case EVFILT_WRITE: 4489 kn->kn_fop = &sowrite_filtops; 4490 knl = &so->so_wrsel.si_note; 4491 sb = &so->so_snd; 4492 which = SO_SND; 4493 break; 4494 case EVFILT_EMPTY: 4495 kn->kn_fop = &soempty_filtops; 4496 knl = &so->so_wrsel.si_note; 4497 sb = &so->so_snd; 4498 which = SO_SND; 4499 break; 4500 default: 4501 return (EINVAL); 4502 } 4503 4504 SOCK_LOCK(so); 4505 if (SOLISTENING(so)) { 4506 knlist_add(knl, kn, 1); 4507 } else { 4508 SOCK_BUF_LOCK(so, which); 4509 knlist_add(knl, kn, 1); 4510 sb->sb_flags |= SB_KNOTE; 4511 SOCK_BUF_UNLOCK(so, which); 4512 } 4513 SOCK_UNLOCK(so); 4514 return (0); 4515 } 4516 4517 static void 4518 filt_sordetach(struct knote *kn) 4519 { 4520 struct socket *so = kn->kn_fp->f_data; 4521 4522 so_rdknl_lock(so); 4523 knlist_remove(&so->so_rdsel.si_note, kn, 1); 4524 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note)) 4525 so->so_rcv.sb_flags &= ~SB_KNOTE; 4526 so_rdknl_unlock(so); 4527 } 4528 4529 /*ARGSUSED*/ 4530 static int 4531 filt_soread(struct knote *kn, long hint) 4532 { 4533 struct socket *so; 4534 4535 so = kn->kn_fp->f_data; 4536 4537 if (SOLISTENING(so)) { 4538 SOCK_LOCK_ASSERT(so); 4539 kn->kn_data = so->sol_qlen; 4540 if (so->so_error) { 4541 kn->kn_flags |= EV_EOF; 4542 kn->kn_fflags = so->so_error; 4543 return (1); 4544 } 4545 return (!TAILQ_EMPTY(&so->sol_comp)); 4546 } 4547 4548 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) 4549 return (0); 4550 4551 SOCK_RECVBUF_LOCK_ASSERT(so); 4552 4553 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl; 4554 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 4555 kn->kn_flags |= EV_EOF; 4556 kn->kn_fflags = so->so_error; 4557 return (1); 4558 } else if (so->so_error || so->so_rerror) 4559 return (1); 4560 4561 if (kn->kn_sfflags & NOTE_LOWAT) { 4562 if (kn->kn_data >= kn->kn_sdata) 4563 return (1); 4564 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat) 4565 return (1); 4566 4567 #ifdef SOCKET_HHOOK 4568 /* This hook returning non-zero indicates an event, not error */ 4569 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD)); 4570 #else 4571 return (0); 4572 #endif 4573 } 4574 4575 static void 4576 filt_sowdetach(struct knote *kn) 4577 { 4578 struct socket *so = kn->kn_fp->f_data; 4579 4580 so_wrknl_lock(so); 4581 knlist_remove(&so->so_wrsel.si_note, kn, 1); 4582 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note)) 4583 so->so_snd.sb_flags &= ~SB_KNOTE; 4584 so_wrknl_unlock(so); 4585 } 4586 4587 /*ARGSUSED*/ 4588 static int 4589 filt_sowrite(struct knote *kn, long hint) 4590 { 4591 struct socket *so; 4592 4593 so = kn->kn_fp->f_data; 4594 4595 if (SOLISTENING(so)) 4596 return (0); 4597 4598 SOCK_SENDBUF_LOCK_ASSERT(so); 4599 kn->kn_data = sbspace(&so->so_snd); 4600 4601 #ifdef SOCKET_HHOOK 4602 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE); 4603 #endif 4604 4605 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 4606 kn->kn_flags |= EV_EOF; 4607 kn->kn_fflags = so->so_error; 4608 return (1); 4609 } else if (so->so_error) /* temporary udp error */ 4610 return (1); 4611 else if (((so->so_state & SS_ISCONNECTED) == 0) && 4612 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 4613 return (0); 4614 else if (kn->kn_sfflags & NOTE_LOWAT) 4615 return (kn->kn_data >= kn->kn_sdata); 4616 else 4617 return (kn->kn_data >= so->so_snd.sb_lowat); 4618 } 4619 4620 static int 4621 filt_soempty(struct knote *kn, long hint) 4622 { 4623 struct socket *so; 4624 4625 so = kn->kn_fp->f_data; 4626 4627 if (SOLISTENING(so)) 4628 return (1); 4629 4630 SOCK_SENDBUF_LOCK_ASSERT(so); 4631 kn->kn_data = sbused(&so->so_snd); 4632 4633 if (kn->kn_data == 0) 4634 return (1); 4635 else 4636 return (0); 4637 } 4638 4639 int 4640 socheckuid(struct socket *so, uid_t uid) 4641 { 4642 4643 if (so == NULL) 4644 return (EPERM); 4645 if (so->so_cred->cr_uid != uid) 4646 return (EPERM); 4647 return (0); 4648 } 4649 4650 /* 4651 * These functions are used by protocols to notify the socket layer (and its 4652 * consumers) of state changes in the sockets driven by protocol-side events. 4653 */ 4654 4655 /* 4656 * Procedures to manipulate state flags of socket and do appropriate wakeups. 4657 * 4658 * Normal sequence from the active (originating) side is that 4659 * soisconnecting() is called during processing of connect() call, resulting 4660 * in an eventual call to soisconnected() if/when the connection is 4661 * established. When the connection is torn down soisdisconnecting() is 4662 * called during processing of disconnect() call, and soisdisconnected() is 4663 * called when the connection to the peer is totally severed. The semantics 4664 * of these routines are such that connectionless protocols can call 4665 * soisconnected() and soisdisconnected() only, bypassing the in-progress 4666 * calls when setting up a ``connection'' takes no time. 4667 * 4668 * From the passive side, a socket is created with two queues of sockets: 4669 * so_incomp for connections in progress and so_comp for connections already 4670 * made and awaiting user acceptance. As a protocol is preparing incoming 4671 * connections, it creates a socket structure queued on so_incomp by calling 4672 * sonewconn(). When the connection is established, soisconnected() is 4673 * called, and transfers the socket structure to so_comp, making it available 4674 * to accept(). 4675 * 4676 * If a socket is closed with sockets on either so_incomp or so_comp, these 4677 * sockets are dropped. 4678 * 4679 * If higher-level protocols are implemented in the kernel, the wakeups done 4680 * here will sometimes cause software-interrupt process scheduling. 4681 */ 4682 void 4683 soisconnecting(struct socket *so) 4684 { 4685 4686 SOCK_LOCK(so); 4687 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 4688 so->so_state |= SS_ISCONNECTING; 4689 SOCK_UNLOCK(so); 4690 } 4691 4692 void 4693 soisconnected(struct socket *so) 4694 { 4695 bool last __diagused; 4696 4697 SOCK_LOCK(so); 4698 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING); 4699 so->so_state |= SS_ISCONNECTED; 4700 4701 if (so->so_qstate == SQ_INCOMP) { 4702 struct socket *head = so->so_listen; 4703 int ret; 4704 4705 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so)); 4706 /* 4707 * Promoting a socket from incomplete queue to complete, we 4708 * need to go through reverse order of locking. We first do 4709 * trylock, and if that doesn't succeed, we go the hard way 4710 * leaving a reference and rechecking consistency after proper 4711 * locking. 4712 */ 4713 if (__predict_false(SOLISTEN_TRYLOCK(head) == 0)) { 4714 soref(head); 4715 SOCK_UNLOCK(so); 4716 SOLISTEN_LOCK(head); 4717 SOCK_LOCK(so); 4718 if (__predict_false(head != so->so_listen)) { 4719 /* 4720 * The socket went off the listen queue, 4721 * should be lost race to close(2) of sol. 4722 * The socket is about to soabort(). 4723 */ 4724 SOCK_UNLOCK(so); 4725 sorele_locked(head); 4726 return; 4727 } 4728 last = refcount_release(&head->so_count); 4729 KASSERT(!last, ("%s: released last reference for %p", 4730 __func__, head)); 4731 } 4732 again: 4733 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 4734 TAILQ_REMOVE(&head->sol_incomp, so, so_list); 4735 head->sol_incqlen--; 4736 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 4737 head->sol_qlen++; 4738 so->so_qstate = SQ_COMP; 4739 SOCK_UNLOCK(so); 4740 solisten_wakeup(head); /* unlocks */ 4741 } else { 4742 SOCK_RECVBUF_LOCK(so); 4743 soupcall_set(so, SO_RCV, 4744 head->sol_accept_filter->accf_callback, 4745 head->sol_accept_filter_arg); 4746 so->so_options &= ~SO_ACCEPTFILTER; 4747 ret = head->sol_accept_filter->accf_callback(so, 4748 head->sol_accept_filter_arg, M_NOWAIT); 4749 if (ret == SU_ISCONNECTED) { 4750 soupcall_clear(so, SO_RCV); 4751 SOCK_RECVBUF_UNLOCK(so); 4752 goto again; 4753 } 4754 SOCK_RECVBUF_UNLOCK(so); 4755 SOCK_UNLOCK(so); 4756 SOLISTEN_UNLOCK(head); 4757 } 4758 return; 4759 } 4760 SOCK_UNLOCK(so); 4761 wakeup(&so->so_timeo); 4762 sorwakeup(so); 4763 sowwakeup(so); 4764 } 4765 4766 void 4767 soisdisconnecting(struct socket *so) 4768 { 4769 4770 SOCK_LOCK(so); 4771 so->so_state &= ~SS_ISCONNECTING; 4772 so->so_state |= SS_ISDISCONNECTING; 4773 4774 if (!SOLISTENING(so)) { 4775 SOCK_RECVBUF_LOCK(so); 4776 socantrcvmore_locked(so); 4777 SOCK_SENDBUF_LOCK(so); 4778 socantsendmore_locked(so); 4779 } 4780 SOCK_UNLOCK(so); 4781 wakeup(&so->so_timeo); 4782 } 4783 4784 void 4785 soisdisconnected(struct socket *so) 4786 { 4787 4788 SOCK_LOCK(so); 4789 4790 /* 4791 * There is at least one reader of so_state that does not 4792 * acquire socket lock, namely soreceive_generic(). Ensure 4793 * that it never sees all flags that track connection status 4794 * cleared, by ordering the update with a barrier semantic of 4795 * our release thread fence. 4796 */ 4797 so->so_state |= SS_ISDISCONNECTED; 4798 atomic_thread_fence_rel(); 4799 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 4800 4801 if (!SOLISTENING(so)) { 4802 SOCK_UNLOCK(so); 4803 SOCK_RECVBUF_LOCK(so); 4804 socantrcvmore_locked(so); 4805 SOCK_SENDBUF_LOCK(so); 4806 sbdrop_locked(&so->so_snd, sbused(&so->so_snd)); 4807 socantsendmore_locked(so); 4808 } else 4809 SOCK_UNLOCK(so); 4810 wakeup(&so->so_timeo); 4811 } 4812 4813 int 4814 soiolock(struct socket *so, struct sx *sx, int flags) 4815 { 4816 int error; 4817 4818 KASSERT((flags & SBL_VALID) == flags, 4819 ("soiolock: invalid flags %#x", flags)); 4820 4821 if ((flags & SBL_WAIT) != 0) { 4822 if ((flags & SBL_NOINTR) != 0) { 4823 sx_xlock(sx); 4824 } else { 4825 error = sx_xlock_sig(sx); 4826 if (error != 0) 4827 return (error); 4828 } 4829 } else if (!sx_try_xlock(sx)) { 4830 return (EWOULDBLOCK); 4831 } 4832 4833 if (__predict_false(SOLISTENING(so))) { 4834 sx_xunlock(sx); 4835 return (ENOTCONN); 4836 } 4837 return (0); 4838 } 4839 4840 void 4841 soiounlock(struct sx *sx) 4842 { 4843 sx_xunlock(sx); 4844 } 4845 4846 /* 4847 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME. 4848 */ 4849 struct sockaddr * 4850 sodupsockaddr(const struct sockaddr *sa, int mflags) 4851 { 4852 struct sockaddr *sa2; 4853 4854 sa2 = malloc(sa->sa_len, M_SONAME, mflags); 4855 if (sa2) 4856 bcopy(sa, sa2, sa->sa_len); 4857 return sa2; 4858 } 4859 4860 /* 4861 * Register per-socket destructor. 4862 */ 4863 void 4864 sodtor_set(struct socket *so, so_dtor_t *func) 4865 { 4866 4867 SOCK_LOCK_ASSERT(so); 4868 so->so_dtor = func; 4869 } 4870 4871 /* 4872 * Register per-socket buffer upcalls. 4873 */ 4874 void 4875 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg) 4876 { 4877 struct sockbuf *sb; 4878 4879 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 4880 4881 switch (which) { 4882 case SO_RCV: 4883 sb = &so->so_rcv; 4884 break; 4885 case SO_SND: 4886 sb = &so->so_snd; 4887 break; 4888 } 4889 SOCK_BUF_LOCK_ASSERT(so, which); 4890 sb->sb_upcall = func; 4891 sb->sb_upcallarg = arg; 4892 sb->sb_flags |= SB_UPCALL; 4893 } 4894 4895 void 4896 soupcall_clear(struct socket *so, sb_which which) 4897 { 4898 struct sockbuf *sb; 4899 4900 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 4901 4902 switch (which) { 4903 case SO_RCV: 4904 sb = &so->so_rcv; 4905 break; 4906 case SO_SND: 4907 sb = &so->so_snd; 4908 break; 4909 } 4910 SOCK_BUF_LOCK_ASSERT(so, which); 4911 KASSERT(sb->sb_upcall != NULL, 4912 ("%s: so %p no upcall to clear", __func__, so)); 4913 sb->sb_upcall = NULL; 4914 sb->sb_upcallarg = NULL; 4915 sb->sb_flags &= ~SB_UPCALL; 4916 } 4917 4918 void 4919 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) 4920 { 4921 4922 SOLISTEN_LOCK_ASSERT(so); 4923 so->sol_upcall = func; 4924 so->sol_upcallarg = arg; 4925 } 4926 4927 static void 4928 so_rdknl_lock(void *arg) 4929 { 4930 struct socket *so = arg; 4931 4932 retry: 4933 if (SOLISTENING(so)) { 4934 SOLISTEN_LOCK(so); 4935 } else { 4936 SOCK_RECVBUF_LOCK(so); 4937 if (__predict_false(SOLISTENING(so))) { 4938 SOCK_RECVBUF_UNLOCK(so); 4939 goto retry; 4940 } 4941 } 4942 } 4943 4944 static void 4945 so_rdknl_unlock(void *arg) 4946 { 4947 struct socket *so = arg; 4948 4949 if (SOLISTENING(so)) 4950 SOLISTEN_UNLOCK(so); 4951 else 4952 SOCK_RECVBUF_UNLOCK(so); 4953 } 4954 4955 static void 4956 so_rdknl_assert_lock(void *arg, int what) 4957 { 4958 struct socket *so = arg; 4959 4960 if (what == LA_LOCKED) { 4961 if (SOLISTENING(so)) 4962 SOLISTEN_LOCK_ASSERT(so); 4963 else 4964 SOCK_RECVBUF_LOCK_ASSERT(so); 4965 } else { 4966 if (SOLISTENING(so)) 4967 SOLISTEN_UNLOCK_ASSERT(so); 4968 else 4969 SOCK_RECVBUF_UNLOCK_ASSERT(so); 4970 } 4971 } 4972 4973 static void 4974 so_wrknl_lock(void *arg) 4975 { 4976 struct socket *so = arg; 4977 4978 retry: 4979 if (SOLISTENING(so)) { 4980 SOLISTEN_LOCK(so); 4981 } else { 4982 SOCK_SENDBUF_LOCK(so); 4983 if (__predict_false(SOLISTENING(so))) { 4984 SOCK_SENDBUF_UNLOCK(so); 4985 goto retry; 4986 } 4987 } 4988 } 4989 4990 static void 4991 so_wrknl_unlock(void *arg) 4992 { 4993 struct socket *so = arg; 4994 4995 if (SOLISTENING(so)) 4996 SOLISTEN_UNLOCK(so); 4997 else 4998 SOCK_SENDBUF_UNLOCK(so); 4999 } 5000 5001 static void 5002 so_wrknl_assert_lock(void *arg, int what) 5003 { 5004 struct socket *so = arg; 5005 5006 if (what == LA_LOCKED) { 5007 if (SOLISTENING(so)) 5008 SOLISTEN_LOCK_ASSERT(so); 5009 else 5010 SOCK_SENDBUF_LOCK_ASSERT(so); 5011 } else { 5012 if (SOLISTENING(so)) 5013 SOLISTEN_UNLOCK_ASSERT(so); 5014 else 5015 SOCK_SENDBUF_UNLOCK_ASSERT(so); 5016 } 5017 } 5018 5019 /* 5020 * Create an external-format (``xsocket'') structure using the information in 5021 * the kernel-format socket structure pointed to by so. This is done to 5022 * reduce the spew of irrelevant information over this interface, to isolate 5023 * user code from changes in the kernel structure, and potentially to provide 5024 * information-hiding if we decide that some of this information should be 5025 * hidden from users. 5026 */ 5027 void 5028 sotoxsocket(struct socket *so, struct xsocket *xso) 5029 { 5030 5031 bzero(xso, sizeof(*xso)); 5032 xso->xso_len = sizeof *xso; 5033 xso->xso_so = (uintptr_t)so; 5034 xso->so_type = so->so_type; 5035 xso->so_options = so->so_options; 5036 xso->so_linger = so->so_linger; 5037 xso->so_state = so->so_state; 5038 xso->so_pcb = (uintptr_t)so->so_pcb; 5039 xso->xso_protocol = so->so_proto->pr_protocol; 5040 xso->xso_family = so->so_proto->pr_domain->dom_family; 5041 xso->so_timeo = so->so_timeo; 5042 xso->so_error = so->so_error; 5043 xso->so_uid = so->so_cred->cr_uid; 5044 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0; 5045 SOCK_LOCK(so); 5046 xso->so_fibnum = so->so_fibnum; 5047 if (SOLISTENING(so)) { 5048 xso->so_qlen = so->sol_qlen; 5049 xso->so_incqlen = so->sol_incqlen; 5050 xso->so_qlimit = so->sol_qlimit; 5051 xso->so_oobmark = 0; 5052 } else { 5053 xso->so_state |= so->so_qstate; 5054 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0; 5055 xso->so_oobmark = so->so_oobmark; 5056 sbtoxsockbuf(&so->so_snd, &xso->so_snd); 5057 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv); 5058 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) 5059 xso->so_splice_so = (uintptr_t)so->so_splice->dst; 5060 } 5061 SOCK_UNLOCK(so); 5062 } 5063 5064 int 5065 so_options_get(const struct socket *so) 5066 { 5067 5068 return (so->so_options); 5069 } 5070 5071 void 5072 so_options_set(struct socket *so, int val) 5073 { 5074 5075 so->so_options = val; 5076 } 5077 5078 int 5079 so_error_get(const struct socket *so) 5080 { 5081 5082 return (so->so_error); 5083 } 5084 5085 void 5086 so_error_set(struct socket *so, int val) 5087 { 5088 5089 so->so_error = val; 5090 } 5091