1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1988, 1990, 1993 5 * The Regents of the University of California. 6 * Copyright (c) 2004 The FreeBSD Foundation 7 * Copyright (c) 2004-2008 Robert N. M. Watson 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pr_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pr_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pr_attach() has 50 * been successfully called. If pr_attach() returned an error, 51 * pr_detach() will not be called. Socket layer private. 52 * 53 * pr_abort() and pr_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pr_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. 58 * 59 * socreate() creates a socket and attaches protocol state. This is a public 60 * interface that may be used by socket layer consumers to create new 61 * sockets. 62 * 63 * sonewconn() creates a socket and attaches protocol state. This is a 64 * public interface that may be used by protocols to create new sockets when 65 * a new connection is received and will be available for accept() on a 66 * listen socket. 67 * 68 * soclose() destroys a socket after possibly waiting for it to disconnect. 69 * This is a public interface that socket consumers should use to close and 70 * release a socket when done with it. 71 * 72 * soabort() destroys a socket without waiting for it to disconnect (used 73 * only for incoming connections that are already partially or fully 74 * connected). This is used internally by the socket layer when clearing 75 * listen socket queues (due to overflow or close on the listen socket), but 76 * is also a public interface protocols may use to abort connections in 77 * their incomplete listen queues should they no longer be required. Sockets 78 * placed in completed connection listen queues should not be aborted for 79 * reasons described in the comment above the soclose() implementation. This 80 * is not a general purpose close routine, and except in the specific 81 * circumstances described here, should not be used. 82 * 83 * sofree() will free a socket and its protocol state if all references on 84 * the socket have been released, and is the public interface to attempt to 85 * free a socket when a reference is removed. This is a socket layer private 86 * interface. 87 * 88 * NOTE: In addition to socreate() and soclose(), which provide a single 89 * socket reference to the consumer to be managed as required, there are two 90 * calls to explicitly manage socket references, soref(), and sorele(). 91 * Currently, these are generally required only when transitioning a socket 92 * from a listen queue to a file descriptor, in order to prevent garbage 93 * collection of the socket at an untimely moment. For a number of reasons, 94 * these interfaces are not preferred, and should be avoided. 95 * 96 * NOTE: With regard to VNETs the general rule is that callers do not set 97 * curvnet. Exceptions to this rule include soabort(), sodisconnect(), 98 * sofree(), sorele(), sonewconn() and sorflush(), which are usually called 99 * from a pre-set VNET context. sopoll_generic() currently does not need a 100 * VNET context to be set. 101 */ 102 103 #include <sys/cdefs.h> 104 #include "opt_inet.h" 105 #include "opt_inet6.h" 106 #include "opt_kern_tls.h" 107 #include "opt_ktrace.h" 108 #include "opt_sctp.h" 109 110 #include <sys/param.h> 111 #include <sys/systm.h> 112 #include <sys/capsicum.h> 113 #include <sys/fcntl.h> 114 #include <sys/limits.h> 115 #include <sys/lock.h> 116 #include <sys/mac.h> 117 #include <sys/malloc.h> 118 #include <sys/mbuf.h> 119 #include <sys/mutex.h> 120 #include <sys/domain.h> 121 #include <sys/file.h> /* for struct knote */ 122 #include <sys/hhook.h> 123 #include <sys/kernel.h> 124 #include <sys/khelp.h> 125 #include <sys/kthread.h> 126 #include <sys/ktls.h> 127 #include <sys/event.h> 128 #include <sys/eventhandler.h> 129 #include <sys/poll.h> 130 #include <sys/proc.h> 131 #include <sys/protosw.h> 132 #include <sys/sbuf.h> 133 #include <sys/socket.h> 134 #include <sys/socketvar.h> 135 #include <sys/resourcevar.h> 136 #include <net/route.h> 137 #include <sys/sched.h> 138 #include <sys/signalvar.h> 139 #include <sys/smp.h> 140 #include <sys/stat.h> 141 #include <sys/sx.h> 142 #include <sys/sysctl.h> 143 #include <sys/taskqueue.h> 144 #include <sys/uio.h> 145 #include <sys/un.h> 146 #include <sys/unpcb.h> 147 #include <sys/jail.h> 148 #include <sys/syslog.h> 149 #include <netinet/in.h> 150 #include <netinet/in_pcb.h> 151 #include <netinet/tcp.h> 152 153 #include <net/vnet.h> 154 155 #include <security/mac/mac_framework.h> 156 #include <security/mac/mac_internal.h> 157 158 #include <vm/uma.h> 159 160 #ifdef COMPAT_FREEBSD32 161 #include <sys/mount.h> 162 #include <sys/sysent.h> 163 #include <compat/freebsd32/freebsd32.h> 164 #endif 165 166 static int soreceive_generic_locked(struct socket *so, 167 struct sockaddr **psa, struct uio *uio, struct mbuf **mp, 168 struct mbuf **controlp, int *flagsp); 169 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 170 int flags); 171 static int soreceive_stream_locked(struct socket *so, struct sockbuf *sb, 172 struct sockaddr **psa, struct uio *uio, struct mbuf **mp, 173 struct mbuf **controlp, int flags); 174 static int sosend_generic_locked(struct socket *so, struct sockaddr *addr, 175 struct uio *uio, struct mbuf *top, struct mbuf *control, 176 int flags, struct thread *td); 177 static void so_rdknl_lock(void *); 178 static void so_rdknl_unlock(void *); 179 static void so_rdknl_assert_lock(void *, int); 180 static void so_wrknl_lock(void *); 181 static void so_wrknl_unlock(void *); 182 static void so_wrknl_assert_lock(void *, int); 183 184 static void filt_sordetach(struct knote *kn); 185 static int filt_soread(struct knote *kn, long hint); 186 static void filt_sowdetach(struct knote *kn); 187 static int filt_sowrite(struct knote *kn, long hint); 188 static int filt_soempty(struct knote *kn, long hint); 189 190 static const struct filterops soread_filtops = { 191 .f_isfd = 1, 192 .f_detach = filt_sordetach, 193 .f_event = filt_soread, 194 .f_copy = knote_triv_copy, 195 }; 196 static const struct filterops sowrite_filtops = { 197 .f_isfd = 1, 198 .f_detach = filt_sowdetach, 199 .f_event = filt_sowrite, 200 .f_copy = knote_triv_copy, 201 }; 202 static const struct filterops soempty_filtops = { 203 .f_isfd = 1, 204 .f_detach = filt_sowdetach, 205 .f_event = filt_soempty, 206 .f_copy = knote_triv_copy, 207 }; 208 209 so_gen_t so_gencnt; /* generation count for sockets */ 210 211 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 212 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 213 214 #define VNET_SO_ASSERT(so) \ 215 VNET_ASSERT(curvnet != NULL, \ 216 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so))); 217 218 #ifdef SOCKET_HHOOK 219 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]); 220 #define V_socket_hhh VNET(socket_hhh) 221 static inline int hhook_run_socket(struct socket *, void *, int32_t); 222 #endif 223 224 #ifdef COMPAT_FREEBSD32 225 #ifdef __amd64__ 226 /* off_t has 4-byte alignment on i386 but not on other 32-bit platforms. */ 227 #define __splice32_packed __packed 228 #else 229 #define __splice32_packed 230 #endif 231 struct splice32 { 232 int32_t sp_fd; 233 int64_t sp_max; 234 struct timeval32 sp_idle; 235 } __splice32_packed; 236 #undef __splice32_packed 237 #endif 238 239 /* 240 * Limit on the number of connections in the listen queue waiting 241 * for accept(2). 242 * NB: The original sysctl somaxconn is still available but hidden 243 * to prevent confusion about the actual purpose of this number. 244 */ 245 VNET_DEFINE_STATIC(u_int, somaxconn) = SOMAXCONN; 246 #define V_somaxconn VNET(somaxconn) 247 248 static int 249 sysctl_somaxconn(SYSCTL_HANDLER_ARGS) 250 { 251 int error; 252 u_int val; 253 254 val = V_somaxconn; 255 error = sysctl_handle_int(oidp, &val, 0, req); 256 if (error || !req->newptr ) 257 return (error); 258 259 /* 260 * The purpose of the UINT_MAX / 3 limit, is so that the formula 261 * 3 * sol_qlimit / 2 262 * below, will not overflow. 263 */ 264 265 if (val < 1 || val > UINT_MAX / 3) 266 return (EINVAL); 267 268 V_somaxconn = val; 269 return (0); 270 } 271 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue, 272 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, sizeof(u_int), 273 sysctl_somaxconn, "IU", 274 "Maximum listen socket pending connection accept queue size"); 275 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, 276 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, 277 sizeof(u_int), sysctl_somaxconn, "IU", 278 "Maximum listen socket pending connection accept queue size (compat)"); 279 280 static u_int numopensockets; 281 static int 282 sysctl_numopensockets(SYSCTL_HANDLER_ARGS) 283 { 284 u_int val; 285 286 #ifdef VIMAGE 287 if(!IS_DEFAULT_VNET(curvnet)) 288 val = curvnet->vnet_sockcnt; 289 else 290 #endif 291 val = numopensockets; 292 return (sysctl_handle_int(oidp, &val, 0, req)); 293 } 294 SYSCTL_PROC(_kern_ipc, OID_AUTO, numopensockets, 295 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE | CTLFLAG_VNET, 0, sizeof(u_int), 296 sysctl_numopensockets, "IU", "Number of open sockets"); 297 298 /* 299 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 300 * so_gencnt field. 301 */ 302 static struct mtx so_global_mtx; 303 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 304 305 /* 306 * General IPC sysctl name space, used by sockets and a variety of other IPC 307 * types. 308 */ 309 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 310 "IPC"); 311 312 /* 313 * Initialize the socket subsystem and set up the socket 314 * memory allocator. 315 */ 316 static uma_zone_t socket_zone; 317 int maxsockets; 318 319 static void 320 socket_zone_change(void *tag) 321 { 322 323 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 324 } 325 326 static int splice_init_state; 327 static struct sx splice_init_lock; 328 SX_SYSINIT(splice_init_lock, &splice_init_lock, "splice_init"); 329 330 static SYSCTL_NODE(_kern_ipc, OID_AUTO, splice, CTLFLAG_RW, 0, 331 "Settings relating to the SO_SPLICE socket option"); 332 333 static bool splice_receive_stream = true; 334 SYSCTL_BOOL(_kern_ipc_splice, OID_AUTO, receive_stream, CTLFLAG_RWTUN, 335 &splice_receive_stream, 0, 336 "Use soreceive_stream() for stream splices"); 337 338 static uma_zone_t splice_zone; 339 static struct proc *splice_proc; 340 struct splice_wq { 341 struct mtx mtx; 342 STAILQ_HEAD(, so_splice) head; 343 bool running; 344 } __aligned(CACHE_LINE_SIZE); 345 static struct splice_wq *splice_wq; 346 static uint32_t splice_index = 0; 347 348 static void so_splice_timeout(void *arg, int pending); 349 static void so_splice_xfer(struct so_splice *s); 350 static int so_unsplice(struct socket *so, bool timeout); 351 352 static void 353 splice_work_thread(void *ctx) 354 { 355 struct splice_wq *wq = ctx; 356 struct so_splice *s, *s_temp; 357 STAILQ_HEAD(, so_splice) local_head; 358 int cpu; 359 360 cpu = wq - splice_wq; 361 if (bootverbose) 362 printf("starting so_splice worker thread for CPU %d\n", cpu); 363 364 for (;;) { 365 mtx_lock(&wq->mtx); 366 while (STAILQ_EMPTY(&wq->head)) { 367 wq->running = false; 368 mtx_sleep(wq, &wq->mtx, 0, "-", 0); 369 wq->running = true; 370 } 371 STAILQ_INIT(&local_head); 372 STAILQ_CONCAT(&local_head, &wq->head); 373 STAILQ_INIT(&wq->head); 374 mtx_unlock(&wq->mtx); 375 STAILQ_FOREACH_SAFE(s, &local_head, next, s_temp) { 376 mtx_lock(&s->mtx); 377 CURVNET_SET(s->src->so_vnet); 378 so_splice_xfer(s); 379 CURVNET_RESTORE(); 380 } 381 } 382 } 383 384 static void 385 so_splice_dispatch_async(struct so_splice *sp) 386 { 387 struct splice_wq *wq; 388 bool running; 389 390 wq = &splice_wq[sp->wq_index]; 391 mtx_lock(&wq->mtx); 392 STAILQ_INSERT_TAIL(&wq->head, sp, next); 393 running = wq->running; 394 mtx_unlock(&wq->mtx); 395 if (!running) 396 wakeup(wq); 397 } 398 399 void 400 so_splice_dispatch(struct so_splice *sp) 401 { 402 mtx_assert(&sp->mtx, MA_OWNED); 403 404 if (sp->state != SPLICE_IDLE) { 405 mtx_unlock(&sp->mtx); 406 } else { 407 sp->state = SPLICE_QUEUED; 408 mtx_unlock(&sp->mtx); 409 so_splice_dispatch_async(sp); 410 } 411 } 412 413 static int 414 splice_zinit(void *mem, int size __unused, int flags __unused) 415 { 416 struct so_splice *s; 417 418 s = (struct so_splice *)mem; 419 mtx_init(&s->mtx, "so_splice", NULL, MTX_DEF); 420 return (0); 421 } 422 423 static void 424 splice_zfini(void *mem, int size) 425 { 426 struct so_splice *s; 427 428 s = (struct so_splice *)mem; 429 mtx_destroy(&s->mtx); 430 } 431 432 static int 433 splice_init(void) 434 { 435 struct thread *td; 436 int error, i, state; 437 438 state = atomic_load_acq_int(&splice_init_state); 439 if (__predict_true(state > 0)) 440 return (0); 441 if (state < 0) 442 return (ENXIO); 443 sx_xlock(&splice_init_lock); 444 if (splice_init_state != 0) { 445 sx_xunlock(&splice_init_lock); 446 return (0); 447 } 448 449 splice_zone = uma_zcreate("splice", sizeof(struct so_splice), NULL, 450 NULL, splice_zinit, splice_zfini, UMA_ALIGN_CACHE, 0); 451 452 splice_wq = mallocarray(mp_maxid + 1, sizeof(*splice_wq), M_TEMP, 453 M_WAITOK | M_ZERO); 454 455 /* 456 * Initialize the workqueues to run the splice work. We create a 457 * work queue for each CPU. 458 */ 459 CPU_FOREACH(i) { 460 STAILQ_INIT(&splice_wq[i].head); 461 mtx_init(&splice_wq[i].mtx, "splice work queue", NULL, MTX_DEF); 462 } 463 464 /* Start kthreads for each workqueue. */ 465 error = 0; 466 CPU_FOREACH(i) { 467 error = kproc_kthread_add(splice_work_thread, &splice_wq[i], 468 &splice_proc, &td, 0, 0, "so_splice", "thr_%d", i); 469 if (error) { 470 printf("Can't add so_splice thread %d error %d\n", 471 i, error); 472 break; 473 } 474 475 /* 476 * It's possible to create loops with SO_SPLICE; ensure that 477 * worker threads aren't able to starve the system too easily. 478 */ 479 thread_lock(td); 480 sched_prio(td, PUSER); 481 thread_unlock(td); 482 } 483 484 splice_init_state = error != 0 ? -1 : 1; 485 sx_xunlock(&splice_init_lock); 486 487 return (error); 488 } 489 490 /* 491 * Lock a pair of socket's I/O locks for splicing. Avoid blocking while holding 492 * one lock in order to avoid potential deadlocks in case there is some other 493 * code path which acquires more than one I/O lock at a time. 494 */ 495 static void 496 splice_lock_pair(struct socket *so_src, struct socket *so_dst) 497 { 498 int error; 499 500 for (;;) { 501 error = SOCK_IO_SEND_LOCK(so_dst, SBL_WAIT | SBL_NOINTR); 502 KASSERT(error == 0, 503 ("%s: failed to lock send I/O lock: %d", __func__, error)); 504 error = SOCK_IO_RECV_LOCK(so_src, 0); 505 KASSERT(error == 0 || error == EWOULDBLOCK, 506 ("%s: failed to lock recv I/O lock: %d", __func__, error)); 507 if (error == 0) 508 break; 509 SOCK_IO_SEND_UNLOCK(so_dst); 510 511 error = SOCK_IO_RECV_LOCK(so_src, SBL_WAIT | SBL_NOINTR); 512 KASSERT(error == 0, 513 ("%s: failed to lock recv I/O lock: %d", __func__, error)); 514 error = SOCK_IO_SEND_LOCK(so_dst, 0); 515 KASSERT(error == 0 || error == EWOULDBLOCK, 516 ("%s: failed to lock send I/O lock: %d", __func__, error)); 517 if (error == 0) 518 break; 519 SOCK_IO_RECV_UNLOCK(so_src); 520 } 521 } 522 523 static void 524 splice_unlock_pair(struct socket *so_src, struct socket *so_dst) 525 { 526 SOCK_IO_RECV_UNLOCK(so_src); 527 SOCK_IO_SEND_UNLOCK(so_dst); 528 } 529 530 /* 531 * Move data from the source to the sink. Assumes that both of the relevant 532 * socket I/O locks are held. 533 */ 534 static int 535 so_splice_xfer_data(struct socket *so_src, struct socket *so_dst, off_t max, 536 ssize_t *lenp) 537 { 538 struct uio uio; 539 struct mbuf *m; 540 struct sockbuf *sb_src, *sb_dst; 541 ssize_t len; 542 long space; 543 int error, flags; 544 545 SOCK_IO_RECV_ASSERT_LOCKED(so_src); 546 SOCK_IO_SEND_ASSERT_LOCKED(so_dst); 547 548 error = 0; 549 m = NULL; 550 memset(&uio, 0, sizeof(uio)); 551 552 sb_src = &so_src->so_rcv; 553 sb_dst = &so_dst->so_snd; 554 555 space = sbspace(sb_dst); 556 if (space < 0) 557 space = 0; 558 len = MIN(max, MIN(space, sbavail(sb_src))); 559 if (len == 0) { 560 SOCK_RECVBUF_LOCK(so_src); 561 if ((sb_src->sb_state & SBS_CANTRCVMORE) != 0) 562 error = EPIPE; 563 SOCK_RECVBUF_UNLOCK(so_src); 564 } else { 565 flags = MSG_DONTWAIT; 566 uio.uio_resid = len; 567 if (splice_receive_stream && sb_src->sb_tls_info == NULL) { 568 error = soreceive_stream_locked(so_src, sb_src, NULL, 569 &uio, &m, NULL, flags); 570 } else { 571 error = soreceive_generic_locked(so_src, NULL, 572 &uio, &m, NULL, &flags); 573 } 574 if (error != 0 && m != NULL) { 575 m_freem(m); 576 m = NULL; 577 } 578 } 579 if (m != NULL) { 580 len -= uio.uio_resid; 581 error = sosend_generic_locked(so_dst, NULL, NULL, m, NULL, 582 MSG_DONTWAIT, curthread); 583 } else if (error == 0) { 584 len = 0; 585 SOCK_SENDBUF_LOCK(so_dst); 586 if ((sb_dst->sb_state & SBS_CANTSENDMORE) != 0) 587 error = EPIPE; 588 SOCK_SENDBUF_UNLOCK(so_dst); 589 } 590 if (error == 0) 591 *lenp = len; 592 return (error); 593 } 594 595 /* 596 * Transfer data from the source to the sink. 597 */ 598 static void 599 so_splice_xfer(struct so_splice *sp) 600 { 601 struct socket *so_src, *so_dst; 602 off_t max; 603 ssize_t len; 604 int error; 605 606 mtx_assert(&sp->mtx, MA_OWNED); 607 KASSERT(sp->state == SPLICE_QUEUED || sp->state == SPLICE_CLOSING, 608 ("so_splice_xfer: invalid state %d", sp->state)); 609 KASSERT(sp->max != 0, ("so_splice_xfer: max == 0")); 610 611 if (sp->state == SPLICE_CLOSING) { 612 /* Userspace asked us to close the splice. */ 613 goto closing; 614 } 615 616 sp->state = SPLICE_RUNNING; 617 so_src = sp->src; 618 so_dst = sp->dst; 619 max = sp->max > 0 ? sp->max - so_src->so_splice_sent : OFF_MAX; 620 if (max < 0) 621 max = 0; 622 623 /* 624 * Lock the sockets in order to block userspace from doing anything 625 * sneaky. If an error occurs or one of the sockets can no longer 626 * transfer data, we will automatically unsplice. 627 */ 628 mtx_unlock(&sp->mtx); 629 splice_lock_pair(so_src, so_dst); 630 631 error = so_splice_xfer_data(so_src, so_dst, max, &len); 632 633 mtx_lock(&sp->mtx); 634 635 /* 636 * Update our stats while still holding the socket locks. This 637 * synchronizes with getsockopt(SO_SPLICE), see the comment there. 638 */ 639 if (error == 0) { 640 KASSERT(len >= 0, ("%s: len %zd < 0", __func__, len)); 641 so_src->so_splice_sent += len; 642 } 643 splice_unlock_pair(so_src, so_dst); 644 645 switch (sp->state) { 646 case SPLICE_CLOSING: 647 closing: 648 sp->state = SPLICE_CLOSED; 649 wakeup(sp); 650 mtx_unlock(&sp->mtx); 651 break; 652 case SPLICE_RUNNING: 653 if (error != 0 || 654 (sp->max > 0 && so_src->so_splice_sent >= sp->max)) { 655 sp->state = SPLICE_EXCEPTION; 656 soref(so_src); 657 mtx_unlock(&sp->mtx); 658 (void)so_unsplice(so_src, false); 659 sorele(so_src); 660 } else { 661 /* 662 * Locklessly check for additional bytes in the source's 663 * receive buffer and queue more work if possible. We 664 * may end up queuing needless work, but that's ok, and 665 * if we race with a thread inserting more data into the 666 * buffer and observe sbavail() == 0, the splice mutex 667 * ensures that splice_push() will queue more work for 668 * us. 669 */ 670 if (sbavail(&so_src->so_rcv) > 0 && 671 sbspace(&so_dst->so_snd) > 0) { 672 sp->state = SPLICE_QUEUED; 673 mtx_unlock(&sp->mtx); 674 so_splice_dispatch_async(sp); 675 } else { 676 sp->state = SPLICE_IDLE; 677 mtx_unlock(&sp->mtx); 678 } 679 } 680 break; 681 default: 682 __assert_unreachable(); 683 } 684 } 685 686 static void 687 socket_init(void *tag) 688 { 689 690 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL, 691 NULL, NULL, UMA_ALIGN_PTR, 0); 692 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 693 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached"); 694 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL, 695 EVENTHANDLER_PRI_FIRST); 696 } 697 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL); 698 699 #ifdef SOCKET_HHOOK 700 static void 701 socket_hhook_register(int subtype) 702 { 703 704 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype, 705 &V_socket_hhh[subtype], 706 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0) 707 printf("%s: WARNING: unable to register hook\n", __func__); 708 } 709 710 static void 711 socket_hhook_deregister(int subtype) 712 { 713 714 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0) 715 printf("%s: WARNING: unable to deregister hook\n", __func__); 716 } 717 718 static void 719 socket_vnet_init(const void *unused __unused) 720 { 721 int i; 722 723 /* We expect a contiguous range */ 724 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 725 socket_hhook_register(i); 726 } 727 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 728 socket_vnet_init, NULL); 729 730 static void 731 socket_vnet_uninit(const void *unused __unused) 732 { 733 int i; 734 735 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 736 socket_hhook_deregister(i); 737 } 738 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 739 socket_vnet_uninit, NULL); 740 #endif /* SOCKET_HHOOK */ 741 742 /* 743 * Initialise maxsockets. This SYSINIT must be run after 744 * tunable_mbinit(). 745 */ 746 static void 747 init_maxsockets(void *ignored) 748 { 749 750 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 751 maxsockets = imax(maxsockets, maxfiles); 752 } 753 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 754 755 /* 756 * Sysctl to get and set the maximum global sockets limit. Notify protocols 757 * of the change so that they can update their dependent limits as required. 758 */ 759 static int 760 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 761 { 762 int error, newmaxsockets; 763 764 newmaxsockets = maxsockets; 765 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req); 766 if (error == 0 && req->newptr && newmaxsockets != maxsockets) { 767 if (newmaxsockets > maxsockets && 768 newmaxsockets <= maxfiles) { 769 maxsockets = newmaxsockets; 770 EVENTHANDLER_INVOKE(maxsockets_change); 771 } else 772 error = EINVAL; 773 } 774 return (error); 775 } 776 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, 777 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, 778 &maxsockets, 0, sysctl_maxsockets, "IU", 779 "Maximum number of sockets available"); 780 781 /* 782 * Socket operation routines. These routines are called by the routines in 783 * sys_socket.c or from a system process, and implement the semantics of 784 * socket operations by switching out to the protocol specific routines. 785 */ 786 787 /* 788 * Get a socket structure from our zone, and initialize it. Note that it 789 * would probably be better to allocate socket and PCB at the same time, but 790 * I'm not convinced that all the protocols can be easily modified to do 791 * this. 792 * 793 * soalloc() returns a socket with a ref count of 0. 794 */ 795 static struct socket * 796 soalloc(struct vnet *vnet) 797 { 798 struct socket *so; 799 800 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); 801 if (so == NULL) 802 return (NULL); 803 #ifdef MAC 804 if (mac_socket_init(so, M_NOWAIT) != 0) { 805 uma_zfree(socket_zone, so); 806 return (NULL); 807 } 808 #endif 809 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { 810 uma_zfree(socket_zone, so); 811 return (NULL); 812 } 813 814 /* 815 * The socket locking protocol allows to lock 2 sockets at a time, 816 * however, the first one must be a listening socket. WITNESS lacks 817 * a feature to change class of an existing lock, so we use DUPOK. 818 */ 819 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); 820 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); 821 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); 822 so->so_rcv.sb_sel = &so->so_rdsel; 823 so->so_snd.sb_sel = &so->so_wrsel; 824 sx_init(&so->so_snd_sx, "so_snd_sx"); 825 sx_init(&so->so_rcv_sx, "so_rcv_sx"); 826 TAILQ_INIT(&so->so_snd.sb_aiojobq); 827 TAILQ_INIT(&so->so_rcv.sb_aiojobq); 828 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); 829 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); 830 #ifdef VIMAGE 831 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", 832 __func__, __LINE__, so)); 833 so->so_vnet = vnet; 834 #endif 835 #ifdef SOCKET_HHOOK 836 /* We shouldn't need the so_global_mtx */ 837 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { 838 /* Do we need more comprehensive error returns? */ 839 uma_zfree(socket_zone, so); 840 return (NULL); 841 } 842 #endif 843 mtx_lock(&so_global_mtx); 844 so->so_gencnt = ++so_gencnt; 845 ++numopensockets; 846 #ifdef VIMAGE 847 vnet->vnet_sockcnt++; 848 #endif 849 mtx_unlock(&so_global_mtx); 850 851 return (so); 852 } 853 854 /* 855 * Free the storage associated with a socket at the socket layer, tear down 856 * locks, labels, etc. All protocol state is assumed already to have been 857 * torn down (and possibly never set up) by the caller. 858 */ 859 void 860 sodealloc(struct socket *so) 861 { 862 863 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 864 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 865 866 mtx_lock(&so_global_mtx); 867 so->so_gencnt = ++so_gencnt; 868 --numopensockets; /* Could be below, but faster here. */ 869 #ifdef VIMAGE 870 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", 871 __func__, __LINE__, so)); 872 so->so_vnet->vnet_sockcnt--; 873 #endif 874 mtx_unlock(&so_global_mtx); 875 #ifdef MAC 876 mac_socket_destroy(so); 877 #endif 878 #ifdef SOCKET_HHOOK 879 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); 880 #endif 881 882 khelp_destroy_osd(&so->osd); 883 if (SOLISTENING(so)) { 884 if (so->sol_accept_filter != NULL) 885 accept_filt_setopt(so, NULL); 886 } else { 887 if (so->so_rcv.sb_hiwat) 888 (void)chgsbsize(so->so_cred->cr_uidinfo, 889 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 890 if (so->so_snd.sb_hiwat) 891 (void)chgsbsize(so->so_cred->cr_uidinfo, 892 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 893 sx_destroy(&so->so_snd_sx); 894 sx_destroy(&so->so_rcv_sx); 895 mtx_destroy(&so->so_snd_mtx); 896 mtx_destroy(&so->so_rcv_mtx); 897 } 898 crfree(so->so_cred); 899 mtx_destroy(&so->so_lock); 900 uma_zfree(socket_zone, so); 901 } 902 903 /* 904 * socreate returns a socket with a ref count of 1 and a file descriptor 905 * reference. The socket should be closed with soclose(). 906 */ 907 int 908 socreate(int dom, struct socket **aso, int type, int proto, 909 struct ucred *cred, struct thread *td) 910 { 911 struct protosw *prp; 912 struct socket *so; 913 int error; 914 915 /* 916 * XXX: divert(4) historically abused PF_INET. Keep this compatibility 917 * shim until all applications have been updated. 918 */ 919 if (__predict_false(dom == PF_INET && type == SOCK_RAW && 920 proto == IPPROTO_DIVERT)) { 921 dom = PF_DIVERT; 922 printf("%s uses obsolete way to create divert(4) socket\n", 923 td->td_proc->p_comm); 924 } 925 926 prp = pffindproto(dom, type, proto); 927 if (prp == NULL) { 928 /* No support for domain. */ 929 if (pffinddomain(dom) == NULL) 930 return (EAFNOSUPPORT); 931 /* No support for socket type. */ 932 if (proto == 0 && type != 0) 933 return (EPROTOTYPE); 934 return (EPROTONOSUPPORT); 935 } 936 937 MPASS(prp->pr_attach); 938 939 if ((prp->pr_flags & PR_CAPATTACH) == 0) { 940 if (CAP_TRACING(td)) 941 ktrcapfail(CAPFAIL_PROTO, &proto); 942 if (IN_CAPABILITY_MODE(td)) 943 return (ECAPMODE); 944 } 945 946 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0) 947 return (EPROTONOSUPPORT); 948 949 so = soalloc(CRED_TO_VNET(cred)); 950 if (so == NULL) 951 return (ENOBUFS); 952 953 so->so_type = type; 954 so->so_cred = crhold(cred); 955 if ((prp->pr_domain->dom_family == PF_INET) || 956 (prp->pr_domain->dom_family == PF_INET6) || 957 (prp->pr_domain->dom_family == PF_ROUTE)) 958 so->so_fibnum = td->td_proc->p_fibnum; 959 else 960 so->so_fibnum = 0; 961 so->so_proto = prp; 962 #ifdef MAC 963 mac_socket_create(cred, so); 964 #endif 965 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 966 so_rdknl_assert_lock); 967 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 968 so_wrknl_assert_lock); 969 if ((prp->pr_flags & PR_SOCKBUF) == 0) { 970 so->so_snd.sb_mtx = &so->so_snd_mtx; 971 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 972 } 973 /* 974 * Auto-sizing of socket buffers is managed by the protocols and 975 * the appropriate flags must be set in the pr_attach() method. 976 */ 977 CURVNET_SET(so->so_vnet); 978 error = prp->pr_attach(so, proto, td); 979 CURVNET_RESTORE(); 980 if (error) { 981 sodealloc(so); 982 return (error); 983 } 984 soref(so); 985 *aso = so; 986 return (0); 987 } 988 989 #ifdef REGRESSION 990 static int regression_sonewconn_earlytest = 1; 991 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 992 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 993 #endif 994 995 static int sooverprio = LOG_DEBUG; 996 SYSCTL_INT(_kern_ipc, OID_AUTO, sooverprio, CTLFLAG_RW, 997 &sooverprio, 0, "Log priority for listen socket overflows: 0..7 or -1 to disable"); 998 999 static struct timeval overinterval = { 60, 0 }; 1000 SYSCTL_TIMEVAL_SEC(_kern_ipc, OID_AUTO, sooverinterval, CTLFLAG_RW, 1001 &overinterval, 1002 "Delay in seconds between warnings for listen socket overflows"); 1003 1004 /* 1005 * When an attempt at a new connection is noted on a socket which supports 1006 * accept(2), the protocol has two options: 1007 * 1) Call legacy sonewconn() function, which would call protocol attach 1008 * method, same as used for socket(2). 1009 * 2) Call solisten_clone(), do attach that is specific to a cloned connection, 1010 * and then call solisten_enqueue(). 1011 * 1012 * Note: the ref count on the socket is 0 on return. 1013 */ 1014 struct socket * 1015 solisten_clone(struct socket *head) 1016 { 1017 struct sbuf descrsb; 1018 struct socket *so; 1019 int len, overcount; 1020 u_int qlen; 1021 const char localprefix[] = "local:"; 1022 char descrbuf[SUNPATHLEN + sizeof(localprefix)]; 1023 #if defined(INET6) 1024 char addrbuf[INET6_ADDRSTRLEN]; 1025 #elif defined(INET) 1026 char addrbuf[INET_ADDRSTRLEN]; 1027 #endif 1028 bool dolog, over; 1029 1030 SOLISTEN_LOCK(head); 1031 over = (head->sol_qlen > 3 * head->sol_qlimit / 2); 1032 #ifdef REGRESSION 1033 if (regression_sonewconn_earlytest && over) { 1034 #else 1035 if (over) { 1036 #endif 1037 head->sol_overcount++; 1038 dolog = (sooverprio >= 0) && 1039 !!ratecheck(&head->sol_lastover, &overinterval); 1040 1041 /* 1042 * If we're going to log, copy the overflow count and queue 1043 * length from the listen socket before dropping the lock. 1044 * Also, reset the overflow count. 1045 */ 1046 if (dolog) { 1047 overcount = head->sol_overcount; 1048 head->sol_overcount = 0; 1049 qlen = head->sol_qlen; 1050 } 1051 SOLISTEN_UNLOCK(head); 1052 1053 if (dolog) { 1054 /* 1055 * Try to print something descriptive about the 1056 * socket for the error message. 1057 */ 1058 sbuf_new(&descrsb, descrbuf, sizeof(descrbuf), 1059 SBUF_FIXEDLEN); 1060 switch (head->so_proto->pr_domain->dom_family) { 1061 #if defined(INET) || defined(INET6) 1062 #ifdef INET 1063 case AF_INET: 1064 #endif 1065 #ifdef INET6 1066 case AF_INET6: 1067 if (head->so_proto->pr_domain->dom_family == 1068 AF_INET6 || 1069 (sotoinpcb(head)->inp_inc.inc_flags & 1070 INC_ISIPV6)) { 1071 ip6_sprintf(addrbuf, 1072 &sotoinpcb(head)->inp_inc.inc6_laddr); 1073 sbuf_printf(&descrsb, "[%s]", addrbuf); 1074 } else 1075 #endif 1076 { 1077 #ifdef INET 1078 inet_ntoa_r( 1079 sotoinpcb(head)->inp_inc.inc_laddr, 1080 addrbuf); 1081 sbuf_cat(&descrsb, addrbuf); 1082 #endif 1083 } 1084 sbuf_printf(&descrsb, ":%hu (proto %u)", 1085 ntohs(sotoinpcb(head)->inp_inc.inc_lport), 1086 head->so_proto->pr_protocol); 1087 break; 1088 #endif /* INET || INET6 */ 1089 case AF_UNIX: 1090 sbuf_cat(&descrsb, localprefix); 1091 if (sotounpcb(head)->unp_addr != NULL) 1092 len = 1093 sotounpcb(head)->unp_addr->sun_len - 1094 offsetof(struct sockaddr_un, 1095 sun_path); 1096 else 1097 len = 0; 1098 if (len > 0) 1099 sbuf_bcat(&descrsb, 1100 sotounpcb(head)->unp_addr->sun_path, 1101 len); 1102 else 1103 sbuf_cat(&descrsb, "(unknown)"); 1104 break; 1105 } 1106 1107 /* 1108 * If we can't print something more specific, at least 1109 * print the domain name. 1110 */ 1111 if (sbuf_finish(&descrsb) != 0 || 1112 sbuf_len(&descrsb) <= 0) { 1113 sbuf_clear(&descrsb); 1114 sbuf_cat(&descrsb, 1115 head->so_proto->pr_domain->dom_name ?: 1116 "unknown"); 1117 sbuf_finish(&descrsb); 1118 } 1119 KASSERT(sbuf_len(&descrsb) > 0, 1120 ("%s: sbuf creation failed", __func__)); 1121 /* 1122 * Preserve the historic listen queue overflow log 1123 * message, that starts with "sonewconn:". It has 1124 * been known to sysadmins for years and also test 1125 * sys/kern/sonewconn_overflow checks for it. 1126 */ 1127 if (head->so_cred == 0) { 1128 log(LOG_PRI(sooverprio), 1129 "sonewconn: pcb %p (%s): " 1130 "Listen queue overflow: %i already in " 1131 "queue awaiting acceptance (%d " 1132 "occurrences)\n", head->so_pcb, 1133 sbuf_data(&descrsb), 1134 qlen, overcount); 1135 } else { 1136 log(LOG_PRI(sooverprio), 1137 "sonewconn: pcb %p (%s): " 1138 "Listen queue overflow: " 1139 "%i already in queue awaiting acceptance " 1140 "(%d occurrences), euid %d, rgid %d, jail %s\n", 1141 head->so_pcb, sbuf_data(&descrsb), qlen, 1142 overcount, head->so_cred->cr_uid, 1143 head->so_cred->cr_rgid, 1144 head->so_cred->cr_prison ? 1145 head->so_cred->cr_prison->pr_name : 1146 "not_jailed"); 1147 } 1148 sbuf_delete(&descrsb); 1149 1150 overcount = 0; 1151 } 1152 1153 return (NULL); 1154 } 1155 SOLISTEN_UNLOCK(head); 1156 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL", 1157 __func__, head)); 1158 so = soalloc(head->so_vnet); 1159 if (so == NULL) { 1160 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 1161 "limit reached or out of memory\n", 1162 __func__, head->so_pcb); 1163 return (NULL); 1164 } 1165 so->so_listen = head; 1166 so->so_type = head->so_type; 1167 /* 1168 * POSIX is ambiguous on what options an accept(2)ed socket should 1169 * inherit from the listener. Words "create a new socket" may be 1170 * interpreted as not inheriting anything. Best programming practice 1171 * for application developers is to not rely on such inheritance. 1172 * FreeBSD had historically inherited all so_options excluding 1173 * SO_ACCEPTCONN, which virtually means all SOL_SOCKET level options, 1174 * including those completely irrelevant to a new born socket. For 1175 * compatibility with older versions we will inherit a list of 1176 * meaningful options. 1177 * The crucial bit to inherit is SO_ACCEPTFILTER. We need it present 1178 * in the child socket for soisconnected() promoting socket from the 1179 * incomplete queue to complete. It will be cleared before the child 1180 * gets available to accept(2). 1181 */ 1182 so->so_options = head->so_options & (SO_ACCEPTFILTER | SO_KEEPALIVE | 1183 SO_DONTROUTE | SO_LINGER | SO_OOBINLINE | SO_NOSIGPIPE); 1184 so->so_linger = head->so_linger; 1185 so->so_state = head->so_state; 1186 so->so_fibnum = head->so_fibnum; 1187 so->so_proto = head->so_proto; 1188 so->so_cred = crhold(head->so_cred); 1189 #ifdef SOCKET_HHOOK 1190 if (V_socket_hhh[HHOOK_SOCKET_NEWCONN]->hhh_nhooks > 0) { 1191 if (hhook_run_socket(so, head, HHOOK_SOCKET_NEWCONN)) { 1192 sodealloc(so); 1193 log(LOG_DEBUG, "%s: hhook run failed\n", __func__); 1194 return (NULL); 1195 } 1196 } 1197 #endif 1198 #ifdef MAC 1199 mac_socket_newconn(head, so); 1200 #endif 1201 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1202 so_rdknl_assert_lock); 1203 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1204 so_wrknl_assert_lock); 1205 VNET_SO_ASSERT(head); 1206 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) { 1207 sodealloc(so); 1208 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 1209 __func__, head->so_pcb); 1210 return (NULL); 1211 } 1212 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; 1213 so->so_snd.sb_lowat = head->sol_sbsnd_lowat; 1214 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; 1215 so->so_snd.sb_timeo = head->sol_sbsnd_timeo; 1216 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE; 1217 so->so_snd.sb_flags = head->sol_sbsnd_flags & 1218 (SB_AUTOSIZE | SB_AUTOLOWAT); 1219 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { 1220 so->so_snd.sb_mtx = &so->so_snd_mtx; 1221 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 1222 } 1223 1224 return (so); 1225 } 1226 1227 /* Connstatus may be 0 or SS_ISCONNECTED. */ 1228 struct socket * 1229 sonewconn(struct socket *head, int connstatus) 1230 { 1231 struct socket *so; 1232 1233 if ((so = solisten_clone(head)) == NULL) 1234 return (NULL); 1235 1236 if (so->so_proto->pr_attach(so, 0, NULL) != 0) { 1237 sodealloc(so); 1238 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", 1239 __func__, head->so_pcb); 1240 return (NULL); 1241 } 1242 1243 (void)solisten_enqueue(so, connstatus); 1244 1245 return (so); 1246 } 1247 1248 /* 1249 * Enqueue socket cloned by solisten_clone() to the listen queue of the 1250 * listener it has been cloned from. 1251 * 1252 * Return 'true' if socket landed on complete queue, otherwise 'false'. 1253 */ 1254 bool 1255 solisten_enqueue(struct socket *so, int connstatus) 1256 { 1257 struct socket *head = so->so_listen; 1258 1259 MPASS(refcount_load(&so->so_count) == 0); 1260 refcount_init(&so->so_count, 1); 1261 1262 SOLISTEN_LOCK(head); 1263 if (head->sol_accept_filter != NULL) 1264 connstatus = 0; 1265 so->so_state |= connstatus; 1266 soref(head); /* A socket on (in)complete queue refs head. */ 1267 if (connstatus) { 1268 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 1269 so->so_qstate = SQ_COMP; 1270 head->sol_qlen++; 1271 solisten_wakeup(head); /* unlocks */ 1272 return (true); 1273 } else { 1274 /* 1275 * Keep removing sockets from the head until there's room for 1276 * us to insert on the tail. In pre-locking revisions, this 1277 * was a simple if(), but as we could be racing with other 1278 * threads and soabort() requires dropping locks, we must 1279 * loop waiting for the condition to be true. 1280 */ 1281 while (head->sol_incqlen > head->sol_qlimit) { 1282 struct socket *sp; 1283 1284 sp = TAILQ_FIRST(&head->sol_incomp); 1285 TAILQ_REMOVE(&head->sol_incomp, sp, so_list); 1286 head->sol_incqlen--; 1287 SOCK_LOCK(sp); 1288 sp->so_qstate = SQ_NONE; 1289 sp->so_listen = NULL; 1290 SOCK_UNLOCK(sp); 1291 sorele_locked(head); /* does SOLISTEN_UNLOCK, head stays */ 1292 soabort(sp); 1293 SOLISTEN_LOCK(head); 1294 } 1295 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list); 1296 so->so_qstate = SQ_INCOMP; 1297 head->sol_incqlen++; 1298 SOLISTEN_UNLOCK(head); 1299 return (false); 1300 } 1301 } 1302 1303 #if defined(SCTP) || defined(SCTP_SUPPORT) 1304 /* 1305 * Socket part of sctp_peeloff(). Detach a new socket from an 1306 * association. The new socket is returned with a reference. 1307 * 1308 * XXXGL: reduce copy-paste with solisten_clone(). 1309 */ 1310 struct socket * 1311 sopeeloff(struct socket *head) 1312 { 1313 struct socket *so; 1314 1315 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", 1316 __func__, __LINE__, head)); 1317 so = soalloc(head->so_vnet); 1318 if (so == NULL) { 1319 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 1320 "limit reached or out of memory\n", 1321 __func__, head->so_pcb); 1322 return (NULL); 1323 } 1324 so->so_type = head->so_type; 1325 so->so_options = head->so_options; 1326 so->so_linger = head->so_linger; 1327 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; 1328 so->so_fibnum = head->so_fibnum; 1329 so->so_proto = head->so_proto; 1330 so->so_cred = crhold(head->so_cred); 1331 #ifdef MAC 1332 mac_socket_newconn(head, so); 1333 #endif 1334 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 1335 so_rdknl_assert_lock); 1336 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 1337 so_wrknl_assert_lock); 1338 VNET_SO_ASSERT(head); 1339 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 1340 sodealloc(so); 1341 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 1342 __func__, head->so_pcb); 1343 return (NULL); 1344 } 1345 if (so->so_proto->pr_attach(so, 0, NULL)) { 1346 sodealloc(so); 1347 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n", 1348 __func__, head->so_pcb); 1349 return (NULL); 1350 } 1351 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 1352 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 1353 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 1354 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 1355 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 1356 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 1357 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) { 1358 so->so_snd.sb_mtx = &so->so_snd_mtx; 1359 so->so_rcv.sb_mtx = &so->so_rcv_mtx; 1360 } 1361 1362 soref(so); 1363 1364 return (so); 1365 } 1366 #endif /* SCTP */ 1367 1368 int 1369 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) 1370 { 1371 int error; 1372 1373 CURVNET_SET(so->so_vnet); 1374 error = so->so_proto->pr_bind(so, nam, td); 1375 CURVNET_RESTORE(); 1376 return (error); 1377 } 1378 1379 int 1380 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 1381 { 1382 int error; 1383 1384 CURVNET_SET(so->so_vnet); 1385 error = so->so_proto->pr_bindat(fd, so, nam, td); 1386 CURVNET_RESTORE(); 1387 return (error); 1388 } 1389 1390 /* 1391 * solisten() transitions a socket from a non-listening state to a listening 1392 * state, but can also be used to update the listen queue depth on an 1393 * existing listen socket. The protocol will call back into the sockets 1394 * layer using solisten_proto_check() and solisten_proto() to check and set 1395 * socket-layer listen state. Call backs are used so that the protocol can 1396 * acquire both protocol and socket layer locks in whatever order is required 1397 * by the protocol. 1398 * 1399 * Protocol implementors are advised to hold the socket lock across the 1400 * socket-layer test and set to avoid races at the socket layer. 1401 */ 1402 int 1403 solisten(struct socket *so, int backlog, struct thread *td) 1404 { 1405 int error; 1406 1407 CURVNET_SET(so->so_vnet); 1408 error = so->so_proto->pr_listen(so, backlog, td); 1409 CURVNET_RESTORE(); 1410 return (error); 1411 } 1412 1413 /* 1414 * Prepare for a call to solisten_proto(). Acquire all socket buffer locks in 1415 * order to interlock with socket I/O. 1416 */ 1417 int 1418 solisten_proto_check(struct socket *so) 1419 { 1420 SOCK_LOCK_ASSERT(so); 1421 1422 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 1423 SS_ISDISCONNECTING)) != 0) 1424 return (EINVAL); 1425 1426 /* 1427 * Sleeping is not permitted here, so simply fail if userspace is 1428 * attempting to transmit or receive on the socket. This kind of 1429 * transient failure is not ideal, but it should occur only if userspace 1430 * is misusing the socket interfaces. 1431 */ 1432 if (!sx_try_xlock(&so->so_snd_sx)) 1433 return (EAGAIN); 1434 if (!sx_try_xlock(&so->so_rcv_sx)) { 1435 sx_xunlock(&so->so_snd_sx); 1436 return (EAGAIN); 1437 } 1438 mtx_lock(&so->so_snd_mtx); 1439 mtx_lock(&so->so_rcv_mtx); 1440 1441 /* Interlock with soo_aio_queue() and KTLS. */ 1442 if (!SOLISTENING(so)) { 1443 bool ktls; 1444 1445 #ifdef KERN_TLS 1446 ktls = so->so_snd.sb_tls_info != NULL || 1447 so->so_rcv.sb_tls_info != NULL; 1448 #else 1449 ktls = false; 1450 #endif 1451 if (ktls || 1452 (so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 || 1453 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0) { 1454 solisten_proto_abort(so); 1455 return (EINVAL); 1456 } 1457 } 1458 1459 return (0); 1460 } 1461 1462 /* 1463 * Undo the setup done by solisten_proto_check(). 1464 */ 1465 void 1466 solisten_proto_abort(struct socket *so) 1467 { 1468 mtx_unlock(&so->so_snd_mtx); 1469 mtx_unlock(&so->so_rcv_mtx); 1470 sx_xunlock(&so->so_snd_sx); 1471 sx_xunlock(&so->so_rcv_sx); 1472 } 1473 1474 void 1475 solisten_proto(struct socket *so, int backlog) 1476 { 1477 int sbrcv_lowat, sbsnd_lowat; 1478 u_int sbrcv_hiwat, sbsnd_hiwat; 1479 short sbrcv_flags, sbsnd_flags; 1480 sbintime_t sbrcv_timeo, sbsnd_timeo; 1481 1482 SOCK_LOCK_ASSERT(so); 1483 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 1484 SS_ISDISCONNECTING)) == 0, 1485 ("%s: bad socket state %p", __func__, so)); 1486 1487 if (SOLISTENING(so)) 1488 goto listening; 1489 1490 /* 1491 * Change this socket to listening state. 1492 */ 1493 sbrcv_lowat = so->so_rcv.sb_lowat; 1494 sbsnd_lowat = so->so_snd.sb_lowat; 1495 sbrcv_hiwat = so->so_rcv.sb_hiwat; 1496 sbsnd_hiwat = so->so_snd.sb_hiwat; 1497 sbrcv_flags = so->so_rcv.sb_flags; 1498 sbsnd_flags = so->so_snd.sb_flags; 1499 sbrcv_timeo = so->so_rcv.sb_timeo; 1500 sbsnd_timeo = so->so_snd.sb_timeo; 1501 1502 #ifdef MAC 1503 mac_socketpeer_label_free(so->so_peerlabel); 1504 #endif 1505 1506 if (!(so->so_proto->pr_flags & PR_SOCKBUF)) { 1507 sbdestroy(so, SO_SND); 1508 sbdestroy(so, SO_RCV); 1509 } 1510 1511 #ifdef INVARIANTS 1512 bzero(&so->so_rcv, 1513 sizeof(struct socket) - offsetof(struct socket, so_rcv)); 1514 #endif 1515 1516 so->sol_sbrcv_lowat = sbrcv_lowat; 1517 so->sol_sbsnd_lowat = sbsnd_lowat; 1518 so->sol_sbrcv_hiwat = sbrcv_hiwat; 1519 so->sol_sbsnd_hiwat = sbsnd_hiwat; 1520 so->sol_sbrcv_flags = sbrcv_flags; 1521 so->sol_sbsnd_flags = sbsnd_flags; 1522 so->sol_sbrcv_timeo = sbrcv_timeo; 1523 so->sol_sbsnd_timeo = sbsnd_timeo; 1524 1525 so->sol_qlen = so->sol_incqlen = 0; 1526 TAILQ_INIT(&so->sol_incomp); 1527 TAILQ_INIT(&so->sol_comp); 1528 1529 so->sol_accept_filter = NULL; 1530 so->sol_accept_filter_arg = NULL; 1531 so->sol_accept_filter_str = NULL; 1532 1533 so->sol_upcall = NULL; 1534 so->sol_upcallarg = NULL; 1535 1536 so->so_options |= SO_ACCEPTCONN; 1537 1538 listening: 1539 if (backlog < 0 || backlog > V_somaxconn) 1540 backlog = V_somaxconn; 1541 so->sol_qlimit = backlog; 1542 1543 mtx_unlock(&so->so_snd_mtx); 1544 mtx_unlock(&so->so_rcv_mtx); 1545 sx_xunlock(&so->so_snd_sx); 1546 sx_xunlock(&so->so_rcv_sx); 1547 } 1548 1549 /* 1550 * Wakeup listeners/subsystems once we have a complete connection. 1551 * Enters with lock, returns unlocked. 1552 */ 1553 void 1554 solisten_wakeup(struct socket *sol) 1555 { 1556 1557 if (sol->sol_upcall != NULL) 1558 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT); 1559 else { 1560 selwakeuppri(&sol->so_rdsel, PSOCK); 1561 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0); 1562 } 1563 SOLISTEN_UNLOCK(sol); 1564 wakeup_one(&sol->sol_comp); 1565 if ((sol->so_state & SS_ASYNC) && sol->so_sigio != NULL) 1566 pgsigio(&sol->so_sigio, SIGIO, 0); 1567 } 1568 1569 /* 1570 * Return single connection off a listening socket queue. Main consumer of 1571 * the function is kern_accept4(). Some modules, that do their own accept 1572 * management also use the function. The socket reference held by the 1573 * listen queue is handed to the caller. 1574 * 1575 * Listening socket must be locked on entry and is returned unlocked on 1576 * return. 1577 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT. 1578 */ 1579 int 1580 solisten_dequeue(struct socket *head, struct socket **ret, int flags) 1581 { 1582 struct socket *so; 1583 int error; 1584 1585 SOLISTEN_LOCK_ASSERT(head); 1586 1587 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) && 1588 head->so_error == 0) { 1589 error = msleep(&head->sol_comp, SOCK_MTX(head), PSOCK | PCATCH, 1590 "accept", 0); 1591 if (error != 0) { 1592 SOLISTEN_UNLOCK(head); 1593 return (error); 1594 } 1595 } 1596 if (head->so_error) { 1597 error = head->so_error; 1598 head->so_error = 0; 1599 } else if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp)) 1600 error = EWOULDBLOCK; 1601 else 1602 error = 0; 1603 if (error) { 1604 SOLISTEN_UNLOCK(head); 1605 return (error); 1606 } 1607 so = TAILQ_FIRST(&head->sol_comp); 1608 SOCK_LOCK(so); 1609 KASSERT(so->so_qstate == SQ_COMP, 1610 ("%s: so %p not SQ_COMP", __func__, so)); 1611 head->sol_qlen--; 1612 so->so_qstate = SQ_NONE; 1613 so->so_listen = NULL; 1614 TAILQ_REMOVE(&head->sol_comp, so, so_list); 1615 if (flags & ACCEPT4_INHERIT) 1616 so->so_state |= (head->so_state & SS_NBIO); 1617 else 1618 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 1619 SOCK_UNLOCK(so); 1620 sorele_locked(head); 1621 1622 *ret = so; 1623 return (0); 1624 } 1625 1626 static struct so_splice * 1627 so_splice_alloc(off_t max) 1628 { 1629 struct so_splice *sp; 1630 1631 sp = uma_zalloc(splice_zone, M_WAITOK); 1632 sp->src = NULL; 1633 sp->dst = NULL; 1634 sp->max = max > 0 ? max : -1; 1635 do { 1636 sp->wq_index = atomic_fetchadd_32(&splice_index, 1) % 1637 (mp_maxid + 1); 1638 } while (CPU_ABSENT(sp->wq_index)); 1639 sp->state = SPLICE_INIT; 1640 TIMEOUT_TASK_INIT(taskqueue_thread, &sp->timeout, 0, so_splice_timeout, 1641 sp); 1642 return (sp); 1643 } 1644 1645 static void 1646 so_splice_free(struct so_splice *sp) 1647 { 1648 KASSERT(sp->state == SPLICE_CLOSED, 1649 ("so_splice_free: sp %p not closed", sp)); 1650 uma_zfree(splice_zone, sp); 1651 } 1652 1653 static void 1654 so_splice_timeout(void *arg, int pending __unused) 1655 { 1656 struct so_splice *sp; 1657 1658 sp = arg; 1659 (void)so_unsplice(sp->src, true); 1660 } 1661 1662 /* 1663 * Splice the output from so to the input of so2. 1664 */ 1665 static int 1666 so_splice(struct socket *so, struct socket *so2, struct splice *splice) 1667 { 1668 struct so_splice *sp; 1669 int error; 1670 1671 if (splice->sp_max < 0) 1672 return (EINVAL); 1673 /* Handle only TCP for now; TODO: other streaming protos */ 1674 if (so->so_proto->pr_protocol != IPPROTO_TCP || 1675 so2->so_proto->pr_protocol != IPPROTO_TCP) 1676 return (EPROTONOSUPPORT); 1677 if (so->so_vnet != so2->so_vnet) 1678 return (EINVAL); 1679 1680 /* so_splice_xfer() assumes that we're using these implementations. */ 1681 KASSERT(so->so_proto->pr_sosend == sosend_generic, 1682 ("so_splice: sosend not sosend_generic")); 1683 KASSERT(so2->so_proto->pr_soreceive == soreceive_generic || 1684 so2->so_proto->pr_soreceive == soreceive_stream, 1685 ("so_splice: soreceive not soreceive_generic/stream")); 1686 1687 sp = so_splice_alloc(splice->sp_max); 1688 so->so_splice_sent = 0; 1689 sp->src = so; 1690 sp->dst = so2; 1691 1692 error = 0; 1693 SOCK_LOCK(so); 1694 if (SOLISTENING(so)) 1695 error = EINVAL; 1696 else if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0) 1697 error = ENOTCONN; 1698 else if (so->so_splice != NULL) 1699 error = EBUSY; 1700 if (error != 0) { 1701 SOCK_UNLOCK(so); 1702 uma_zfree(splice_zone, sp); 1703 return (error); 1704 } 1705 SOCK_RECVBUF_LOCK(so); 1706 if (so->so_rcv.sb_tls_info != NULL) { 1707 SOCK_RECVBUF_UNLOCK(so); 1708 SOCK_UNLOCK(so); 1709 uma_zfree(splice_zone, sp); 1710 return (EINVAL); 1711 } 1712 so->so_rcv.sb_flags |= SB_SPLICED; 1713 so->so_splice = sp; 1714 soref(so); 1715 SOCK_RECVBUF_UNLOCK(so); 1716 SOCK_UNLOCK(so); 1717 1718 error = 0; 1719 SOCK_LOCK(so2); 1720 if (SOLISTENING(so2)) 1721 error = EINVAL; 1722 else if ((so2->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0) 1723 error = ENOTCONN; 1724 else if (so2->so_splice_back != NULL) 1725 error = EBUSY; 1726 if (error != 0) { 1727 SOCK_UNLOCK(so2); 1728 so_unsplice(so, false); 1729 return (error); 1730 } 1731 SOCK_SENDBUF_LOCK(so2); 1732 if (so->so_snd.sb_tls_info != NULL) { 1733 SOCK_SENDBUF_UNLOCK(so2); 1734 SOCK_UNLOCK(so2); 1735 so_unsplice(so, false); 1736 return (EINVAL); 1737 } 1738 so2->so_snd.sb_flags |= SB_SPLICED; 1739 so2->so_splice_back = sp; 1740 soref(so2); 1741 mtx_lock(&sp->mtx); 1742 SOCK_SENDBUF_UNLOCK(so2); 1743 SOCK_UNLOCK(so2); 1744 1745 if (splice->sp_idle.tv_sec != 0 || splice->sp_idle.tv_usec != 0) { 1746 taskqueue_enqueue_timeout_sbt(taskqueue_thread, &sp->timeout, 1747 tvtosbt(splice->sp_idle), 0, C_PREL(4)); 1748 } 1749 1750 /* 1751 * Transfer any data already present in the socket buffer. 1752 */ 1753 KASSERT(sp->state == SPLICE_INIT, 1754 ("so_splice: splice %p state %d", sp, sp->state)); 1755 sp->state = SPLICE_QUEUED; 1756 so_splice_xfer(sp); 1757 return (0); 1758 } 1759 1760 static int 1761 so_unsplice(struct socket *so, bool timeout) 1762 { 1763 struct socket *so2; 1764 struct so_splice *sp; 1765 bool drain, so2rele; 1766 1767 /* 1768 * First unset SB_SPLICED and hide the splice structure so that 1769 * wakeup routines will stop enqueuing work. This also ensures that 1770 * a only a single thread will proceed with the unsplice. 1771 */ 1772 SOCK_LOCK(so); 1773 if (SOLISTENING(so)) { 1774 SOCK_UNLOCK(so); 1775 return (EINVAL); 1776 } 1777 SOCK_RECVBUF_LOCK(so); 1778 if ((so->so_rcv.sb_flags & SB_SPLICED) == 0) { 1779 SOCK_RECVBUF_UNLOCK(so); 1780 SOCK_UNLOCK(so); 1781 return (ENOTCONN); 1782 } 1783 sp = so->so_splice; 1784 mtx_lock(&sp->mtx); 1785 if (sp->state == SPLICE_INIT) { 1786 /* 1787 * A splice is in the middle of being set up. 1788 */ 1789 mtx_unlock(&sp->mtx); 1790 SOCK_RECVBUF_UNLOCK(so); 1791 SOCK_UNLOCK(so); 1792 return (ENOTCONN); 1793 } 1794 mtx_unlock(&sp->mtx); 1795 so->so_rcv.sb_flags &= ~SB_SPLICED; 1796 so->so_splice = NULL; 1797 SOCK_RECVBUF_UNLOCK(so); 1798 SOCK_UNLOCK(so); 1799 1800 so2 = sp->dst; 1801 SOCK_LOCK(so2); 1802 KASSERT(!SOLISTENING(so2), ("%s: so2 is listening", __func__)); 1803 SOCK_SENDBUF_LOCK(so2); 1804 KASSERT(sp->state == SPLICE_INIT || 1805 (so2->so_snd.sb_flags & SB_SPLICED) != 0, 1806 ("%s: so2 is not spliced", __func__)); 1807 KASSERT(sp->state == SPLICE_INIT || 1808 so2->so_splice_back == sp, 1809 ("%s: so_splice_back != sp", __func__)); 1810 so2->so_snd.sb_flags &= ~SB_SPLICED; 1811 so2rele = so2->so_splice_back != NULL; 1812 so2->so_splice_back = NULL; 1813 SOCK_SENDBUF_UNLOCK(so2); 1814 SOCK_UNLOCK(so2); 1815 1816 /* 1817 * No new work is being enqueued. The worker thread might be 1818 * splicing data right now, in which case we want to wait for it to 1819 * finish before proceeding. 1820 */ 1821 mtx_lock(&sp->mtx); 1822 switch (sp->state) { 1823 case SPLICE_QUEUED: 1824 case SPLICE_RUNNING: 1825 sp->state = SPLICE_CLOSING; 1826 while (sp->state == SPLICE_CLOSING) 1827 msleep(sp, &sp->mtx, PSOCK, "unsplice", 0); 1828 break; 1829 case SPLICE_INIT: 1830 case SPLICE_IDLE: 1831 case SPLICE_EXCEPTION: 1832 sp->state = SPLICE_CLOSED; 1833 break; 1834 default: 1835 __assert_unreachable(); 1836 } 1837 if (!timeout) { 1838 drain = taskqueue_cancel_timeout(taskqueue_thread, &sp->timeout, 1839 NULL) != 0; 1840 } else { 1841 drain = false; 1842 } 1843 mtx_unlock(&sp->mtx); 1844 if (drain) 1845 taskqueue_drain_timeout(taskqueue_thread, &sp->timeout); 1846 1847 /* 1848 * Now we hold the sole reference to the splice structure. 1849 * Clean up: signal userspace and release socket references. 1850 */ 1851 sorwakeup(so); 1852 CURVNET_SET(so->so_vnet); 1853 sorele(so); 1854 sowwakeup(so2); 1855 if (so2rele) 1856 sorele(so2); 1857 CURVNET_RESTORE(); 1858 so_splice_free(sp); 1859 return (0); 1860 } 1861 1862 /* 1863 * Free socket upon release of the very last reference. 1864 */ 1865 static void 1866 sofree(struct socket *so) 1867 { 1868 struct protosw *pr = so->so_proto; 1869 1870 SOCK_LOCK_ASSERT(so); 1871 KASSERT(refcount_load(&so->so_count) == 0, 1872 ("%s: so %p has references", __func__, so)); 1873 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE, 1874 ("%s: so %p is on listen queue", __func__, so)); 1875 KASSERT(SOLISTENING(so) || (so->so_rcv.sb_flags & SB_SPLICED) == 0, 1876 ("%s: so %p rcvbuf is spliced", __func__, so)); 1877 KASSERT(SOLISTENING(so) || (so->so_snd.sb_flags & SB_SPLICED) == 0, 1878 ("%s: so %p sndbuf is spliced", __func__, so)); 1879 KASSERT(so->so_splice == NULL && so->so_splice_back == NULL, 1880 ("%s: so %p has spliced data", __func__, so)); 1881 1882 SOCK_UNLOCK(so); 1883 1884 if (so->so_dtor != NULL) 1885 so->so_dtor(so); 1886 1887 VNET_SO_ASSERT(so); 1888 if (pr->pr_detach != NULL) 1889 pr->pr_detach(so); 1890 1891 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) { 1892 /* 1893 * From this point on, we assume that no other references to 1894 * this socket exist anywhere else in the stack. Therefore, 1895 * no locks need to be acquired or held. 1896 */ 1897 #ifdef INVARIANTS 1898 SOCK_SENDBUF_LOCK(so); 1899 SOCK_RECVBUF_LOCK(so); 1900 #endif 1901 sbdestroy(so, SO_SND); 1902 sbdestroy(so, SO_RCV); 1903 #ifdef INVARIANTS 1904 SOCK_SENDBUF_UNLOCK(so); 1905 SOCK_RECVBUF_UNLOCK(so); 1906 #endif 1907 } 1908 seldrain(&so->so_rdsel); 1909 seldrain(&so->so_wrsel); 1910 knlist_destroy(&so->so_rdsel.si_note); 1911 knlist_destroy(&so->so_wrsel.si_note); 1912 sodealloc(so); 1913 } 1914 1915 /* 1916 * Release a reference on a socket while holding the socket lock. 1917 * Unlocks the socket lock before returning. 1918 */ 1919 void 1920 sorele_locked(struct socket *so) 1921 { 1922 SOCK_LOCK_ASSERT(so); 1923 if (refcount_release(&so->so_count)) 1924 sofree(so); 1925 else 1926 SOCK_UNLOCK(so); 1927 } 1928 1929 /* 1930 * Close a socket on last file table reference removal. Initiate disconnect 1931 * if connected. Free socket when disconnect complete. 1932 * 1933 * This function will sorele() the socket. Note that soclose() may be called 1934 * prior to the ref count reaching zero. The actual socket structure will 1935 * not be freed until the ref count reaches zero. 1936 */ 1937 int 1938 soclose(struct socket *so) 1939 { 1940 struct accept_queue lqueue; 1941 int error = 0; 1942 bool listening, last __diagused; 1943 1944 CURVNET_SET(so->so_vnet); 1945 funsetown(&so->so_sigio); 1946 if (so->so_state & SS_ISCONNECTED) { 1947 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 1948 error = sodisconnect(so); 1949 if (error) { 1950 if (error == ENOTCONN) 1951 error = 0; 1952 goto drop; 1953 } 1954 } 1955 1956 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) { 1957 if ((so->so_state & SS_ISDISCONNECTING) && 1958 (so->so_state & SS_NBIO)) 1959 goto drop; 1960 while (so->so_state & SS_ISCONNECTED) { 1961 error = tsleep(&so->so_timeo, 1962 PSOCK | PCATCH, "soclos", 1963 so->so_linger * hz); 1964 if (error) 1965 break; 1966 } 1967 } 1968 } 1969 1970 drop: 1971 if (so->so_proto->pr_close != NULL) 1972 so->so_proto->pr_close(so); 1973 1974 SOCK_LOCK(so); 1975 if ((listening = SOLISTENING(so))) { 1976 struct socket *sp; 1977 1978 TAILQ_INIT(&lqueue); 1979 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list); 1980 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list); 1981 1982 so->sol_qlen = so->sol_incqlen = 0; 1983 1984 TAILQ_FOREACH(sp, &lqueue, so_list) { 1985 SOCK_LOCK(sp); 1986 sp->so_qstate = SQ_NONE; 1987 sp->so_listen = NULL; 1988 SOCK_UNLOCK(sp); 1989 last = refcount_release(&so->so_count); 1990 KASSERT(!last, ("%s: released last reference for %p", 1991 __func__, so)); 1992 } 1993 } 1994 sorele_locked(so); 1995 if (listening) { 1996 struct socket *sp, *tsp; 1997 1998 TAILQ_FOREACH_SAFE(sp, &lqueue, so_list, tsp) 1999 soabort(sp); 2000 } 2001 CURVNET_RESTORE(); 2002 return (error); 2003 } 2004 2005 /* 2006 * soabort() is used to abruptly tear down a connection, such as when a 2007 * resource limit is reached (listen queue depth exceeded), or if a listen 2008 * socket is closed while there are sockets waiting to be accepted. 2009 * 2010 * This interface is tricky, because it is called on an unreferenced socket, 2011 * and must be called only by a thread that has actually removed the socket 2012 * from the listen queue it was on. Likely this thread holds the last 2013 * reference on the socket and soabort() will proceed with sofree(). But 2014 * it might be not the last, as the sockets on the listen queues are seen 2015 * from the protocol side. 2016 * 2017 * This interface will call into the protocol code, so must not be called 2018 * with any socket locks held. Protocols do call it while holding their own 2019 * recursible protocol mutexes, but this is something that should be subject 2020 * to review in the future. 2021 * 2022 * Usually socket should have a single reference left, but this is not a 2023 * requirement. In the past, when we have had named references for file 2024 * descriptor and protocol, we asserted that none of them are being held. 2025 */ 2026 void 2027 soabort(struct socket *so) 2028 { 2029 2030 VNET_SO_ASSERT(so); 2031 2032 if (so->so_proto->pr_abort != NULL) 2033 so->so_proto->pr_abort(so); 2034 SOCK_LOCK(so); 2035 sorele_locked(so); 2036 } 2037 2038 int 2039 soaccept(struct socket *so, struct sockaddr *sa) 2040 { 2041 #ifdef INVARIANTS 2042 u_char len = sa->sa_len; 2043 #endif 2044 int error; 2045 2046 CURVNET_SET(so->so_vnet); 2047 error = so->so_proto->pr_accept(so, sa); 2048 KASSERT(sa->sa_len <= len, 2049 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2050 CURVNET_RESTORE(); 2051 return (error); 2052 } 2053 2054 int 2055 sopeeraddr(struct socket *so, struct sockaddr *sa) 2056 { 2057 #ifdef INVARIANTS 2058 u_char len = sa->sa_len; 2059 #endif 2060 int error; 2061 2062 CURVNET_ASSERT_SET(); 2063 2064 error = so->so_proto->pr_peeraddr(so, sa); 2065 KASSERT(sa->sa_len <= len, 2066 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2067 2068 return (error); 2069 } 2070 2071 int 2072 sosockaddr(struct socket *so, struct sockaddr *sa) 2073 { 2074 #ifdef INVARIANTS 2075 u_char len = sa->sa_len; 2076 #endif 2077 int error; 2078 2079 CURVNET_SET(so->so_vnet); 2080 error = so->so_proto->pr_sockaddr(so, sa); 2081 KASSERT(sa->sa_len <= len, 2082 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto)); 2083 CURVNET_RESTORE(); 2084 2085 return (error); 2086 } 2087 2088 int 2089 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) 2090 { 2091 2092 return (soconnectat(AT_FDCWD, so, nam, td)); 2093 } 2094 2095 int 2096 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 2097 { 2098 int error; 2099 2100 CURVNET_SET(so->so_vnet); 2101 2102 /* 2103 * If protocol is connection-based, can only connect once. 2104 * Otherwise, if connected, try to disconnect first. This allows 2105 * user to disconnect by connecting to, e.g., a null address. 2106 * 2107 * Note, this check is racy and may need to be re-evaluated at the 2108 * protocol layer. 2109 */ 2110 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 2111 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 2112 (error = sodisconnect(so)))) { 2113 error = EISCONN; 2114 } else { 2115 /* 2116 * Prevent accumulated error from previous connection from 2117 * biting us. 2118 */ 2119 so->so_error = 0; 2120 if (fd == AT_FDCWD) { 2121 error = so->so_proto->pr_connect(so, nam, td); 2122 } else { 2123 error = so->so_proto->pr_connectat(fd, so, nam, td); 2124 } 2125 } 2126 CURVNET_RESTORE(); 2127 2128 return (error); 2129 } 2130 2131 int 2132 soconnect2(struct socket *so1, struct socket *so2) 2133 { 2134 int error; 2135 2136 CURVNET_SET(so1->so_vnet); 2137 error = so1->so_proto->pr_connect2(so1, so2); 2138 CURVNET_RESTORE(); 2139 return (error); 2140 } 2141 2142 int 2143 sodisconnect(struct socket *so) 2144 { 2145 int error; 2146 2147 if ((so->so_state & SS_ISCONNECTED) == 0) 2148 return (ENOTCONN); 2149 if (so->so_state & SS_ISDISCONNECTING) 2150 return (EALREADY); 2151 VNET_SO_ASSERT(so); 2152 error = so->so_proto->pr_disconnect(so); 2153 return (error); 2154 } 2155 2156 int 2157 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 2158 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2159 { 2160 long space; 2161 ssize_t resid; 2162 int clen = 0, error, dontroute; 2163 2164 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM")); 2165 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 2166 ("sosend_dgram: !PR_ATOMIC")); 2167 2168 if (uio != NULL) 2169 resid = uio->uio_resid; 2170 else 2171 resid = top->m_pkthdr.len; 2172 /* 2173 * In theory resid should be unsigned. However, space must be 2174 * signed, as it might be less than 0 if we over-committed, and we 2175 * must use a signed comparison of space and resid. On the other 2176 * hand, a negative resid causes us to loop sending 0-length 2177 * segments to the protocol. 2178 */ 2179 if (resid < 0) { 2180 error = EINVAL; 2181 goto out; 2182 } 2183 2184 dontroute = 2185 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 2186 if (td != NULL) 2187 td->td_ru.ru_msgsnd++; 2188 if (control != NULL) 2189 clen = control->m_len; 2190 2191 SOCKBUF_LOCK(&so->so_snd); 2192 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2193 SOCKBUF_UNLOCK(&so->so_snd); 2194 error = EPIPE; 2195 goto out; 2196 } 2197 if (so->so_error) { 2198 error = so->so_error; 2199 so->so_error = 0; 2200 SOCKBUF_UNLOCK(&so->so_snd); 2201 goto out; 2202 } 2203 if ((so->so_state & SS_ISCONNECTED) == 0) { 2204 /* 2205 * `sendto' and `sendmsg' is allowed on a connection-based 2206 * socket if it supports implied connect. Return ENOTCONN if 2207 * not connected and no address is supplied. 2208 */ 2209 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 2210 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 2211 if (!(resid == 0 && clen != 0)) { 2212 SOCKBUF_UNLOCK(&so->so_snd); 2213 error = ENOTCONN; 2214 goto out; 2215 } 2216 } else if (addr == NULL) { 2217 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2218 error = ENOTCONN; 2219 else 2220 error = EDESTADDRREQ; 2221 SOCKBUF_UNLOCK(&so->so_snd); 2222 goto out; 2223 } 2224 } 2225 2226 /* 2227 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 2228 * problem and need fixing. 2229 */ 2230 space = sbspace(&so->so_snd); 2231 if (flags & MSG_OOB) 2232 space += 1024; 2233 space -= clen; 2234 SOCKBUF_UNLOCK(&so->so_snd); 2235 if (resid > space) { 2236 error = EMSGSIZE; 2237 goto out; 2238 } 2239 if (uio == NULL) { 2240 resid = 0; 2241 if (flags & MSG_EOR) 2242 top->m_flags |= M_EOR; 2243 } else { 2244 /* 2245 * Copy the data from userland into a mbuf chain. 2246 * If no data is to be copied in, a single empty mbuf 2247 * is returned. 2248 */ 2249 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 2250 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 2251 if (top == NULL) { 2252 error = EFAULT; /* only possible error */ 2253 goto out; 2254 } 2255 space -= resid - uio->uio_resid; 2256 resid = uio->uio_resid; 2257 } 2258 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 2259 /* 2260 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 2261 * than with. 2262 */ 2263 if (dontroute) { 2264 SOCK_LOCK(so); 2265 so->so_options |= SO_DONTROUTE; 2266 SOCK_UNLOCK(so); 2267 } 2268 /* 2269 * XXX all the SBS_CANTSENDMORE checks previously done could be out 2270 * of date. We could have received a reset packet in an interrupt or 2271 * maybe we slept while doing page faults in uiomove() etc. We could 2272 * probably recheck again inside the locking protection here, but 2273 * there are probably other places that this also happens. We must 2274 * rethink this. 2275 */ 2276 VNET_SO_ASSERT(so); 2277 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB : 2278 /* 2279 * If the user set MSG_EOF, the protocol understands this flag and 2280 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 2281 */ 2282 ((flags & MSG_EOF) && 2283 (so->so_proto->pr_flags & PR_IMPLOPCL) && 2284 (resid <= 0)) ? 2285 PRUS_EOF : 2286 /* If there is more to send set PRUS_MORETOCOME */ 2287 (flags & MSG_MORETOCOME) || 2288 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 2289 top, addr, control, td); 2290 if (dontroute) { 2291 SOCK_LOCK(so); 2292 so->so_options &= ~SO_DONTROUTE; 2293 SOCK_UNLOCK(so); 2294 } 2295 clen = 0; 2296 control = NULL; 2297 top = NULL; 2298 out: 2299 if (top != NULL) 2300 m_freem(top); 2301 if (control != NULL) 2302 m_freem(control); 2303 return (error); 2304 } 2305 2306 /* 2307 * Send on a socket. If send must go all at once and message is larger than 2308 * send buffering, then hard error. Lock against other senders. If must go 2309 * all at once and not enough room now, then inform user that this would 2310 * block and do nothing. Otherwise, if nonblocking, send as much as 2311 * possible. The data to be sent is described by "uio" if nonzero, otherwise 2312 * by the mbuf chain "top" (which must be null if uio is not). Data provided 2313 * in mbuf chain must be small enough to send all at once. 2314 * 2315 * Returns nonzero on error, timeout or signal; callers must check for short 2316 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 2317 * on return. 2318 */ 2319 static int 2320 sosend_generic_locked(struct socket *so, struct sockaddr *addr, struct uio *uio, 2321 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2322 { 2323 long space; 2324 ssize_t resid; 2325 int clen = 0, error, dontroute; 2326 int atomic = sosendallatonce(so) || top; 2327 int pr_send_flag; 2328 #ifdef KERN_TLS 2329 struct ktls_session *tls; 2330 int tls_enq_cnt, tls_send_flag; 2331 uint8_t tls_rtype; 2332 2333 tls = NULL; 2334 tls_rtype = TLS_RLTYPE_APP; 2335 #endif 2336 2337 SOCK_IO_SEND_ASSERT_LOCKED(so); 2338 2339 if (uio != NULL) 2340 resid = uio->uio_resid; 2341 else if ((top->m_flags & M_PKTHDR) != 0) 2342 resid = top->m_pkthdr.len; 2343 else 2344 resid = m_length(top, NULL); 2345 /* 2346 * In theory resid should be unsigned. However, space must be 2347 * signed, as it might be less than 0 if we over-committed, and we 2348 * must use a signed comparison of space and resid. On the other 2349 * hand, a negative resid causes us to loop sending 0-length 2350 * segments to the protocol. 2351 * 2352 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 2353 * type sockets since that's an error. 2354 */ 2355 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 2356 error = EINVAL; 2357 goto out; 2358 } 2359 2360 dontroute = 2361 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 2362 (so->so_proto->pr_flags & PR_ATOMIC); 2363 if (td != NULL) 2364 td->td_ru.ru_msgsnd++; 2365 if (control != NULL) 2366 clen = control->m_len; 2367 2368 #ifdef KERN_TLS 2369 tls_send_flag = 0; 2370 tls = ktls_hold(so->so_snd.sb_tls_info); 2371 if (tls != NULL) { 2372 if (tls->mode == TCP_TLS_MODE_SW) 2373 tls_send_flag = PRUS_NOTREADY; 2374 2375 if (control != NULL) { 2376 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 2377 2378 if (clen >= sizeof(*cm) && 2379 cm->cmsg_type == TLS_SET_RECORD_TYPE) { 2380 tls_rtype = *((uint8_t *)CMSG_DATA(cm)); 2381 clen = 0; 2382 m_freem(control); 2383 control = NULL; 2384 atomic = 1; 2385 } 2386 } 2387 2388 if (resid == 0 && !ktls_permit_empty_frames(tls)) { 2389 error = EINVAL; 2390 goto out; 2391 } 2392 } 2393 #endif 2394 2395 restart: 2396 do { 2397 SOCKBUF_LOCK(&so->so_snd); 2398 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2399 SOCKBUF_UNLOCK(&so->so_snd); 2400 error = EPIPE; 2401 goto out; 2402 } 2403 if (so->so_error) { 2404 error = so->so_error; 2405 so->so_error = 0; 2406 SOCKBUF_UNLOCK(&so->so_snd); 2407 goto out; 2408 } 2409 if ((so->so_state & SS_ISCONNECTED) == 0) { 2410 /* 2411 * `sendto' and `sendmsg' is allowed on a connection- 2412 * based socket if it supports implied connect. 2413 * Return ENOTCONN if not connected and no address is 2414 * supplied. 2415 */ 2416 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 2417 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 2418 if (!(resid == 0 && clen != 0)) { 2419 SOCKBUF_UNLOCK(&so->so_snd); 2420 error = ENOTCONN; 2421 goto out; 2422 } 2423 } else if (addr == NULL) { 2424 SOCKBUF_UNLOCK(&so->so_snd); 2425 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2426 error = ENOTCONN; 2427 else 2428 error = EDESTADDRREQ; 2429 goto out; 2430 } 2431 } 2432 space = sbspace(&so->so_snd); 2433 if (flags & MSG_OOB) 2434 space += 1024; 2435 if ((atomic && resid > so->so_snd.sb_hiwat) || 2436 clen > so->so_snd.sb_hiwat) { 2437 SOCKBUF_UNLOCK(&so->so_snd); 2438 error = EMSGSIZE; 2439 goto out; 2440 } 2441 if (space < resid + clen && 2442 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 2443 if ((so->so_state & SS_NBIO) || 2444 (flags & (MSG_NBIO | MSG_DONTWAIT)) != 0) { 2445 SOCKBUF_UNLOCK(&so->so_snd); 2446 error = EWOULDBLOCK; 2447 goto out; 2448 } 2449 error = sbwait(so, SO_SND); 2450 SOCKBUF_UNLOCK(&so->so_snd); 2451 if (error) 2452 goto out; 2453 goto restart; 2454 } 2455 SOCKBUF_UNLOCK(&so->so_snd); 2456 space -= clen; 2457 do { 2458 if (uio == NULL) { 2459 resid = 0; 2460 if (flags & MSG_EOR) 2461 top->m_flags |= M_EOR; 2462 #ifdef KERN_TLS 2463 if (tls != NULL) { 2464 ktls_frame(top, tls, &tls_enq_cnt, 2465 tls_rtype); 2466 tls_rtype = TLS_RLTYPE_APP; 2467 } 2468 #endif 2469 } else { 2470 /* 2471 * Copy the data from userland into a mbuf 2472 * chain. If resid is 0, which can happen 2473 * only if we have control to send, then 2474 * a single empty mbuf is returned. This 2475 * is a workaround to prevent protocol send 2476 * methods to panic. 2477 */ 2478 #ifdef KERN_TLS 2479 if (tls != NULL) { 2480 top = m_uiotombuf(uio, M_WAITOK, space, 2481 tls->params.max_frame_len, 2482 M_EXTPG | 2483 ((flags & MSG_EOR) ? M_EOR : 0)); 2484 if (top != NULL) { 2485 ktls_frame(top, tls, 2486 &tls_enq_cnt, tls_rtype); 2487 } 2488 tls_rtype = TLS_RLTYPE_APP; 2489 } else 2490 #endif 2491 top = m_uiotombuf(uio, M_WAITOK, space, 2492 (atomic ? max_hdr : 0), 2493 (atomic ? M_PKTHDR : 0) | 2494 ((flags & MSG_EOR) ? M_EOR : 0)); 2495 if (top == NULL) { 2496 error = EFAULT; /* only possible error */ 2497 goto out; 2498 } 2499 space -= resid - uio->uio_resid; 2500 resid = uio->uio_resid; 2501 } 2502 if (dontroute) { 2503 SOCK_LOCK(so); 2504 so->so_options |= SO_DONTROUTE; 2505 SOCK_UNLOCK(so); 2506 } 2507 /* 2508 * XXX all the SBS_CANTSENDMORE checks previously 2509 * done could be out of date. We could have received 2510 * a reset packet in an interrupt or maybe we slept 2511 * while doing page faults in uiomove() etc. We 2512 * could probably recheck again inside the locking 2513 * protection here, but there are probably other 2514 * places that this also happens. We must rethink 2515 * this. 2516 */ 2517 VNET_SO_ASSERT(so); 2518 2519 pr_send_flag = (flags & MSG_OOB) ? PRUS_OOB : 2520 /* 2521 * If the user set MSG_EOF, the protocol understands 2522 * this flag and nothing left to send then use 2523 * PRU_SEND_EOF instead of PRU_SEND. 2524 */ 2525 ((flags & MSG_EOF) && 2526 (so->so_proto->pr_flags & PR_IMPLOPCL) && 2527 (resid <= 0)) ? 2528 PRUS_EOF : 2529 /* If there is more to send set PRUS_MORETOCOME. */ 2530 (flags & MSG_MORETOCOME) || 2531 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0; 2532 2533 #ifdef KERN_TLS 2534 pr_send_flag |= tls_send_flag; 2535 #endif 2536 2537 error = so->so_proto->pr_send(so, pr_send_flag, top, 2538 addr, control, td); 2539 2540 if (dontroute) { 2541 SOCK_LOCK(so); 2542 so->so_options &= ~SO_DONTROUTE; 2543 SOCK_UNLOCK(so); 2544 } 2545 2546 #ifdef KERN_TLS 2547 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { 2548 if (error != 0) { 2549 m_freem(top); 2550 top = NULL; 2551 } else { 2552 soref(so); 2553 ktls_enqueue(top, so, tls_enq_cnt); 2554 } 2555 } 2556 #endif 2557 clen = 0; 2558 control = NULL; 2559 top = NULL; 2560 if (error) 2561 goto out; 2562 } while (resid && space > 0); 2563 } while (resid); 2564 2565 out: 2566 #ifdef KERN_TLS 2567 if (tls != NULL) 2568 ktls_free(tls); 2569 #endif 2570 if (top != NULL) 2571 m_freem(top); 2572 if (control != NULL) 2573 m_freem(control); 2574 return (error); 2575 } 2576 2577 int 2578 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, 2579 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2580 { 2581 int error; 2582 2583 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2584 if (error) 2585 return (error); 2586 error = sosend_generic_locked(so, addr, uio, top, control, flags, td); 2587 SOCK_IO_SEND_UNLOCK(so); 2588 return (error); 2589 } 2590 2591 /* 2592 * Send to a socket from a kernel thread. 2593 * 2594 * XXXGL: in almost all cases uio is NULL and the mbuf is supplied. 2595 * Exception is nfs/bootp_subr.c. It is arguable that the VNET context needs 2596 * to be set at all. This function should just boil down to a static inline 2597 * calling the protocol method. 2598 */ 2599 int 2600 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, 2601 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 2602 { 2603 int error; 2604 2605 CURVNET_SET(so->so_vnet); 2606 error = so->so_proto->pr_sosend(so, addr, uio, 2607 top, control, flags, td); 2608 CURVNET_RESTORE(); 2609 return (error); 2610 } 2611 2612 /* 2613 * send(2), write(2) or aio_write(2) on a socket. 2614 */ 2615 int 2616 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio, 2617 struct mbuf *control, int flags, struct proc *userproc) 2618 { 2619 struct thread *td; 2620 ssize_t len; 2621 int error; 2622 2623 td = uio->uio_td; 2624 len = uio->uio_resid; 2625 CURVNET_SET(so->so_vnet); 2626 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags, 2627 td); 2628 CURVNET_RESTORE(); 2629 if (error != 0) { 2630 /* 2631 * Clear transient errors for stream protocols if they made 2632 * some progress. Make exclusion for aio(4) that would 2633 * schedule a new write in case of EWOULDBLOCK and clear 2634 * error itself. See soaio_process_job(). 2635 */ 2636 if (uio->uio_resid != len && 2637 (so->so_proto->pr_flags & PR_ATOMIC) == 0 && 2638 userproc == NULL && 2639 (error == ERESTART || error == EINTR || 2640 error == EWOULDBLOCK)) 2641 error = 0; 2642 /* Generation of SIGPIPE can be controlled per socket. */ 2643 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 && 2644 (flags & MSG_NOSIGNAL) == 0) { 2645 if (userproc != NULL) { 2646 /* aio(4) job */ 2647 PROC_LOCK(userproc); 2648 kern_psignal(userproc, SIGPIPE); 2649 PROC_UNLOCK(userproc); 2650 } else { 2651 PROC_LOCK(td->td_proc); 2652 tdsignal(td, SIGPIPE); 2653 PROC_UNLOCK(td->td_proc); 2654 } 2655 } 2656 } 2657 return (error); 2658 } 2659 2660 /* 2661 * The part of soreceive() that implements reading non-inline out-of-band 2662 * data from a socket. For more complete comments, see soreceive(), from 2663 * which this code originated. 2664 * 2665 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 2666 * unable to return an mbuf chain to the caller. 2667 */ 2668 static int 2669 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) 2670 { 2671 struct protosw *pr = so->so_proto; 2672 struct mbuf *m; 2673 int error; 2674 2675 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 2676 VNET_SO_ASSERT(so); 2677 2678 m = m_get(M_WAITOK, MT_DATA); 2679 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK); 2680 if (error) 2681 goto bad; 2682 do { 2683 error = uiomove(mtod(m, void *), 2684 (int) min(uio->uio_resid, m->m_len), uio); 2685 m = m_free(m); 2686 } while (uio->uio_resid && error == 0 && m); 2687 bad: 2688 if (m != NULL) 2689 m_freem(m); 2690 return (error); 2691 } 2692 2693 /* 2694 * Following replacement or removal of the first mbuf on the first mbuf chain 2695 * of a socket buffer, push necessary state changes back into the socket 2696 * buffer so that other consumers see the values consistently. 'nextrecord' 2697 * is the callers locally stored value of the original value of 2698 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 2699 * NOTE: 'nextrecord' may be NULL. 2700 */ 2701 static __inline void 2702 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 2703 { 2704 2705 SOCKBUF_LOCK_ASSERT(sb); 2706 /* 2707 * First, update for the new value of nextrecord. If necessary, make 2708 * it the first record. 2709 */ 2710 if (sb->sb_mb != NULL) 2711 sb->sb_mb->m_nextpkt = nextrecord; 2712 else 2713 sb->sb_mb = nextrecord; 2714 2715 /* 2716 * Now update any dependent socket buffer fields to reflect the new 2717 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 2718 * addition of a second clause that takes care of the case where 2719 * sb_mb has been updated, but remains the last record. 2720 */ 2721 if (sb->sb_mb == NULL) { 2722 sb->sb_mbtail = NULL; 2723 sb->sb_lastrecord = NULL; 2724 } else if (sb->sb_mb->m_nextpkt == NULL) 2725 sb->sb_lastrecord = sb->sb_mb; 2726 } 2727 2728 /* 2729 * Implement receive operations on a socket. We depend on the way that 2730 * records are added to the sockbuf by sbappend. In particular, each record 2731 * (mbufs linked through m_next) must begin with an address if the protocol 2732 * so specifies, followed by an optional mbuf or mbufs containing ancillary 2733 * data, and then zero or more mbufs of data. In order to allow parallelism 2734 * between network receive and copying to user space, as well as avoid 2735 * sleeping with a mutex held, we release the socket buffer mutex during the 2736 * user space copy. Although the sockbuf is locked, new data may still be 2737 * appended, and thus we must maintain consistency of the sockbuf during that 2738 * time. 2739 * 2740 * The caller may receive the data as a single mbuf chain by supplying an 2741 * mbuf **mp0 for use in returning the chain. The uio is then used only for 2742 * the count in uio_resid. 2743 */ 2744 static int 2745 soreceive_generic_locked(struct socket *so, struct sockaddr **psa, 2746 struct uio *uio, struct mbuf **mp, struct mbuf **controlp, int *flagsp) 2747 { 2748 struct mbuf *m; 2749 int flags, error, offset; 2750 ssize_t len; 2751 struct protosw *pr = so->so_proto; 2752 struct mbuf *nextrecord; 2753 int moff, type = 0; 2754 ssize_t orig_resid = uio->uio_resid; 2755 bool report_real_len = false; 2756 2757 SOCK_IO_RECV_ASSERT_LOCKED(so); 2758 2759 error = 0; 2760 if (flagsp != NULL) { 2761 report_real_len = *flagsp & MSG_TRUNC; 2762 *flagsp &= ~MSG_TRUNC; 2763 flags = *flagsp &~ MSG_EOR; 2764 } else 2765 flags = 0; 2766 2767 restart: 2768 SOCKBUF_LOCK(&so->so_rcv); 2769 m = so->so_rcv.sb_mb; 2770 /* 2771 * If we have less data than requested, block awaiting more (subject 2772 * to any timeout) if: 2773 * 1. the current count is less than the low water mark, or 2774 * 2. MSG_DONTWAIT is not set 2775 */ 2776 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 2777 sbavail(&so->so_rcv) < uio->uio_resid) && 2778 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat && 2779 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 2780 KASSERT(m != NULL || !sbavail(&so->so_rcv), 2781 ("receive: m == %p sbavail == %u", 2782 m, sbavail(&so->so_rcv))); 2783 if (so->so_error || so->so_rerror) { 2784 if (m != NULL) 2785 goto dontblock; 2786 if (so->so_error) 2787 error = so->so_error; 2788 else 2789 error = so->so_rerror; 2790 if ((flags & MSG_PEEK) == 0) { 2791 if (so->so_error) 2792 so->so_error = 0; 2793 else 2794 so->so_rerror = 0; 2795 } 2796 SOCKBUF_UNLOCK(&so->so_rcv); 2797 goto release; 2798 } 2799 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2800 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 2801 if (m != NULL) 2802 goto dontblock; 2803 #ifdef KERN_TLS 2804 else if (so->so_rcv.sb_tlsdcc == 0 && 2805 so->so_rcv.sb_tlscc == 0) { 2806 #else 2807 else { 2808 #endif 2809 SOCKBUF_UNLOCK(&so->so_rcv); 2810 goto release; 2811 } 2812 } 2813 for (; m != NULL; m = m->m_next) 2814 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 2815 m = so->so_rcv.sb_mb; 2816 goto dontblock; 2817 } 2818 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED | 2819 SS_ISDISCONNECTING | SS_ISDISCONNECTED)) == 0 && 2820 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) { 2821 SOCKBUF_UNLOCK(&so->so_rcv); 2822 error = ENOTCONN; 2823 goto release; 2824 } 2825 if (uio->uio_resid == 0 && !report_real_len) { 2826 SOCKBUF_UNLOCK(&so->so_rcv); 2827 goto release; 2828 } 2829 if ((so->so_state & SS_NBIO) || 2830 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 2831 SOCKBUF_UNLOCK(&so->so_rcv); 2832 error = EWOULDBLOCK; 2833 goto release; 2834 } 2835 SBLASTRECORDCHK(&so->so_rcv); 2836 SBLASTMBUFCHK(&so->so_rcv); 2837 error = sbwait(so, SO_RCV); 2838 SOCKBUF_UNLOCK(&so->so_rcv); 2839 if (error) 2840 goto release; 2841 goto restart; 2842 } 2843 dontblock: 2844 /* 2845 * From this point onward, we maintain 'nextrecord' as a cache of the 2846 * pointer to the next record in the socket buffer. We must keep the 2847 * various socket buffer pointers and local stack versions of the 2848 * pointers in sync, pushing out modifications before dropping the 2849 * socket buffer mutex, and re-reading them when picking it up. 2850 * 2851 * Otherwise, we will race with the network stack appending new data 2852 * or records onto the socket buffer by using inconsistent/stale 2853 * versions of the field, possibly resulting in socket buffer 2854 * corruption. 2855 * 2856 * By holding the high-level sblock(), we prevent simultaneous 2857 * readers from pulling off the front of the socket buffer. 2858 */ 2859 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2860 if (uio->uio_td) 2861 uio->uio_td->td_ru.ru_msgrcv++; 2862 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 2863 SBLASTRECORDCHK(&so->so_rcv); 2864 SBLASTMBUFCHK(&so->so_rcv); 2865 nextrecord = m->m_nextpkt; 2866 if (pr->pr_flags & PR_ADDR) { 2867 KASSERT(m->m_type == MT_SONAME, 2868 ("m->m_type == %d", m->m_type)); 2869 orig_resid = 0; 2870 if (psa != NULL) 2871 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 2872 M_NOWAIT); 2873 if (flags & MSG_PEEK) { 2874 m = m->m_next; 2875 } else { 2876 sbfree(&so->so_rcv, m); 2877 so->so_rcv.sb_mb = m_free(m); 2878 m = so->so_rcv.sb_mb; 2879 sockbuf_pushsync(&so->so_rcv, nextrecord); 2880 } 2881 } 2882 2883 /* 2884 * Process one or more MT_CONTROL mbufs present before any data mbufs 2885 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 2886 * just copy the data; if !MSG_PEEK, we call into the protocol to 2887 * perform externalization (or freeing if controlp == NULL). 2888 */ 2889 if (m != NULL && m->m_type == MT_CONTROL) { 2890 struct mbuf *cm = NULL, *cmn; 2891 struct mbuf **cme = &cm; 2892 #ifdef KERN_TLS 2893 struct cmsghdr *cmsg; 2894 struct tls_get_record tgr; 2895 2896 /* 2897 * For MSG_TLSAPPDATA, check for an alert record. 2898 * If found, return ENXIO without removing 2899 * it from the receive queue. This allows a subsequent 2900 * call without MSG_TLSAPPDATA to receive it. 2901 * Note that, for TLS, there should only be a single 2902 * control mbuf with the TLS_GET_RECORD message in it. 2903 */ 2904 if (flags & MSG_TLSAPPDATA) { 2905 cmsg = mtod(m, struct cmsghdr *); 2906 if (cmsg->cmsg_type == TLS_GET_RECORD && 2907 cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) { 2908 memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr)); 2909 if (__predict_false(tgr.tls_type == 2910 TLS_RLTYPE_ALERT)) { 2911 SOCKBUF_UNLOCK(&so->so_rcv); 2912 error = ENXIO; 2913 goto release; 2914 } 2915 } 2916 } 2917 #endif 2918 2919 do { 2920 if (flags & MSG_PEEK) { 2921 if (controlp != NULL) { 2922 *controlp = m_copym(m, 0, m->m_len, 2923 M_NOWAIT); 2924 controlp = &(*controlp)->m_next; 2925 } 2926 m = m->m_next; 2927 } else { 2928 sbfree(&so->so_rcv, m); 2929 so->so_rcv.sb_mb = m->m_next; 2930 m->m_next = NULL; 2931 *cme = m; 2932 cme = &(*cme)->m_next; 2933 m = so->so_rcv.sb_mb; 2934 } 2935 } while (m != NULL && m->m_type == MT_CONTROL); 2936 if ((flags & MSG_PEEK) == 0) 2937 sockbuf_pushsync(&so->so_rcv, nextrecord); 2938 while (cm != NULL) { 2939 cmn = cm->m_next; 2940 cm->m_next = NULL; 2941 if (controlp != NULL) 2942 *controlp = cm; 2943 else 2944 m_freem(cm); 2945 if (controlp != NULL) { 2946 while (*controlp != NULL) 2947 controlp = &(*controlp)->m_next; 2948 } 2949 cm = cmn; 2950 } 2951 if (m != NULL) 2952 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 2953 else 2954 nextrecord = so->so_rcv.sb_mb; 2955 orig_resid = 0; 2956 } 2957 if (m != NULL) { 2958 if ((flags & MSG_PEEK) == 0) { 2959 KASSERT(m->m_nextpkt == nextrecord, 2960 ("soreceive: post-control, nextrecord !sync")); 2961 if (nextrecord == NULL) { 2962 KASSERT(so->so_rcv.sb_mb == m, 2963 ("soreceive: post-control, sb_mb!=m")); 2964 KASSERT(so->so_rcv.sb_lastrecord == m, 2965 ("soreceive: post-control, lastrecord!=m")); 2966 } 2967 } 2968 type = m->m_type; 2969 if (type == MT_OOBDATA) 2970 flags |= MSG_OOB; 2971 } else { 2972 if ((flags & MSG_PEEK) == 0) { 2973 KASSERT(so->so_rcv.sb_mb == nextrecord, 2974 ("soreceive: sb_mb != nextrecord")); 2975 if (so->so_rcv.sb_mb == NULL) { 2976 KASSERT(so->so_rcv.sb_lastrecord == NULL, 2977 ("soreceive: sb_lastercord != NULL")); 2978 } 2979 } 2980 } 2981 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2982 SBLASTRECORDCHK(&so->so_rcv); 2983 SBLASTMBUFCHK(&so->so_rcv); 2984 2985 /* 2986 * Now continue to read any data mbufs off of the head of the socket 2987 * buffer until the read request is satisfied. Note that 'type' is 2988 * used to store the type of any mbuf reads that have happened so far 2989 * such that soreceive() can stop reading if the type changes, which 2990 * causes soreceive() to return only one of regular data and inline 2991 * out-of-band data in a single socket receive operation. 2992 */ 2993 moff = 0; 2994 offset = 0; 2995 while (m != NULL && !(m->m_flags & M_NOTREADY) && uio->uio_resid > 0 && 2996 error == 0) { 2997 /* 2998 * If the type of mbuf has changed since the last mbuf 2999 * examined ('type'), end the receive operation. 3000 */ 3001 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3002 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) { 3003 if (type != m->m_type) 3004 break; 3005 } else if (type == MT_OOBDATA) 3006 break; 3007 else 3008 KASSERT(m->m_type == MT_DATA, 3009 ("m->m_type == %d", m->m_type)); 3010 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 3011 len = uio->uio_resid; 3012 if (so->so_oobmark && len > so->so_oobmark - offset) 3013 len = so->so_oobmark - offset; 3014 if (len > m->m_len - moff) 3015 len = m->m_len - moff; 3016 /* 3017 * If mp is set, just pass back the mbufs. Otherwise copy 3018 * them out via the uio, then free. Sockbuf must be 3019 * consistent here (points to current mbuf, it points to next 3020 * record) when we drop priority; we must note any additions 3021 * to the sockbuf when we block interrupts again. 3022 */ 3023 if (mp == NULL) { 3024 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3025 SBLASTRECORDCHK(&so->so_rcv); 3026 SBLASTMBUFCHK(&so->so_rcv); 3027 SOCKBUF_UNLOCK(&so->so_rcv); 3028 if ((m->m_flags & M_EXTPG) != 0) 3029 error = m_unmapped_uiomove(m, moff, uio, 3030 (int)len); 3031 else 3032 error = uiomove(mtod(m, char *) + moff, 3033 (int)len, uio); 3034 SOCKBUF_LOCK(&so->so_rcv); 3035 if (error) { 3036 /* 3037 * The MT_SONAME mbuf has already been removed 3038 * from the record, so it is necessary to 3039 * remove the data mbufs, if any, to preserve 3040 * the invariant in the case of PR_ADDR that 3041 * requires MT_SONAME mbufs at the head of 3042 * each record. 3043 */ 3044 if (pr->pr_flags & PR_ATOMIC && 3045 ((flags & MSG_PEEK) == 0)) 3046 (void)sbdroprecord_locked(&so->so_rcv); 3047 SOCKBUF_UNLOCK(&so->so_rcv); 3048 goto release; 3049 } 3050 } else 3051 uio->uio_resid -= len; 3052 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3053 if (len == m->m_len - moff) { 3054 if (m->m_flags & M_EOR) 3055 flags |= MSG_EOR; 3056 if (flags & MSG_PEEK) { 3057 m = m->m_next; 3058 moff = 0; 3059 } else { 3060 nextrecord = m->m_nextpkt; 3061 sbfree(&so->so_rcv, m); 3062 if (mp != NULL) { 3063 m->m_nextpkt = NULL; 3064 *mp = m; 3065 mp = &m->m_next; 3066 so->so_rcv.sb_mb = m = m->m_next; 3067 *mp = NULL; 3068 } else { 3069 so->so_rcv.sb_mb = m_free(m); 3070 m = so->so_rcv.sb_mb; 3071 } 3072 sockbuf_pushsync(&so->so_rcv, nextrecord); 3073 SBLASTRECORDCHK(&so->so_rcv); 3074 SBLASTMBUFCHK(&so->so_rcv); 3075 } 3076 } else { 3077 if (flags & MSG_PEEK) 3078 moff += len; 3079 else { 3080 if (mp != NULL) { 3081 if (flags & MSG_DONTWAIT) { 3082 *mp = m_copym(m, 0, len, 3083 M_NOWAIT); 3084 if (*mp == NULL) { 3085 /* 3086 * m_copym() couldn't 3087 * allocate an mbuf. 3088 * Adjust uio_resid back 3089 * (it was adjusted 3090 * down by len bytes, 3091 * which we didn't end 3092 * up "copying" over). 3093 */ 3094 uio->uio_resid += len; 3095 break; 3096 } 3097 } else { 3098 SOCKBUF_UNLOCK(&so->so_rcv); 3099 *mp = m_copym(m, 0, len, 3100 M_WAITOK); 3101 SOCKBUF_LOCK(&so->so_rcv); 3102 } 3103 } 3104 sbcut_locked(&so->so_rcv, len); 3105 } 3106 } 3107 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3108 if (so->so_oobmark) { 3109 if ((flags & MSG_PEEK) == 0) { 3110 so->so_oobmark -= len; 3111 if (so->so_oobmark == 0) { 3112 so->so_rcv.sb_state |= SBS_RCVATMARK; 3113 break; 3114 } 3115 } else { 3116 offset += len; 3117 if (offset == so->so_oobmark) 3118 break; 3119 } 3120 } 3121 if (flags & MSG_EOR) 3122 break; 3123 /* 3124 * If the MSG_WAITALL flag is set (for non-atomic socket), we 3125 * must not quit until "uio->uio_resid == 0" or an error 3126 * termination. If a signal/timeout occurs, return with a 3127 * short count but without error. Keep sockbuf locked 3128 * against other readers. 3129 */ 3130 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 3131 !sosendallatonce(so) && nextrecord == NULL) { 3132 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3133 if (so->so_error || so->so_rerror || 3134 so->so_rcv.sb_state & SBS_CANTRCVMORE) 3135 break; 3136 /* 3137 * Notify the protocol that some data has been 3138 * drained before blocking. 3139 */ 3140 if (pr->pr_flags & PR_WANTRCVD) { 3141 SOCKBUF_UNLOCK(&so->so_rcv); 3142 VNET_SO_ASSERT(so); 3143 pr->pr_rcvd(so, flags); 3144 SOCKBUF_LOCK(&so->so_rcv); 3145 if (__predict_false(so->so_rcv.sb_mb == NULL && 3146 (so->so_error || so->so_rerror || 3147 so->so_rcv.sb_state & SBS_CANTRCVMORE))) 3148 break; 3149 } 3150 SBLASTRECORDCHK(&so->so_rcv); 3151 SBLASTMBUFCHK(&so->so_rcv); 3152 /* 3153 * We could receive some data while was notifying 3154 * the protocol. Skip blocking in this case. 3155 */ 3156 if (so->so_rcv.sb_mb == NULL) { 3157 error = sbwait(so, SO_RCV); 3158 if (error) { 3159 SOCKBUF_UNLOCK(&so->so_rcv); 3160 goto release; 3161 } 3162 } 3163 m = so->so_rcv.sb_mb; 3164 if (m != NULL) 3165 nextrecord = m->m_nextpkt; 3166 } 3167 } 3168 3169 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3170 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 3171 if (report_real_len) 3172 uio->uio_resid -= m_length(m, NULL) - moff; 3173 flags |= MSG_TRUNC; 3174 if ((flags & MSG_PEEK) == 0) 3175 (void) sbdroprecord_locked(&so->so_rcv); 3176 } 3177 if ((flags & MSG_PEEK) == 0) { 3178 if (m == NULL) { 3179 /* 3180 * First part is an inline SB_EMPTY_FIXUP(). Second 3181 * part makes sure sb_lastrecord is up-to-date if 3182 * there is still data in the socket buffer. 3183 */ 3184 so->so_rcv.sb_mb = nextrecord; 3185 if (so->so_rcv.sb_mb == NULL) { 3186 so->so_rcv.sb_mbtail = NULL; 3187 so->so_rcv.sb_lastrecord = NULL; 3188 } else if (nextrecord->m_nextpkt == NULL) 3189 so->so_rcv.sb_lastrecord = nextrecord; 3190 } 3191 SBLASTRECORDCHK(&so->so_rcv); 3192 SBLASTMBUFCHK(&so->so_rcv); 3193 /* 3194 * If soreceive() is being done from the socket callback, 3195 * then don't need to generate ACK to peer to update window, 3196 * since ACK will be generated on return to TCP. 3197 */ 3198 if (!(flags & MSG_SOCALLBCK) && 3199 (pr->pr_flags & PR_WANTRCVD)) { 3200 SOCKBUF_UNLOCK(&so->so_rcv); 3201 VNET_SO_ASSERT(so); 3202 pr->pr_rcvd(so, flags); 3203 SOCKBUF_LOCK(&so->so_rcv); 3204 } 3205 } 3206 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3207 if (orig_resid == uio->uio_resid && orig_resid && 3208 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 3209 SOCKBUF_UNLOCK(&so->so_rcv); 3210 goto restart; 3211 } 3212 SOCKBUF_UNLOCK(&so->so_rcv); 3213 3214 if (flagsp != NULL) 3215 *flagsp |= flags; 3216 release: 3217 return (error); 3218 } 3219 3220 int 3221 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, 3222 struct mbuf **mp, struct mbuf **controlp, int *flagsp) 3223 { 3224 int error, flags; 3225 3226 if (psa != NULL) 3227 *psa = NULL; 3228 if (controlp != NULL) 3229 *controlp = NULL; 3230 if (flagsp != NULL) { 3231 flags = *flagsp; 3232 if ((flags & MSG_OOB) != 0) 3233 return (soreceive_rcvoob(so, uio, flags)); 3234 } else { 3235 flags = 0; 3236 } 3237 if (mp != NULL) 3238 *mp = NULL; 3239 3240 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 3241 if (error) 3242 return (error); 3243 error = soreceive_generic_locked(so, psa, uio, mp, controlp, flagsp); 3244 SOCK_IO_RECV_UNLOCK(so); 3245 return (error); 3246 } 3247 3248 /* 3249 * Optimized version of soreceive() for stream (TCP) sockets. 3250 */ 3251 static int 3252 soreceive_stream_locked(struct socket *so, struct sockbuf *sb, 3253 struct sockaddr **psa, struct uio *uio, struct mbuf **mp0, 3254 struct mbuf **controlp, int flags) 3255 { 3256 int len = 0, error = 0, oresid; 3257 struct mbuf *m, *n = NULL; 3258 3259 SOCK_IO_RECV_ASSERT_LOCKED(so); 3260 3261 /* Easy one, no space to copyout anything. */ 3262 if (uio->uio_resid == 0) 3263 return (EINVAL); 3264 oresid = uio->uio_resid; 3265 3266 SOCKBUF_LOCK(sb); 3267 /* We will never ever get anything unless we are or were connected. */ 3268 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { 3269 error = ENOTCONN; 3270 goto out; 3271 } 3272 3273 restart: 3274 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3275 3276 /* Abort if socket has reported problems. */ 3277 if (so->so_error) { 3278 if (sbavail(sb) > 0) 3279 goto deliver; 3280 if (oresid > uio->uio_resid) 3281 goto out; 3282 error = so->so_error; 3283 if (!(flags & MSG_PEEK)) 3284 so->so_error = 0; 3285 goto out; 3286 } 3287 3288 /* Door is closed. Deliver what is left, if any. */ 3289 if (sb->sb_state & SBS_CANTRCVMORE) { 3290 if (sbavail(sb) > 0) 3291 goto deliver; 3292 else 3293 goto out; 3294 } 3295 3296 /* Socket buffer is empty and we shall not block. */ 3297 if (sbavail(sb) == 0 && 3298 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 3299 error = EAGAIN; 3300 goto out; 3301 } 3302 3303 /* Socket buffer got some data that we shall deliver now. */ 3304 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) && 3305 ((so->so_state & SS_NBIO) || 3306 (flags & (MSG_DONTWAIT|MSG_NBIO)) || 3307 sbavail(sb) >= sb->sb_lowat || 3308 sbavail(sb) >= uio->uio_resid || 3309 sbavail(sb) >= sb->sb_hiwat) ) { 3310 goto deliver; 3311 } 3312 3313 /* On MSG_WAITALL we must wait until all data or error arrives. */ 3314 if ((flags & MSG_WAITALL) && 3315 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat)) 3316 goto deliver; 3317 3318 /* 3319 * Wait and block until (more) data comes in. 3320 * NB: Drops the sockbuf lock during wait. 3321 */ 3322 error = sbwait(so, SO_RCV); 3323 if (error) 3324 goto out; 3325 goto restart; 3326 3327 deliver: 3328 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3329 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__)); 3330 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 3331 3332 /* Statistics. */ 3333 if (uio->uio_td) 3334 uio->uio_td->td_ru.ru_msgrcv++; 3335 3336 /* Fill uio until full or current end of socket buffer is reached. */ 3337 len = min(uio->uio_resid, sbavail(sb)); 3338 if (mp0 != NULL) { 3339 /* Dequeue as many mbufs as possible. */ 3340 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 3341 if (*mp0 == NULL) 3342 *mp0 = sb->sb_mb; 3343 else 3344 m_cat(*mp0, sb->sb_mb); 3345 for (m = sb->sb_mb; 3346 m != NULL && m->m_len <= len; 3347 m = m->m_next) { 3348 KASSERT(!(m->m_flags & M_NOTREADY), 3349 ("%s: m %p not available", __func__, m)); 3350 len -= m->m_len; 3351 uio->uio_resid -= m->m_len; 3352 sbfree(sb, m); 3353 n = m; 3354 } 3355 n->m_next = NULL; 3356 sb->sb_mb = m; 3357 sb->sb_lastrecord = sb->sb_mb; 3358 if (sb->sb_mb == NULL) 3359 SB_EMPTY_FIXUP(sb); 3360 } 3361 /* Copy the remainder. */ 3362 if (len > 0) { 3363 KASSERT(sb->sb_mb != NULL, 3364 ("%s: len > 0 && sb->sb_mb empty", __func__)); 3365 3366 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT); 3367 if (m == NULL) 3368 len = 0; /* Don't flush data from sockbuf. */ 3369 else 3370 uio->uio_resid -= len; 3371 if (*mp0 != NULL) 3372 m_cat(*mp0, m); 3373 else 3374 *mp0 = m; 3375 if (*mp0 == NULL) { 3376 error = ENOBUFS; 3377 goto out; 3378 } 3379 } 3380 } else { 3381 /* NB: Must unlock socket buffer as uiomove may sleep. */ 3382 SOCKBUF_UNLOCK(sb); 3383 error = m_mbuftouio(uio, sb->sb_mb, len); 3384 SOCKBUF_LOCK(sb); 3385 if (error) 3386 goto out; 3387 } 3388 SBLASTRECORDCHK(sb); 3389 SBLASTMBUFCHK(sb); 3390 3391 /* 3392 * Remove the delivered data from the socket buffer unless we 3393 * were only peeking. 3394 */ 3395 if (!(flags & MSG_PEEK)) { 3396 if (len > 0) 3397 sbdrop_locked(sb, len); 3398 3399 /* Notify protocol that we drained some data. */ 3400 if ((so->so_proto->pr_flags & PR_WANTRCVD) && 3401 (((flags & MSG_WAITALL) && uio->uio_resid > 0) || 3402 !(flags & MSG_SOCALLBCK))) { 3403 SOCKBUF_UNLOCK(sb); 3404 VNET_SO_ASSERT(so); 3405 so->so_proto->pr_rcvd(so, flags); 3406 SOCKBUF_LOCK(sb); 3407 } 3408 } 3409 3410 /* 3411 * For MSG_WAITALL we may have to loop again and wait for 3412 * more data to come in. 3413 */ 3414 if ((flags & MSG_WAITALL) && uio->uio_resid > 0) 3415 goto restart; 3416 out: 3417 SBLASTRECORDCHK(sb); 3418 SBLASTMBUFCHK(sb); 3419 SOCKBUF_UNLOCK(sb); 3420 return (error); 3421 } 3422 3423 int 3424 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, 3425 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3426 { 3427 struct sockbuf *sb; 3428 int error, flags; 3429 3430 sb = &so->so_rcv; 3431 3432 /* We only do stream sockets. */ 3433 if (so->so_type != SOCK_STREAM) 3434 return (EINVAL); 3435 if (psa != NULL) 3436 *psa = NULL; 3437 if (flagsp != NULL) 3438 flags = *flagsp & ~MSG_EOR; 3439 else 3440 flags = 0; 3441 if (controlp != NULL) 3442 *controlp = NULL; 3443 if (flags & MSG_OOB) 3444 return (soreceive_rcvoob(so, uio, flags)); 3445 if (mp0 != NULL) 3446 *mp0 = NULL; 3447 3448 #ifdef KERN_TLS 3449 /* 3450 * KTLS store TLS records as records with a control message to 3451 * describe the framing. 3452 * 3453 * We check once here before acquiring locks to optimize the 3454 * common case. 3455 */ 3456 if (sb->sb_tls_info != NULL) 3457 return (soreceive_generic(so, psa, uio, mp0, controlp, 3458 flagsp)); 3459 #endif 3460 3461 /* 3462 * Prevent other threads from reading from the socket. This lock may be 3463 * dropped in order to sleep waiting for data to arrive. 3464 */ 3465 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 3466 if (error) 3467 return (error); 3468 #ifdef KERN_TLS 3469 if (__predict_false(sb->sb_tls_info != NULL)) { 3470 SOCK_IO_RECV_UNLOCK(so); 3471 return (soreceive_generic(so, psa, uio, mp0, controlp, 3472 flagsp)); 3473 } 3474 #endif 3475 error = soreceive_stream_locked(so, sb, psa, uio, mp0, controlp, flags); 3476 SOCK_IO_RECV_UNLOCK(so); 3477 return (error); 3478 } 3479 3480 /* 3481 * Optimized version of soreceive() for simple datagram cases from userspace. 3482 * Unlike in the stream case, we're able to drop a datagram if copyout() 3483 * fails, and because we handle datagrams atomically, we don't need to use a 3484 * sleep lock to prevent I/O interlacing. 3485 */ 3486 int 3487 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 3488 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3489 { 3490 struct mbuf *m, *m2; 3491 int flags, error; 3492 ssize_t len; 3493 struct protosw *pr = so->so_proto; 3494 struct mbuf *nextrecord; 3495 3496 if (psa != NULL) 3497 *psa = NULL; 3498 if (controlp != NULL) 3499 *controlp = NULL; 3500 if (flagsp != NULL) 3501 flags = *flagsp &~ MSG_EOR; 3502 else 3503 flags = 0; 3504 3505 /* 3506 * For any complicated cases, fall back to the full 3507 * soreceive_generic(). 3508 */ 3509 if (mp0 != NULL || (flags & (MSG_PEEK | MSG_OOB | MSG_TRUNC))) 3510 return (soreceive_generic(so, psa, uio, mp0, controlp, 3511 flagsp)); 3512 3513 /* 3514 * Enforce restrictions on use. 3515 */ 3516 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0, 3517 ("soreceive_dgram: wantrcvd")); 3518 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic")); 3519 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0, 3520 ("soreceive_dgram: SBS_RCVATMARK")); 3521 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0, 3522 ("soreceive_dgram: P_CONNREQUIRED")); 3523 3524 /* 3525 * Loop blocking while waiting for a datagram. 3526 */ 3527 SOCKBUF_LOCK(&so->so_rcv); 3528 while ((m = so->so_rcv.sb_mb) == NULL) { 3529 KASSERT(sbavail(&so->so_rcv) == 0, 3530 ("soreceive_dgram: sb_mb NULL but sbavail %u", 3531 sbavail(&so->so_rcv))); 3532 if (so->so_error) { 3533 error = so->so_error; 3534 so->so_error = 0; 3535 SOCKBUF_UNLOCK(&so->so_rcv); 3536 return (error); 3537 } 3538 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 3539 uio->uio_resid == 0) { 3540 SOCKBUF_UNLOCK(&so->so_rcv); 3541 return (0); 3542 } 3543 if ((so->so_state & SS_NBIO) || 3544 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 3545 SOCKBUF_UNLOCK(&so->so_rcv); 3546 return (EWOULDBLOCK); 3547 } 3548 SBLASTRECORDCHK(&so->so_rcv); 3549 SBLASTMBUFCHK(&so->so_rcv); 3550 error = sbwait(so, SO_RCV); 3551 if (error) { 3552 SOCKBUF_UNLOCK(&so->so_rcv); 3553 return (error); 3554 } 3555 } 3556 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3557 3558 if (uio->uio_td) 3559 uio->uio_td->td_ru.ru_msgrcv++; 3560 SBLASTRECORDCHK(&so->so_rcv); 3561 SBLASTMBUFCHK(&so->so_rcv); 3562 nextrecord = m->m_nextpkt; 3563 if (nextrecord == NULL) { 3564 KASSERT(so->so_rcv.sb_lastrecord == m, 3565 ("soreceive_dgram: lastrecord != m")); 3566 } 3567 3568 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord, 3569 ("soreceive_dgram: m_nextpkt != nextrecord")); 3570 3571 /* 3572 * Pull 'm' and its chain off the front of the packet queue. 3573 */ 3574 so->so_rcv.sb_mb = NULL; 3575 sockbuf_pushsync(&so->so_rcv, nextrecord); 3576 3577 /* 3578 * Walk 'm's chain and free that many bytes from the socket buffer. 3579 */ 3580 for (m2 = m; m2 != NULL; m2 = m2->m_next) 3581 sbfree(&so->so_rcv, m2); 3582 3583 /* 3584 * Do a few last checks before we let go of the lock. 3585 */ 3586 SBLASTRECORDCHK(&so->so_rcv); 3587 SBLASTMBUFCHK(&so->so_rcv); 3588 SOCKBUF_UNLOCK(&so->so_rcv); 3589 3590 if (pr->pr_flags & PR_ADDR) { 3591 KASSERT(m->m_type == MT_SONAME, 3592 ("m->m_type == %d", m->m_type)); 3593 if (psa != NULL) 3594 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 3595 M_WAITOK); 3596 m = m_free(m); 3597 } 3598 KASSERT(m, ("%s: no data or control after soname", __func__)); 3599 3600 /* 3601 * Packet to copyout() is now in 'm' and it is disconnected from the 3602 * queue. 3603 * 3604 * Process one or more MT_CONTROL mbufs present before any data mbufs 3605 * in the first mbuf chain on the socket buffer. We call into the 3606 * protocol to perform externalization (or freeing if controlp == 3607 * NULL). In some cases there can be only MT_CONTROL mbufs without 3608 * MT_DATA mbufs. 3609 */ 3610 if (m->m_type == MT_CONTROL) { 3611 struct mbuf *cm = NULL, *cmn; 3612 struct mbuf **cme = &cm; 3613 3614 do { 3615 m2 = m->m_next; 3616 m->m_next = NULL; 3617 *cme = m; 3618 cme = &(*cme)->m_next; 3619 m = m2; 3620 } while (m != NULL && m->m_type == MT_CONTROL); 3621 while (cm != NULL) { 3622 cmn = cm->m_next; 3623 cm->m_next = NULL; 3624 if (controlp != NULL) 3625 *controlp = cm; 3626 else 3627 m_freem(cm); 3628 if (controlp != NULL) { 3629 while (*controlp != NULL) 3630 controlp = &(*controlp)->m_next; 3631 } 3632 cm = cmn; 3633 } 3634 } 3635 KASSERT(m == NULL || m->m_type == MT_DATA, 3636 ("soreceive_dgram: !data")); 3637 while (m != NULL && uio->uio_resid > 0) { 3638 len = uio->uio_resid; 3639 if (len > m->m_len) 3640 len = m->m_len; 3641 error = uiomove(mtod(m, char *), (int)len, uio); 3642 if (error) { 3643 m_freem(m); 3644 return (error); 3645 } 3646 if (len == m->m_len) 3647 m = m_free(m); 3648 else { 3649 m->m_data += len; 3650 m->m_len -= len; 3651 } 3652 } 3653 if (m != NULL) { 3654 flags |= MSG_TRUNC; 3655 m_freem(m); 3656 } 3657 if (flagsp != NULL) 3658 *flagsp |= flags; 3659 return (0); 3660 } 3661 3662 int 3663 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, 3664 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3665 { 3666 int error; 3667 3668 CURVNET_SET(so->so_vnet); 3669 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp); 3670 CURVNET_RESTORE(); 3671 return (error); 3672 } 3673 3674 int 3675 soshutdown(struct socket *so, enum shutdown_how how) 3676 { 3677 int error; 3678 3679 CURVNET_SET(so->so_vnet); 3680 error = so->so_proto->pr_shutdown(so, how); 3681 CURVNET_RESTORE(); 3682 3683 return (error); 3684 } 3685 3686 /* 3687 * Used by several pr_shutdown implementations that use generic socket buffers. 3688 */ 3689 void 3690 sorflush(struct socket *so) 3691 { 3692 int error; 3693 3694 VNET_SO_ASSERT(so); 3695 3696 /* 3697 * Dislodge threads currently blocked in receive and wait to acquire 3698 * a lock against other simultaneous readers before clearing the 3699 * socket buffer. Don't let our acquire be interrupted by a signal 3700 * despite any existing socket disposition on interruptable waiting. 3701 * 3702 * The SOCK_IO_RECV_LOCK() is important here as there some pr_soreceive 3703 * methods that read the top of the socket buffer without acquisition 3704 * of the socket buffer mutex, assuming that top of the buffer 3705 * exclusively belongs to the read(2) syscall. This is handy when 3706 * performing MSG_PEEK. 3707 */ 3708 socantrcvmore(so); 3709 3710 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 3711 if (error != 0) { 3712 KASSERT(SOLISTENING(so), 3713 ("%s: soiolock(%p) failed", __func__, so)); 3714 return; 3715 } 3716 3717 sbrelease(so, SO_RCV); 3718 SOCK_IO_RECV_UNLOCK(so); 3719 3720 } 3721 3722 int 3723 sosetfib(struct socket *so, int fibnum) 3724 { 3725 if (fibnum < 0 || fibnum >= rt_numfibs) 3726 return (EINVAL); 3727 3728 SOCK_LOCK(so); 3729 so->so_fibnum = fibnum; 3730 SOCK_UNLOCK(so); 3731 3732 return (0); 3733 } 3734 3735 #ifdef SOCKET_HHOOK 3736 /* 3737 * Wrapper for Socket established helper hook. 3738 * Parameters: socket, context of the hook point, hook id. 3739 */ 3740 static inline int 3741 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) 3742 { 3743 struct socket_hhook_data hhook_data = { 3744 .so = so, 3745 .hctx = hctx, 3746 .m = NULL, 3747 .status = 0 3748 }; 3749 3750 CURVNET_SET(so->so_vnet); 3751 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd); 3752 CURVNET_RESTORE(); 3753 3754 /* Ugly but needed, since hhooks return void for now */ 3755 return (hhook_data.status); 3756 } 3757 #endif 3758 3759 /* 3760 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 3761 * additional variant to handle the case where the option value needs to be 3762 * some kind of integer, but not a specific size. In addition to their use 3763 * here, these functions are also called by the protocol-level pr_ctloutput() 3764 * routines. 3765 */ 3766 int 3767 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 3768 { 3769 size_t valsize; 3770 3771 /* 3772 * If the user gives us more than we wanted, we ignore it, but if we 3773 * don't get the minimum length the caller wants, we return EINVAL. 3774 * On success, sopt->sopt_valsize is set to however much we actually 3775 * retrieved. 3776 */ 3777 if ((valsize = sopt->sopt_valsize) < minlen) 3778 return EINVAL; 3779 if (valsize > len) 3780 sopt->sopt_valsize = valsize = len; 3781 3782 if (sopt->sopt_td != NULL) 3783 return (copyin(sopt->sopt_val, buf, valsize)); 3784 3785 bcopy(sopt->sopt_val, buf, valsize); 3786 return (0); 3787 } 3788 3789 /* 3790 * Kernel version of setsockopt(2). 3791 * 3792 * XXX: optlen is size_t, not socklen_t 3793 */ 3794 int 3795 so_setsockopt(struct socket *so, int level, int optname, void *optval, 3796 size_t optlen) 3797 { 3798 struct sockopt sopt; 3799 3800 sopt.sopt_level = level; 3801 sopt.sopt_name = optname; 3802 sopt.sopt_dir = SOPT_SET; 3803 sopt.sopt_val = optval; 3804 sopt.sopt_valsize = optlen; 3805 sopt.sopt_td = NULL; 3806 return (sosetopt(so, &sopt)); 3807 } 3808 3809 int 3810 sosetopt(struct socket *so, struct sockopt *sopt) 3811 { 3812 int error, optval; 3813 struct linger l; 3814 struct timeval tv; 3815 sbintime_t val, *valp; 3816 uint32_t val32; 3817 #ifdef MAC 3818 struct mac extmac; 3819 #endif 3820 3821 CURVNET_SET(so->so_vnet); 3822 error = 0; 3823 if (sopt->sopt_level != SOL_SOCKET) { 3824 error = so->so_proto->pr_ctloutput(so, sopt); 3825 } else { 3826 switch (sopt->sopt_name) { 3827 case SO_ACCEPTFILTER: 3828 error = accept_filt_setopt(so, sopt); 3829 if (error) 3830 goto bad; 3831 break; 3832 3833 case SO_LINGER: 3834 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 3835 if (error) 3836 goto bad; 3837 if (l.l_linger < 0 || 3838 l.l_linger > USHRT_MAX || 3839 l.l_linger > (INT_MAX / hz)) { 3840 error = EDOM; 3841 goto bad; 3842 } 3843 SOCK_LOCK(so); 3844 so->so_linger = l.l_linger; 3845 if (l.l_onoff) 3846 so->so_options |= SO_LINGER; 3847 else 3848 so->so_options &= ~SO_LINGER; 3849 SOCK_UNLOCK(so); 3850 break; 3851 3852 case SO_DEBUG: 3853 case SO_KEEPALIVE: 3854 case SO_DONTROUTE: 3855 case SO_USELOOPBACK: 3856 case SO_BROADCAST: 3857 case SO_REUSEADDR: 3858 case SO_REUSEPORT: 3859 case SO_REUSEPORT_LB: 3860 case SO_OOBINLINE: 3861 case SO_TIMESTAMP: 3862 case SO_BINTIME: 3863 case SO_NOSIGPIPE: 3864 case SO_NO_DDP: 3865 case SO_NO_OFFLOAD: 3866 case SO_RERROR: 3867 error = sooptcopyin(sopt, &optval, sizeof optval, 3868 sizeof optval); 3869 if (error) 3870 goto bad; 3871 SOCK_LOCK(so); 3872 if (optval) 3873 so->so_options |= sopt->sopt_name; 3874 else 3875 so->so_options &= ~sopt->sopt_name; 3876 SOCK_UNLOCK(so); 3877 break; 3878 3879 case SO_SETFIB: 3880 error = so->so_proto->pr_ctloutput(so, sopt); 3881 break; 3882 3883 case SO_USER_COOKIE: 3884 error = sooptcopyin(sopt, &val32, sizeof val32, 3885 sizeof val32); 3886 if (error) 3887 goto bad; 3888 so->so_user_cookie = val32; 3889 break; 3890 3891 case SO_SNDBUF: 3892 case SO_RCVBUF: 3893 case SO_SNDLOWAT: 3894 case SO_RCVLOWAT: 3895 error = so->so_proto->pr_setsbopt(so, sopt); 3896 if (error) 3897 goto bad; 3898 break; 3899 3900 case SO_SNDTIMEO: 3901 case SO_RCVTIMEO: 3902 #ifdef COMPAT_FREEBSD32 3903 if (SV_CURPROC_FLAG(SV_ILP32)) { 3904 struct timeval32 tv32; 3905 3906 error = sooptcopyin(sopt, &tv32, sizeof tv32, 3907 sizeof tv32); 3908 CP(tv32, tv, tv_sec); 3909 CP(tv32, tv, tv_usec); 3910 } else 3911 #endif 3912 error = sooptcopyin(sopt, &tv, sizeof tv, 3913 sizeof tv); 3914 if (error) 3915 goto bad; 3916 if (tv.tv_sec < 0 || tv.tv_usec < 0 || 3917 tv.tv_usec >= 1000000) { 3918 error = EDOM; 3919 goto bad; 3920 } 3921 if (tv.tv_sec > INT32_MAX) 3922 val = SBT_MAX; 3923 else 3924 val = tvtosbt(tv); 3925 SOCK_LOCK(so); 3926 valp = sopt->sopt_name == SO_SNDTIMEO ? 3927 (SOLISTENING(so) ? &so->sol_sbsnd_timeo : 3928 &so->so_snd.sb_timeo) : 3929 (SOLISTENING(so) ? &so->sol_sbrcv_timeo : 3930 &so->so_rcv.sb_timeo); 3931 *valp = val; 3932 SOCK_UNLOCK(so); 3933 break; 3934 3935 case SO_LABEL: 3936 #ifdef MAC 3937 error = sooptcopyin(sopt, &extmac, sizeof extmac, 3938 sizeof extmac); 3939 if (error) 3940 goto bad; 3941 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 3942 so, &extmac); 3943 #else 3944 error = EOPNOTSUPP; 3945 #endif 3946 break; 3947 3948 case SO_TS_CLOCK: 3949 error = sooptcopyin(sopt, &optval, sizeof optval, 3950 sizeof optval); 3951 if (error) 3952 goto bad; 3953 if (optval < 0 || optval > SO_TS_CLOCK_MAX) { 3954 error = EINVAL; 3955 goto bad; 3956 } 3957 so->so_ts_clock = optval; 3958 break; 3959 3960 case SO_MAX_PACING_RATE: 3961 error = sooptcopyin(sopt, &val32, sizeof(val32), 3962 sizeof(val32)); 3963 if (error) 3964 goto bad; 3965 so->so_max_pacing_rate = val32; 3966 break; 3967 3968 case SO_SPLICE: { 3969 struct splice splice; 3970 3971 #ifdef COMPAT_FREEBSD32 3972 if (SV_CURPROC_FLAG(SV_ILP32)) { 3973 struct splice32 splice32; 3974 3975 error = sooptcopyin(sopt, &splice32, 3976 sizeof(splice32), sizeof(splice32)); 3977 if (error == 0) { 3978 splice.sp_fd = splice32.sp_fd; 3979 splice.sp_max = splice32.sp_max; 3980 CP(splice32.sp_idle, splice.sp_idle, 3981 tv_sec); 3982 CP(splice32.sp_idle, splice.sp_idle, 3983 tv_usec); 3984 } 3985 } else 3986 #endif 3987 { 3988 error = sooptcopyin(sopt, &splice, 3989 sizeof(splice), sizeof(splice)); 3990 } 3991 if (error) 3992 goto bad; 3993 #ifdef KTRACE 3994 if (KTRPOINT(curthread, KTR_STRUCT)) 3995 ktrsplice(&splice); 3996 #endif 3997 3998 error = splice_init(); 3999 if (error != 0) 4000 goto bad; 4001 4002 if (splice.sp_fd >= 0) { 4003 struct file *fp; 4004 struct socket *so2; 4005 4006 if (!cap_rights_contains(sopt->sopt_rights, 4007 &cap_recv_rights)) { 4008 error = ENOTCAPABLE; 4009 goto bad; 4010 } 4011 error = getsock(sopt->sopt_td, splice.sp_fd, 4012 &cap_send_rights, &fp); 4013 if (error != 0) 4014 goto bad; 4015 so2 = fp->f_data; 4016 4017 error = so_splice(so, so2, &splice); 4018 fdrop(fp, sopt->sopt_td); 4019 } else { 4020 error = so_unsplice(so, false); 4021 } 4022 break; 4023 } 4024 default: 4025 #ifdef SOCKET_HHOOK 4026 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 4027 error = hhook_run_socket(so, sopt, 4028 HHOOK_SOCKET_OPT); 4029 else 4030 #endif 4031 error = ENOPROTOOPT; 4032 break; 4033 } 4034 if (error == 0) 4035 (void)so->so_proto->pr_ctloutput(so, sopt); 4036 } 4037 bad: 4038 CURVNET_RESTORE(); 4039 return (error); 4040 } 4041 4042 /* 4043 * Helper routine for getsockopt. 4044 */ 4045 int 4046 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 4047 { 4048 int error; 4049 size_t valsize; 4050 4051 error = 0; 4052 4053 /* 4054 * Documented get behavior is that we always return a value, possibly 4055 * truncated to fit in the user's buffer. Traditional behavior is 4056 * that we always tell the user precisely how much we copied, rather 4057 * than something useful like the total amount we had available for 4058 * her. Note that this interface is not idempotent; the entire 4059 * answer must be generated ahead of time. 4060 */ 4061 valsize = min(len, sopt->sopt_valsize); 4062 sopt->sopt_valsize = valsize; 4063 if (sopt->sopt_val != NULL) { 4064 if (sopt->sopt_td != NULL) 4065 error = copyout(buf, sopt->sopt_val, valsize); 4066 else 4067 bcopy(buf, sopt->sopt_val, valsize); 4068 } 4069 return (error); 4070 } 4071 4072 int 4073 sogetopt(struct socket *so, struct sockopt *sopt) 4074 { 4075 int error, optval; 4076 struct linger l; 4077 struct timeval tv; 4078 #ifdef MAC 4079 struct mac extmac; 4080 #endif 4081 4082 CURVNET_SET(so->so_vnet); 4083 error = 0; 4084 if (sopt->sopt_level != SOL_SOCKET) { 4085 error = so->so_proto->pr_ctloutput(so, sopt); 4086 CURVNET_RESTORE(); 4087 return (error); 4088 } else { 4089 switch (sopt->sopt_name) { 4090 case SO_ACCEPTFILTER: 4091 error = accept_filt_getopt(so, sopt); 4092 break; 4093 4094 case SO_LINGER: 4095 SOCK_LOCK(so); 4096 l.l_onoff = so->so_options & SO_LINGER; 4097 l.l_linger = so->so_linger; 4098 SOCK_UNLOCK(so); 4099 error = sooptcopyout(sopt, &l, sizeof l); 4100 break; 4101 4102 case SO_USELOOPBACK: 4103 case SO_DONTROUTE: 4104 case SO_DEBUG: 4105 case SO_KEEPALIVE: 4106 case SO_REUSEADDR: 4107 case SO_REUSEPORT: 4108 case SO_REUSEPORT_LB: 4109 case SO_BROADCAST: 4110 case SO_OOBINLINE: 4111 case SO_ACCEPTCONN: 4112 case SO_TIMESTAMP: 4113 case SO_BINTIME: 4114 case SO_NOSIGPIPE: 4115 case SO_NO_DDP: 4116 case SO_NO_OFFLOAD: 4117 case SO_RERROR: 4118 optval = so->so_options & sopt->sopt_name; 4119 integer: 4120 error = sooptcopyout(sopt, &optval, sizeof optval); 4121 break; 4122 4123 case SO_FIB: 4124 SOCK_LOCK(so); 4125 optval = so->so_fibnum; 4126 SOCK_UNLOCK(so); 4127 goto integer; 4128 4129 case SO_DOMAIN: 4130 optval = so->so_proto->pr_domain->dom_family; 4131 goto integer; 4132 4133 case SO_TYPE: 4134 optval = so->so_type; 4135 goto integer; 4136 4137 case SO_PROTOCOL: 4138 optval = so->so_proto->pr_protocol; 4139 goto integer; 4140 4141 case SO_ERROR: 4142 SOCK_LOCK(so); 4143 if (so->so_error) { 4144 optval = so->so_error; 4145 so->so_error = 0; 4146 } else { 4147 optval = so->so_rerror; 4148 so->so_rerror = 0; 4149 } 4150 SOCK_UNLOCK(so); 4151 goto integer; 4152 4153 case SO_SNDBUF: 4154 SOCK_LOCK(so); 4155 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat : 4156 so->so_snd.sb_hiwat; 4157 SOCK_UNLOCK(so); 4158 goto integer; 4159 4160 case SO_RCVBUF: 4161 SOCK_LOCK(so); 4162 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat : 4163 so->so_rcv.sb_hiwat; 4164 SOCK_UNLOCK(so); 4165 goto integer; 4166 4167 case SO_SNDLOWAT: 4168 SOCK_LOCK(so); 4169 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat : 4170 so->so_snd.sb_lowat; 4171 SOCK_UNLOCK(so); 4172 goto integer; 4173 4174 case SO_RCVLOWAT: 4175 SOCK_LOCK(so); 4176 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat : 4177 so->so_rcv.sb_lowat; 4178 SOCK_UNLOCK(so); 4179 goto integer; 4180 4181 case SO_SNDTIMEO: 4182 case SO_RCVTIMEO: 4183 SOCK_LOCK(so); 4184 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ? 4185 (SOLISTENING(so) ? so->sol_sbsnd_timeo : 4186 so->so_snd.sb_timeo) : 4187 (SOLISTENING(so) ? so->sol_sbrcv_timeo : 4188 so->so_rcv.sb_timeo)); 4189 SOCK_UNLOCK(so); 4190 #ifdef COMPAT_FREEBSD32 4191 if (SV_CURPROC_FLAG(SV_ILP32)) { 4192 struct timeval32 tv32; 4193 4194 CP(tv, tv32, tv_sec); 4195 CP(tv, tv32, tv_usec); 4196 error = sooptcopyout(sopt, &tv32, sizeof tv32); 4197 } else 4198 #endif 4199 error = sooptcopyout(sopt, &tv, sizeof tv); 4200 break; 4201 4202 case SO_LABEL: 4203 #ifdef MAC 4204 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 4205 sizeof(extmac)); 4206 if (error) 4207 goto bad; 4208 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 4209 so, &extmac); 4210 if (error) 4211 goto bad; 4212 /* Don't copy out extmac, it is unchanged. */ 4213 #else 4214 error = EOPNOTSUPP; 4215 #endif 4216 break; 4217 4218 case SO_PEERLABEL: 4219 #ifdef MAC 4220 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 4221 sizeof(extmac)); 4222 if (error) 4223 goto bad; 4224 error = mac_getsockopt_peerlabel( 4225 sopt->sopt_td->td_ucred, so, &extmac); 4226 if (error) 4227 goto bad; 4228 /* Don't copy out extmac, it is unchanged. */ 4229 #else 4230 error = EOPNOTSUPP; 4231 #endif 4232 break; 4233 4234 case SO_LISTENQLIMIT: 4235 SOCK_LOCK(so); 4236 optval = SOLISTENING(so) ? so->sol_qlimit : 0; 4237 SOCK_UNLOCK(so); 4238 goto integer; 4239 4240 case SO_LISTENQLEN: 4241 SOCK_LOCK(so); 4242 optval = SOLISTENING(so) ? so->sol_qlen : 0; 4243 SOCK_UNLOCK(so); 4244 goto integer; 4245 4246 case SO_LISTENINCQLEN: 4247 SOCK_LOCK(so); 4248 optval = SOLISTENING(so) ? so->sol_incqlen : 0; 4249 SOCK_UNLOCK(so); 4250 goto integer; 4251 4252 case SO_TS_CLOCK: 4253 optval = so->so_ts_clock; 4254 goto integer; 4255 4256 case SO_MAX_PACING_RATE: 4257 optval = so->so_max_pacing_rate; 4258 goto integer; 4259 4260 case SO_SPLICE: { 4261 off_t n; 4262 4263 /* 4264 * Acquire the I/O lock to serialize with 4265 * so_splice_xfer(). This is not required for 4266 * correctness, but makes testing simpler: once a byte 4267 * has been transmitted to the sink and observed (e.g., 4268 * by reading from the socket to which the sink is 4269 * connected), a subsequent getsockopt(SO_SPLICE) will 4270 * return an up-to-date value. 4271 */ 4272 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT); 4273 if (error != 0) 4274 goto bad; 4275 SOCK_LOCK(so); 4276 if (SOLISTENING(so)) { 4277 n = 0; 4278 } else { 4279 n = so->so_splice_sent; 4280 } 4281 SOCK_UNLOCK(so); 4282 SOCK_IO_RECV_UNLOCK(so); 4283 error = sooptcopyout(sopt, &n, sizeof(n)); 4284 break; 4285 } 4286 4287 default: 4288 #ifdef SOCKET_HHOOK 4289 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 4290 error = hhook_run_socket(so, sopt, 4291 HHOOK_SOCKET_OPT); 4292 else 4293 #endif 4294 error = ENOPROTOOPT; 4295 break; 4296 } 4297 } 4298 bad: 4299 CURVNET_RESTORE(); 4300 return (error); 4301 } 4302 4303 int 4304 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 4305 { 4306 struct mbuf *m, *m_prev; 4307 int sopt_size = sopt->sopt_valsize; 4308 4309 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 4310 if (m == NULL) 4311 return ENOBUFS; 4312 if (sopt_size > MLEN) { 4313 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT); 4314 if ((m->m_flags & M_EXT) == 0) { 4315 m_free(m); 4316 return ENOBUFS; 4317 } 4318 m->m_len = min(MCLBYTES, sopt_size); 4319 } else { 4320 m->m_len = min(MLEN, sopt_size); 4321 } 4322 sopt_size -= m->m_len; 4323 *mp = m; 4324 m_prev = m; 4325 4326 while (sopt_size) { 4327 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 4328 if (m == NULL) { 4329 m_freem(*mp); 4330 return ENOBUFS; 4331 } 4332 if (sopt_size > MLEN) { 4333 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK : 4334 M_NOWAIT); 4335 if ((m->m_flags & M_EXT) == 0) { 4336 m_freem(m); 4337 m_freem(*mp); 4338 return ENOBUFS; 4339 } 4340 m->m_len = min(MCLBYTES, sopt_size); 4341 } else { 4342 m->m_len = min(MLEN, sopt_size); 4343 } 4344 sopt_size -= m->m_len; 4345 m_prev->m_next = m; 4346 m_prev = m; 4347 } 4348 return (0); 4349 } 4350 4351 int 4352 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 4353 { 4354 struct mbuf *m0 = m; 4355 4356 if (sopt->sopt_val == NULL) 4357 return (0); 4358 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 4359 if (sopt->sopt_td != NULL) { 4360 int error; 4361 4362 error = copyin(sopt->sopt_val, mtod(m, char *), 4363 m->m_len); 4364 if (error != 0) { 4365 m_freem(m0); 4366 return(error); 4367 } 4368 } else 4369 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 4370 sopt->sopt_valsize -= m->m_len; 4371 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 4372 m = m->m_next; 4373 } 4374 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 4375 panic("ip6_sooptmcopyin"); 4376 return (0); 4377 } 4378 4379 int 4380 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 4381 { 4382 struct mbuf *m0 = m; 4383 size_t valsize = 0; 4384 4385 if (sopt->sopt_val == NULL) 4386 return (0); 4387 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 4388 if (sopt->sopt_td != NULL) { 4389 int error; 4390 4391 error = copyout(mtod(m, char *), sopt->sopt_val, 4392 m->m_len); 4393 if (error != 0) { 4394 m_freem(m0); 4395 return(error); 4396 } 4397 } else 4398 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 4399 sopt->sopt_valsize -= m->m_len; 4400 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 4401 valsize += m->m_len; 4402 m = m->m_next; 4403 } 4404 if (m != NULL) { 4405 /* enough soopt buffer should be given from user-land */ 4406 m_freem(m0); 4407 return(EINVAL); 4408 } 4409 sopt->sopt_valsize = valsize; 4410 return (0); 4411 } 4412 4413 /* 4414 * sohasoutofband(): protocol notifies socket layer of the arrival of new 4415 * out-of-band data, which will then notify socket consumers. 4416 */ 4417 void 4418 sohasoutofband(struct socket *so) 4419 { 4420 4421 if (so->so_sigio != NULL) 4422 pgsigio(&so->so_sigio, SIGURG, 0); 4423 selwakeuppri(&so->so_rdsel, PSOCK); 4424 } 4425 4426 int 4427 sopoll_generic(struct socket *so, int events, struct thread *td) 4428 { 4429 int revents; 4430 4431 SOCK_LOCK(so); 4432 if (SOLISTENING(so)) { 4433 if (!(events & (POLLIN | POLLRDNORM))) 4434 revents = 0; 4435 else if (!TAILQ_EMPTY(&so->sol_comp)) 4436 revents = events & (POLLIN | POLLRDNORM); 4437 else if ((events & POLLINIGNEOF) == 0 && so->so_error) 4438 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 4439 else { 4440 selrecord(td, &so->so_rdsel); 4441 revents = 0; 4442 } 4443 } else { 4444 revents = 0; 4445 SOCK_SENDBUF_LOCK(so); 4446 SOCK_RECVBUF_LOCK(so); 4447 if (events & (POLLIN | POLLRDNORM)) 4448 if (soreadabledata(so) && !isspliced(so)) 4449 revents |= events & (POLLIN | POLLRDNORM); 4450 if (events & (POLLOUT | POLLWRNORM)) 4451 if (sowriteable(so) && !issplicedback(so)) 4452 revents |= events & (POLLOUT | POLLWRNORM); 4453 if (events & (POLLPRI | POLLRDBAND)) 4454 if (so->so_oobmark || 4455 (so->so_rcv.sb_state & SBS_RCVATMARK)) 4456 revents |= events & (POLLPRI | POLLRDBAND); 4457 if ((events & POLLINIGNEOF) == 0) { 4458 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 4459 revents |= events & (POLLIN | POLLRDNORM); 4460 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 4461 revents |= POLLHUP; 4462 } 4463 } 4464 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 4465 revents |= events & POLLRDHUP; 4466 if (revents == 0) { 4467 if (events & 4468 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND | POLLRDHUP)) { 4469 selrecord(td, &so->so_rdsel); 4470 so->so_rcv.sb_flags |= SB_SEL; 4471 } 4472 if (events & (POLLOUT | POLLWRNORM)) { 4473 selrecord(td, &so->so_wrsel); 4474 so->so_snd.sb_flags |= SB_SEL; 4475 } 4476 } 4477 SOCK_RECVBUF_UNLOCK(so); 4478 SOCK_SENDBUF_UNLOCK(so); 4479 } 4480 SOCK_UNLOCK(so); 4481 return (revents); 4482 } 4483 4484 int 4485 sokqfilter_generic(struct socket *so, struct knote *kn) 4486 { 4487 struct sockbuf *sb; 4488 sb_which which; 4489 struct knlist *knl; 4490 4491 switch (kn->kn_filter) { 4492 case EVFILT_READ: 4493 kn->kn_fop = &soread_filtops; 4494 knl = &so->so_rdsel.si_note; 4495 sb = &so->so_rcv; 4496 which = SO_RCV; 4497 break; 4498 case EVFILT_WRITE: 4499 kn->kn_fop = &sowrite_filtops; 4500 knl = &so->so_wrsel.si_note; 4501 sb = &so->so_snd; 4502 which = SO_SND; 4503 break; 4504 case EVFILT_EMPTY: 4505 kn->kn_fop = &soempty_filtops; 4506 knl = &so->so_wrsel.si_note; 4507 sb = &so->so_snd; 4508 which = SO_SND; 4509 break; 4510 default: 4511 return (EINVAL); 4512 } 4513 4514 SOCK_LOCK(so); 4515 if (SOLISTENING(so)) { 4516 knlist_add(knl, kn, 1); 4517 } else { 4518 SOCK_BUF_LOCK(so, which); 4519 knlist_add(knl, kn, 1); 4520 sb->sb_flags |= SB_KNOTE; 4521 if ((kn->kn_sfflags & NOTE_LOWAT) && 4522 (sb->sb_flags & SB_AUTOLOWAT)) 4523 sb->sb_flags &= ~SB_AUTOLOWAT; 4524 SOCK_BUF_UNLOCK(so, which); 4525 } 4526 SOCK_UNLOCK(so); 4527 return (0); 4528 } 4529 4530 static void 4531 filt_sordetach(struct knote *kn) 4532 { 4533 struct socket *so = kn->kn_fp->f_data; 4534 4535 so_rdknl_lock(so); 4536 knlist_remove(&so->so_rdsel.si_note, kn, 1); 4537 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note)) 4538 so->so_rcv.sb_flags &= ~SB_KNOTE; 4539 so_rdknl_unlock(so); 4540 } 4541 4542 /*ARGSUSED*/ 4543 static int 4544 filt_soread(struct knote *kn, long hint) 4545 { 4546 struct socket *so; 4547 4548 so = kn->kn_fp->f_data; 4549 4550 if (SOLISTENING(so)) { 4551 SOCK_LOCK_ASSERT(so); 4552 kn->kn_data = so->sol_qlen; 4553 if (so->so_error) { 4554 kn->kn_flags |= EV_EOF; 4555 kn->kn_fflags = so->so_error; 4556 return (1); 4557 } 4558 return (!TAILQ_EMPTY(&so->sol_comp)); 4559 } 4560 4561 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) 4562 return (0); 4563 4564 SOCK_RECVBUF_LOCK_ASSERT(so); 4565 4566 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl; 4567 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 4568 kn->kn_flags |= EV_EOF; 4569 kn->kn_fflags = so->so_error; 4570 return (1); 4571 } else if (so->so_error || so->so_rerror) 4572 return (1); 4573 4574 if (kn->kn_sfflags & NOTE_LOWAT) { 4575 if (kn->kn_data >= kn->kn_sdata) 4576 return (1); 4577 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat) 4578 return (1); 4579 4580 #ifdef SOCKET_HHOOK 4581 /* This hook returning non-zero indicates an event, not error */ 4582 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD)); 4583 #else 4584 return (0); 4585 #endif 4586 } 4587 4588 static void 4589 filt_sowdetach(struct knote *kn) 4590 { 4591 struct socket *so = kn->kn_fp->f_data; 4592 4593 so_wrknl_lock(so); 4594 knlist_remove(&so->so_wrsel.si_note, kn, 1); 4595 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note)) 4596 so->so_snd.sb_flags &= ~SB_KNOTE; 4597 so_wrknl_unlock(so); 4598 } 4599 4600 /*ARGSUSED*/ 4601 static int 4602 filt_sowrite(struct knote *kn, long hint) 4603 { 4604 struct socket *so; 4605 4606 so = kn->kn_fp->f_data; 4607 4608 if (SOLISTENING(so)) 4609 return (0); 4610 4611 SOCK_SENDBUF_LOCK_ASSERT(so); 4612 kn->kn_data = sbspace(&so->so_snd); 4613 4614 #ifdef SOCKET_HHOOK 4615 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE); 4616 #endif 4617 4618 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 4619 kn->kn_flags |= EV_EOF; 4620 kn->kn_fflags = so->so_error; 4621 return (1); 4622 } else if (so->so_error) /* temporary udp error */ 4623 return (1); 4624 else if (((so->so_state & SS_ISCONNECTED) == 0) && 4625 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 4626 return (0); 4627 else if (kn->kn_sfflags & NOTE_LOWAT) 4628 return (kn->kn_data >= kn->kn_sdata); 4629 else 4630 return (kn->kn_data >= so->so_snd.sb_lowat); 4631 } 4632 4633 static int 4634 filt_soempty(struct knote *kn, long hint) 4635 { 4636 struct socket *so; 4637 4638 so = kn->kn_fp->f_data; 4639 4640 if (SOLISTENING(so)) 4641 return (1); 4642 4643 SOCK_SENDBUF_LOCK_ASSERT(so); 4644 kn->kn_data = sbused(&so->so_snd); 4645 4646 if (kn->kn_data == 0) 4647 return (1); 4648 else 4649 return (0); 4650 } 4651 4652 int 4653 socheckuid(struct socket *so, uid_t uid) 4654 { 4655 4656 if (so == NULL) 4657 return (EPERM); 4658 if (so->so_cred->cr_uid != uid) 4659 return (EPERM); 4660 return (0); 4661 } 4662 4663 /* 4664 * These functions are used by protocols to notify the socket layer (and its 4665 * consumers) of state changes in the sockets driven by protocol-side events. 4666 */ 4667 4668 /* 4669 * Procedures to manipulate state flags of socket and do appropriate wakeups. 4670 * 4671 * Normal sequence from the active (originating) side is that 4672 * soisconnecting() is called during processing of connect() call, resulting 4673 * in an eventual call to soisconnected() if/when the connection is 4674 * established. When the connection is torn down soisdisconnecting() is 4675 * called during processing of disconnect() call, and soisdisconnected() is 4676 * called when the connection to the peer is totally severed. The semantics 4677 * of these routines are such that connectionless protocols can call 4678 * soisconnected() and soisdisconnected() only, bypassing the in-progress 4679 * calls when setting up a ``connection'' takes no time. 4680 * 4681 * From the passive side, a socket is created with two queues of sockets: 4682 * so_incomp for connections in progress and so_comp for connections already 4683 * made and awaiting user acceptance. As a protocol is preparing incoming 4684 * connections, it creates a socket structure queued on so_incomp by calling 4685 * sonewconn(). When the connection is established, soisconnected() is 4686 * called, and transfers the socket structure to so_comp, making it available 4687 * to accept(). 4688 * 4689 * If a socket is closed with sockets on either so_incomp or so_comp, these 4690 * sockets are dropped. 4691 * 4692 * If higher-level protocols are implemented in the kernel, the wakeups done 4693 * here will sometimes cause software-interrupt process scheduling. 4694 */ 4695 void 4696 soisconnecting(struct socket *so) 4697 { 4698 4699 SOCK_LOCK(so); 4700 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 4701 so->so_state |= SS_ISCONNECTING; 4702 SOCK_UNLOCK(so); 4703 } 4704 4705 void 4706 soisconnected(struct socket *so) 4707 { 4708 bool last __diagused; 4709 4710 SOCK_LOCK(so); 4711 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING); 4712 so->so_state |= SS_ISCONNECTED; 4713 4714 if (so->so_qstate == SQ_INCOMP) { 4715 struct socket *head = so->so_listen; 4716 int ret; 4717 4718 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so)); 4719 /* 4720 * Promoting a socket from incomplete queue to complete, we 4721 * need to go through reverse order of locking. We first do 4722 * trylock, and if that doesn't succeed, we go the hard way 4723 * leaving a reference and rechecking consistency after proper 4724 * locking. 4725 */ 4726 if (__predict_false(SOLISTEN_TRYLOCK(head) == 0)) { 4727 soref(head); 4728 SOCK_UNLOCK(so); 4729 SOLISTEN_LOCK(head); 4730 SOCK_LOCK(so); 4731 if (__predict_false(head != so->so_listen)) { 4732 /* 4733 * The socket went off the listen queue, 4734 * should be lost race to close(2) of sol. 4735 * The socket is about to soabort(). 4736 */ 4737 SOCK_UNLOCK(so); 4738 sorele_locked(head); 4739 return; 4740 } 4741 last = refcount_release(&head->so_count); 4742 KASSERT(!last, ("%s: released last reference for %p", 4743 __func__, head)); 4744 } 4745 again: 4746 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 4747 TAILQ_REMOVE(&head->sol_incomp, so, so_list); 4748 head->sol_incqlen--; 4749 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 4750 head->sol_qlen++; 4751 so->so_qstate = SQ_COMP; 4752 SOCK_UNLOCK(so); 4753 solisten_wakeup(head); /* unlocks */ 4754 } else { 4755 SOCK_RECVBUF_LOCK(so); 4756 soupcall_set(so, SO_RCV, 4757 head->sol_accept_filter->accf_callback, 4758 head->sol_accept_filter_arg); 4759 so->so_options &= ~SO_ACCEPTFILTER; 4760 ret = head->sol_accept_filter->accf_callback(so, 4761 head->sol_accept_filter_arg, M_NOWAIT); 4762 if (ret == SU_ISCONNECTED) { 4763 soupcall_clear(so, SO_RCV); 4764 SOCK_RECVBUF_UNLOCK(so); 4765 goto again; 4766 } 4767 SOCK_RECVBUF_UNLOCK(so); 4768 SOCK_UNLOCK(so); 4769 SOLISTEN_UNLOCK(head); 4770 } 4771 return; 4772 } 4773 SOCK_UNLOCK(so); 4774 wakeup(&so->so_timeo); 4775 sorwakeup(so); 4776 sowwakeup(so); 4777 } 4778 4779 void 4780 soisdisconnecting(struct socket *so) 4781 { 4782 4783 SOCK_LOCK(so); 4784 so->so_state &= ~SS_ISCONNECTING; 4785 so->so_state |= SS_ISDISCONNECTING; 4786 4787 if (!SOLISTENING(so)) { 4788 SOCK_RECVBUF_LOCK(so); 4789 socantrcvmore_locked(so); 4790 SOCK_SENDBUF_LOCK(so); 4791 socantsendmore_locked(so); 4792 } 4793 SOCK_UNLOCK(so); 4794 wakeup(&so->so_timeo); 4795 } 4796 4797 void 4798 soisdisconnected(struct socket *so) 4799 { 4800 4801 SOCK_LOCK(so); 4802 4803 /* 4804 * There is at least one reader of so_state that does not 4805 * acquire socket lock, namely soreceive_generic(). Ensure 4806 * that it never sees all flags that track connection status 4807 * cleared, by ordering the update with a barrier semantic of 4808 * our release thread fence. 4809 */ 4810 so->so_state |= SS_ISDISCONNECTED; 4811 atomic_thread_fence_rel(); 4812 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 4813 4814 if (!SOLISTENING(so)) { 4815 SOCK_UNLOCK(so); 4816 SOCK_RECVBUF_LOCK(so); 4817 socantrcvmore_locked(so); 4818 SOCK_SENDBUF_LOCK(so); 4819 sbdrop_locked(&so->so_snd, sbused(&so->so_snd)); 4820 socantsendmore_locked(so); 4821 } else 4822 SOCK_UNLOCK(so); 4823 wakeup(&so->so_timeo); 4824 } 4825 4826 int 4827 soiolock(struct socket *so, struct sx *sx, int flags) 4828 { 4829 int error; 4830 4831 KASSERT((flags & SBL_VALID) == flags, 4832 ("soiolock: invalid flags %#x", flags)); 4833 4834 if ((flags & SBL_WAIT) != 0) { 4835 if ((flags & SBL_NOINTR) != 0) { 4836 sx_xlock(sx); 4837 } else { 4838 error = sx_xlock_sig(sx); 4839 if (error != 0) 4840 return (error); 4841 } 4842 } else if (!sx_try_xlock(sx)) { 4843 return (EWOULDBLOCK); 4844 } 4845 4846 if (__predict_false(SOLISTENING(so))) { 4847 sx_xunlock(sx); 4848 return (ENOTCONN); 4849 } 4850 return (0); 4851 } 4852 4853 void 4854 soiounlock(struct sx *sx) 4855 { 4856 sx_xunlock(sx); 4857 } 4858 4859 /* 4860 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME. 4861 */ 4862 struct sockaddr * 4863 sodupsockaddr(const struct sockaddr *sa, int mflags) 4864 { 4865 struct sockaddr *sa2; 4866 4867 sa2 = malloc(sa->sa_len, M_SONAME, mflags); 4868 if (sa2) 4869 bcopy(sa, sa2, sa->sa_len); 4870 return sa2; 4871 } 4872 4873 /* 4874 * Register per-socket destructor. 4875 */ 4876 void 4877 sodtor_set(struct socket *so, so_dtor_t *func) 4878 { 4879 4880 SOCK_LOCK_ASSERT(so); 4881 so->so_dtor = func; 4882 } 4883 4884 /* 4885 * Register per-socket buffer upcalls. 4886 */ 4887 void 4888 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg) 4889 { 4890 struct sockbuf *sb; 4891 4892 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 4893 4894 switch (which) { 4895 case SO_RCV: 4896 sb = &so->so_rcv; 4897 break; 4898 case SO_SND: 4899 sb = &so->so_snd; 4900 break; 4901 } 4902 SOCK_BUF_LOCK_ASSERT(so, which); 4903 sb->sb_upcall = func; 4904 sb->sb_upcallarg = arg; 4905 sb->sb_flags |= SB_UPCALL; 4906 } 4907 4908 void 4909 soupcall_clear(struct socket *so, sb_which which) 4910 { 4911 struct sockbuf *sb; 4912 4913 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 4914 4915 switch (which) { 4916 case SO_RCV: 4917 sb = &so->so_rcv; 4918 break; 4919 case SO_SND: 4920 sb = &so->so_snd; 4921 break; 4922 } 4923 SOCK_BUF_LOCK_ASSERT(so, which); 4924 KASSERT(sb->sb_upcall != NULL, 4925 ("%s: so %p no upcall to clear", __func__, so)); 4926 sb->sb_upcall = NULL; 4927 sb->sb_upcallarg = NULL; 4928 sb->sb_flags &= ~SB_UPCALL; 4929 } 4930 4931 void 4932 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) 4933 { 4934 4935 SOLISTEN_LOCK_ASSERT(so); 4936 so->sol_upcall = func; 4937 so->sol_upcallarg = arg; 4938 } 4939 4940 static void 4941 so_rdknl_lock(void *arg) 4942 { 4943 struct socket *so = arg; 4944 4945 retry: 4946 if (SOLISTENING(so)) { 4947 SOLISTEN_LOCK(so); 4948 } else { 4949 SOCK_RECVBUF_LOCK(so); 4950 if (__predict_false(SOLISTENING(so))) { 4951 SOCK_RECVBUF_UNLOCK(so); 4952 goto retry; 4953 } 4954 } 4955 } 4956 4957 static void 4958 so_rdknl_unlock(void *arg) 4959 { 4960 struct socket *so = arg; 4961 4962 if (SOLISTENING(so)) 4963 SOLISTEN_UNLOCK(so); 4964 else 4965 SOCK_RECVBUF_UNLOCK(so); 4966 } 4967 4968 static void 4969 so_rdknl_assert_lock(void *arg, int what) 4970 { 4971 struct socket *so = arg; 4972 4973 if (what == LA_LOCKED) { 4974 if (SOLISTENING(so)) 4975 SOLISTEN_LOCK_ASSERT(so); 4976 else 4977 SOCK_RECVBUF_LOCK_ASSERT(so); 4978 } else { 4979 if (SOLISTENING(so)) 4980 SOLISTEN_UNLOCK_ASSERT(so); 4981 else 4982 SOCK_RECVBUF_UNLOCK_ASSERT(so); 4983 } 4984 } 4985 4986 static void 4987 so_wrknl_lock(void *arg) 4988 { 4989 struct socket *so = arg; 4990 4991 retry: 4992 if (SOLISTENING(so)) { 4993 SOLISTEN_LOCK(so); 4994 } else { 4995 SOCK_SENDBUF_LOCK(so); 4996 if (__predict_false(SOLISTENING(so))) { 4997 SOCK_SENDBUF_UNLOCK(so); 4998 goto retry; 4999 } 5000 } 5001 } 5002 5003 static void 5004 so_wrknl_unlock(void *arg) 5005 { 5006 struct socket *so = arg; 5007 5008 if (SOLISTENING(so)) 5009 SOLISTEN_UNLOCK(so); 5010 else 5011 SOCK_SENDBUF_UNLOCK(so); 5012 } 5013 5014 static void 5015 so_wrknl_assert_lock(void *arg, int what) 5016 { 5017 struct socket *so = arg; 5018 5019 if (what == LA_LOCKED) { 5020 if (SOLISTENING(so)) 5021 SOLISTEN_LOCK_ASSERT(so); 5022 else 5023 SOCK_SENDBUF_LOCK_ASSERT(so); 5024 } else { 5025 if (SOLISTENING(so)) 5026 SOLISTEN_UNLOCK_ASSERT(so); 5027 else 5028 SOCK_SENDBUF_UNLOCK_ASSERT(so); 5029 } 5030 } 5031 5032 /* 5033 * Create an external-format (``xsocket'') structure using the information in 5034 * the kernel-format socket structure pointed to by so. This is done to 5035 * reduce the spew of irrelevant information over this interface, to isolate 5036 * user code from changes in the kernel structure, and potentially to provide 5037 * information-hiding if we decide that some of this information should be 5038 * hidden from users. 5039 */ 5040 void 5041 sotoxsocket(struct socket *so, struct xsocket *xso) 5042 { 5043 5044 bzero(xso, sizeof(*xso)); 5045 xso->xso_len = sizeof *xso; 5046 xso->xso_so = (uintptr_t)so; 5047 xso->so_type = so->so_type; 5048 xso->so_options = so->so_options; 5049 xso->so_linger = so->so_linger; 5050 xso->so_state = so->so_state; 5051 xso->so_pcb = (uintptr_t)so->so_pcb; 5052 xso->xso_protocol = so->so_proto->pr_protocol; 5053 xso->xso_family = so->so_proto->pr_domain->dom_family; 5054 xso->so_timeo = so->so_timeo; 5055 xso->so_error = so->so_error; 5056 xso->so_uid = so->so_cred->cr_uid; 5057 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0; 5058 SOCK_LOCK(so); 5059 xso->so_fibnum = so->so_fibnum; 5060 if (SOLISTENING(so)) { 5061 xso->so_qlen = so->sol_qlen; 5062 xso->so_incqlen = so->sol_incqlen; 5063 xso->so_qlimit = so->sol_qlimit; 5064 xso->so_oobmark = 0; 5065 } else { 5066 xso->so_state |= so->so_qstate; 5067 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0; 5068 xso->so_oobmark = so->so_oobmark; 5069 sbtoxsockbuf(&so->so_snd, &xso->so_snd); 5070 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv); 5071 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0) 5072 xso->so_splice_so = (uintptr_t)so->so_splice->dst; 5073 } 5074 SOCK_UNLOCK(so); 5075 } 5076 5077 int 5078 so_options_get(const struct socket *so) 5079 { 5080 5081 return (so->so_options); 5082 } 5083 5084 void 5085 so_options_set(struct socket *so, int val) 5086 { 5087 5088 so->so_options = val; 5089 } 5090 5091 int 5092 so_error_get(const struct socket *so) 5093 { 5094 5095 return (so->so_error); 5096 } 5097 5098 void 5099 so_error_set(struct socket *so, int val) 5100 { 5101 5102 so->so_error = val; 5103 } 5104