1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1989, 1991, 1993 5 * The Regents of the University of California. All Rights Reserved. 6 * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved. 7 * Copyright (c) 2018 Matthew Macy 8 * Copyright (c) 2022-2025 Gleb Smirnoff <glebius@FreeBSD.org> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * UNIX Domain (Local) Sockets 37 * 38 * This is an implementation of UNIX (local) domain sockets. Each socket has 39 * an associated struct unpcb (UNIX protocol control block). Stream sockets 40 * may be connected to 0 or 1 other socket. Datagram sockets may be 41 * connected to 0, 1, or many other sockets. Sockets may be created and 42 * connected in pairs (socketpair(2)), or bound/connected to using the file 43 * system name space. For most purposes, only the receive socket buffer is 44 * used, as sending on one socket delivers directly to the receive socket 45 * buffer of a second socket. 46 * 47 * The implementation is substantially complicated by the fact that 48 * "ancillary data", such as file descriptors or credentials, may be passed 49 * across UNIX domain sockets. The potential for passing UNIX domain sockets 50 * over other UNIX domain sockets requires the implementation of a simple 51 * garbage collector to find and tear down cycles of disconnected sockets. 52 * 53 * TODO: 54 * RDM 55 * rethink name space problems 56 * need a proper out-of-band 57 */ 58 59 #include "opt_ddb.h" 60 61 #include <sys/param.h> 62 #include <sys/capsicum.h> 63 #include <sys/domain.h> 64 #include <sys/eventhandler.h> 65 #include <sys/fcntl.h> 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/jail.h> 69 #include <sys/kernel.h> 70 #include <sys/lock.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/mount.h> 74 #include <sys/mutex.h> 75 #include <sys/namei.h> 76 #include <sys/poll.h> 77 #include <sys/proc.h> 78 #include <sys/protosw.h> 79 #include <sys/queue.h> 80 #include <sys/resourcevar.h> 81 #include <sys/rwlock.h> 82 #include <sys/socket.h> 83 #include <sys/socketvar.h> 84 #include <sys/signalvar.h> 85 #include <sys/stat.h> 86 #include <sys/sysent.h> 87 #include <sys/sx.h> 88 #include <sys/sysctl.h> 89 #include <sys/systm.h> 90 #include <sys/taskqueue.h> 91 #include <sys/un.h> 92 #include <sys/unpcb.h> 93 #include <sys/vnode.h> 94 95 #include <net/vnet.h> 96 97 #ifdef DDB 98 #include <ddb/ddb.h> 99 #endif 100 101 #include <security/mac/mac_framework.h> 102 103 #include <vm/uma.h> 104 105 MALLOC_DECLARE(M_FILECAPS); 106 107 static struct domain localdomain; 108 109 static uma_zone_t unp_zone; 110 static unp_gen_t unp_gencnt; /* (l) */ 111 static u_int unp_count; /* (l) Count of local sockets. */ 112 static ino_t unp_ino; /* Prototype for fake inode numbers. */ 113 static int unp_rights; /* (g) File descriptors in flight. */ 114 static struct unp_head unp_shead; /* (l) List of stream sockets. */ 115 static struct unp_head unp_dhead; /* (l) List of datagram sockets. */ 116 static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */ 117 static struct mtx_pool *unp_vp_mtxpool; 118 119 struct unp_defer { 120 SLIST_ENTRY(unp_defer) ud_link; 121 struct file *ud_fp; 122 }; 123 static SLIST_HEAD(, unp_defer) unp_defers; 124 static int unp_defers_count; 125 126 static const struct sockaddr sun_noname = { 127 .sa_len = sizeof(sun_noname), 128 .sa_family = AF_LOCAL, 129 }; 130 131 /* 132 * Garbage collection of cyclic file descriptor/socket references occurs 133 * asynchronously in a taskqueue context in order to avoid recursion and 134 * reentrance in the UNIX domain socket, file descriptor, and socket layer 135 * code. See unp_gc() for a full description. 136 */ 137 static struct timeout_task unp_gc_task; 138 139 /* 140 * The close of unix domain sockets attached as SCM_RIGHTS is 141 * postponed to the taskqueue, to avoid arbitrary recursion depth. 142 * The attached sockets might have another sockets attached. 143 */ 144 static struct task unp_defer_task; 145 146 /* 147 * SOCK_STREAM and SOCK_SEQPACKET unix(4) sockets fully bypass the send buffer, 148 * however the notion of send buffer still makes sense with them. Its size is 149 * the amount of space that a send(2) syscall may copyin(9) before checking 150 * with the receive buffer of a peer. Although not linked anywhere yet, 151 * pointed to by a stack variable, effectively it is a buffer that needs to be 152 * sized. 153 * 154 * SOCK_DGRAM sockets really use the sendspace as the maximum datagram size, 155 * and don't really want to reserve the sendspace. Their recvspace should be 156 * large enough for at least one max-size datagram plus address. 157 */ 158 static u_long unpst_sendspace = 64*1024; 159 static u_long unpst_recvspace = 64*1024; 160 static u_long unpdg_maxdgram = 8*1024; /* support 8KB syslog msgs */ 161 static u_long unpdg_recvspace = 16*1024; 162 static u_long unpsp_sendspace = 64*1024; 163 static u_long unpsp_recvspace = 64*1024; 164 165 static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 166 "Local domain"); 167 static SYSCTL_NODE(_net_local, SOCK_STREAM, stream, 168 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 169 "SOCK_STREAM"); 170 static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, 171 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 172 "SOCK_DGRAM"); 173 static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket, 174 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 175 "SOCK_SEQPACKET"); 176 177 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW, 178 &unpst_sendspace, 0, "Default stream send space."); 179 SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW, 180 &unpst_recvspace, 0, "Default stream receive space."); 181 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW, 182 &unpdg_maxdgram, 0, "Maximum datagram size."); 183 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW, 184 &unpdg_recvspace, 0, "Default datagram receive space."); 185 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW, 186 &unpsp_sendspace, 0, "Default seqpacket send space."); 187 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW, 188 &unpsp_recvspace, 0, "Default seqpacket receive space."); 189 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, 190 "File descriptors in flight."); 191 SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, 192 &unp_defers_count, 0, 193 "File descriptors deferred to taskqueue for close."); 194 195 /* 196 * Locking and synchronization: 197 * 198 * Several types of locks exist in the local domain socket implementation: 199 * - a global linkage lock 200 * - a global connection list lock 201 * - the mtxpool lock 202 * - per-unpcb mutexes 203 * 204 * The linkage lock protects the global socket lists, the generation number 205 * counter and garbage collector state. 206 * 207 * The connection list lock protects the list of referring sockets in a datagram 208 * socket PCB. This lock is also overloaded to protect a global list of 209 * sockets whose buffers contain socket references in the form of SCM_RIGHTS 210 * messages. To avoid recursion, such references are released by a dedicated 211 * thread. 212 * 213 * The mtxpool lock protects the vnode from being modified while referenced. 214 * Lock ordering rules require that it be acquired before any PCB locks. 215 * 216 * The unpcb lock (unp_mtx) protects the most commonly referenced fields in the 217 * unpcb. This includes the unp_conn field, which either links two connected 218 * PCBs together (for connected socket types) or points at the destination 219 * socket (for connectionless socket types). The operations of creating or 220 * destroying a connection therefore involve locking multiple PCBs. To avoid 221 * lock order reversals, in some cases this involves dropping a PCB lock and 222 * using a reference counter to maintain liveness. 223 * 224 * UNIX domain sockets each have an unpcb hung off of their so_pcb pointer, 225 * allocated in pr_attach() and freed in pr_detach(). The validity of that 226 * pointer is an invariant, so no lock is required to dereference the so_pcb 227 * pointer if a valid socket reference is held by the caller. In practice, 228 * this is always true during operations performed on a socket. Each unpcb 229 * has a back-pointer to its socket, unp_socket, which will be stable under 230 * the same circumstances. 231 * 232 * This pointer may only be safely dereferenced as long as a valid reference 233 * to the unpcb is held. Typically, this reference will be from the socket, 234 * or from another unpcb when the referring unpcb's lock is held (in order 235 * that the reference not be invalidated during use). For example, to follow 236 * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee 237 * that detach is not run clearing unp_socket. 238 * 239 * Blocking with UNIX domain sockets is a tricky issue: unlike most network 240 * protocols, bind() is a non-atomic operation, and connect() requires 241 * potential sleeping in the protocol, due to potentially waiting on local or 242 * distributed file systems. We try to separate "lookup" operations, which 243 * may sleep, and the IPC operations themselves, which typically can occur 244 * with relative atomicity as locks can be held over the entire operation. 245 * 246 * Another tricky issue is simultaneous multi-threaded or multi-process 247 * access to a single UNIX domain socket. These are handled by the flags 248 * UNP_CONNECTING and UNP_BINDING, which prevent concurrent connecting or 249 * binding, both of which involve dropping UNIX domain socket locks in order 250 * to perform namei() and other file system operations. 251 */ 252 static struct rwlock unp_link_rwlock; 253 static struct mtx unp_defers_lock; 254 255 #define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \ 256 "unp_link_rwlock") 257 258 #define UNP_LINK_LOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 259 RA_LOCKED) 260 #define UNP_LINK_UNLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 261 RA_UNLOCKED) 262 263 #define UNP_LINK_RLOCK() rw_rlock(&unp_link_rwlock) 264 #define UNP_LINK_RUNLOCK() rw_runlock(&unp_link_rwlock) 265 #define UNP_LINK_WLOCK() rw_wlock(&unp_link_rwlock) 266 #define UNP_LINK_WUNLOCK() rw_wunlock(&unp_link_rwlock) 267 #define UNP_LINK_WLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 268 RA_WLOCKED) 269 #define UNP_LINK_WOWNED() rw_wowned(&unp_link_rwlock) 270 271 #define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \ 272 "unp_defer", NULL, MTX_DEF) 273 #define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock) 274 #define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock) 275 276 #define UNP_REF_LIST_LOCK() UNP_DEFERRED_LOCK(); 277 #define UNP_REF_LIST_UNLOCK() UNP_DEFERRED_UNLOCK(); 278 279 #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \ 280 "unp", "unp", \ 281 MTX_DUPOK|MTX_DEF) 282 #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx) 283 #define UNP_PCB_LOCKPTR(unp) (&(unp)->unp_mtx) 284 #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx) 285 #define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx) 286 #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx) 287 #define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx) 288 #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED) 289 #define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED) 290 291 static int uipc_connect2(struct socket *, struct socket *); 292 static int uipc_ctloutput(struct socket *, struct sockopt *); 293 static int unp_connect(struct socket *, struct sockaddr *, 294 struct thread *); 295 static int unp_connectat(int, struct socket *, struct sockaddr *, 296 struct thread *, bool); 297 static void unp_connect2(struct socket *, struct socket *, bool); 298 static void unp_disconnect(struct unpcb *unp, struct unpcb *unp2); 299 static void unp_dispose(struct socket *so); 300 static void unp_drop(struct unpcb *); 301 static void unp_gc(__unused void *, int); 302 static void unp_scan(struct mbuf *, void (*)(struct filedescent **, int)); 303 static void unp_discard(struct file *); 304 static void unp_freerights(struct filedescent **, int); 305 static int unp_internalize(struct mbuf *, struct mchain *, 306 struct thread *, int *); 307 static void unp_internalize_fp(struct file *); 308 static int unp_externalize(const struct socket *, struct mbuf *, 309 struct mbuf **, int); 310 static int unp_externalize_fp(struct file *); 311 static void unp_addsockcred(struct thread *, struct mchain *, int); 312 static void unp_process_defers(void * __unused, int); 313 314 static void uipc_wrknl_lock(void *); 315 static void uipc_wrknl_unlock(void *); 316 static void uipc_wrknl_assert_lock(void *, int); 317 318 static void 319 unp_pcb_hold(struct unpcb *unp) 320 { 321 u_int old __unused; 322 323 old = refcount_acquire(&unp->unp_refcount); 324 KASSERT(old > 0, ("%s: unpcb %p has no references", __func__, unp)); 325 } 326 327 static __result_use_check bool 328 unp_pcb_rele(struct unpcb *unp) 329 { 330 bool ret; 331 332 UNP_PCB_LOCK_ASSERT(unp); 333 334 if ((ret = refcount_release(&unp->unp_refcount))) { 335 UNP_PCB_UNLOCK(unp); 336 UNP_PCB_LOCK_DESTROY(unp); 337 uma_zfree(unp_zone, unp); 338 } 339 return (ret); 340 } 341 342 static void 343 unp_pcb_rele_notlast(struct unpcb *unp) 344 { 345 bool ret __unused; 346 347 ret = refcount_release(&unp->unp_refcount); 348 KASSERT(!ret, ("%s: unpcb %p has no references", __func__, unp)); 349 } 350 351 static void 352 unp_pcb_lock_pair(struct unpcb *unp, struct unpcb *unp2) 353 { 354 UNP_PCB_UNLOCK_ASSERT(unp); 355 UNP_PCB_UNLOCK_ASSERT(unp2); 356 357 if (unp == unp2) { 358 UNP_PCB_LOCK(unp); 359 } else if ((uintptr_t)unp2 > (uintptr_t)unp) { 360 UNP_PCB_LOCK(unp); 361 UNP_PCB_LOCK(unp2); 362 } else { 363 UNP_PCB_LOCK(unp2); 364 UNP_PCB_LOCK(unp); 365 } 366 } 367 368 static void 369 unp_pcb_unlock_pair(struct unpcb *unp, struct unpcb *unp2) 370 { 371 UNP_PCB_UNLOCK(unp); 372 if (unp != unp2) 373 UNP_PCB_UNLOCK(unp2); 374 } 375 376 /* 377 * Try to lock the connected peer of an already locked socket. In some cases 378 * this requires that we unlock the current socket. The pairbusy counter is 379 * used to block concurrent connection attempts while the lock is dropped. The 380 * caller must be careful to revalidate PCB state. 381 */ 382 static struct unpcb * 383 unp_pcb_lock_peer(struct unpcb *unp) 384 { 385 struct unpcb *unp2; 386 387 UNP_PCB_LOCK_ASSERT(unp); 388 unp2 = unp->unp_conn; 389 if (unp2 == NULL) 390 return (NULL); 391 if (__predict_false(unp == unp2)) 392 return (unp); 393 394 UNP_PCB_UNLOCK_ASSERT(unp2); 395 396 if (__predict_true(UNP_PCB_TRYLOCK(unp2))) 397 return (unp2); 398 if ((uintptr_t)unp2 > (uintptr_t)unp) { 399 UNP_PCB_LOCK(unp2); 400 return (unp2); 401 } 402 unp->unp_pairbusy++; 403 unp_pcb_hold(unp2); 404 UNP_PCB_UNLOCK(unp); 405 406 UNP_PCB_LOCK(unp2); 407 UNP_PCB_LOCK(unp); 408 KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL, 409 ("%s: socket %p was reconnected", __func__, unp)); 410 if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) { 411 unp->unp_flags &= ~UNP_WAITING; 412 wakeup(unp); 413 } 414 if (unp_pcb_rele(unp2)) { 415 /* unp2 is unlocked. */ 416 return (NULL); 417 } 418 if (unp->unp_conn == NULL) { 419 UNP_PCB_UNLOCK(unp2); 420 return (NULL); 421 } 422 return (unp2); 423 } 424 425 /* 426 * Try to lock peer of our socket for purposes of sending data to it. 427 */ 428 static int 429 uipc_lock_peer(struct socket *so, struct unpcb **unp2) 430 { 431 struct unpcb *unp; 432 int error; 433 434 unp = sotounpcb(so); 435 UNP_PCB_LOCK(unp); 436 *unp2 = unp_pcb_lock_peer(unp); 437 if (__predict_false(so->so_error != 0)) { 438 error = so->so_error; 439 so->so_error = 0; 440 UNP_PCB_UNLOCK(unp); 441 if (*unp2 != NULL) 442 UNP_PCB_UNLOCK(*unp2); 443 return (error); 444 } 445 if (__predict_false(*unp2 == NULL)) { 446 /* 447 * Different error code for a previously connected socket and 448 * a never connected one. The SS_ISDISCONNECTED is set in the 449 * unp_soisdisconnected() and is synchronized by the pcb lock. 450 */ 451 error = so->so_state & SS_ISDISCONNECTED ? EPIPE : ENOTCONN; 452 UNP_PCB_UNLOCK(unp); 453 return (error); 454 } 455 UNP_PCB_UNLOCK(unp); 456 457 return (0); 458 } 459 460 static void 461 uipc_abort(struct socket *so) 462 { 463 struct unpcb *unp, *unp2; 464 465 unp = sotounpcb(so); 466 KASSERT(unp != NULL, ("uipc_abort: unp == NULL")); 467 UNP_PCB_UNLOCK_ASSERT(unp); 468 469 UNP_PCB_LOCK(unp); 470 unp2 = unp->unp_conn; 471 if (unp2 != NULL) { 472 unp_pcb_hold(unp2); 473 UNP_PCB_UNLOCK(unp); 474 unp_drop(unp2); 475 } else 476 UNP_PCB_UNLOCK(unp); 477 } 478 479 static int 480 uipc_attach(struct socket *so, int proto, struct thread *td) 481 { 482 u_long sendspace, recvspace; 483 struct unpcb *unp; 484 int error, rcvmtxopts; 485 bool locked; 486 487 KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); 488 switch (so->so_type) { 489 case SOCK_DGRAM: 490 STAILQ_INIT(&so->so_rcv.uxdg_mb); 491 STAILQ_INIT(&so->so_snd.uxdg_mb); 492 TAILQ_INIT(&so->so_rcv.uxdg_conns); 493 /* 494 * Since send buffer is either bypassed or is a part 495 * of one-to-many receive buffer, we assign both space 496 * limits to unpdg_recvspace. 497 */ 498 sendspace = recvspace = unpdg_recvspace; 499 rcvmtxopts = 0; 500 break; 501 502 case SOCK_STREAM: 503 sendspace = unpst_sendspace; 504 recvspace = unpst_recvspace; 505 goto common; 506 507 case SOCK_SEQPACKET: 508 sendspace = unpsp_sendspace; 509 recvspace = unpsp_recvspace; 510 common: 511 rcvmtxopts = MTX_DUPOK; 512 knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, 513 uipc_wrknl_unlock, uipc_wrknl_assert_lock); 514 STAILQ_INIT(&so->so_rcv.uxst_mbq); 515 break; 516 default: 517 panic("uipc_attach"); 518 } 519 mtx_init(&so->so_rcv_mtx, "unix so_rcv", NULL, MTX_DEF | rcvmtxopts); 520 mtx_init(&so->so_snd_mtx, "unix so_snd", NULL, MTX_DEF); 521 error = soreserve(so, sendspace, recvspace); 522 if (error) 523 return (error); 524 unp = uma_zalloc(unp_zone, M_NOWAIT | M_ZERO); 525 if (unp == NULL) 526 return (ENOBUFS); 527 LIST_INIT(&unp->unp_refs); 528 UNP_PCB_LOCK_INIT(unp); 529 unp->unp_socket = so; 530 so->so_pcb = unp; 531 so->so_options |= SO_PASSRIGHTS; 532 refcount_init(&unp->unp_refcount, 1); 533 unp->unp_mode = ACCESSPERMS; 534 535 if ((locked = UNP_LINK_WOWNED()) == false) 536 UNP_LINK_WLOCK(); 537 538 unp->unp_gencnt = ++unp_gencnt; 539 unp->unp_ino = ++unp_ino; 540 unp_count++; 541 switch (so->so_type) { 542 case SOCK_STREAM: 543 LIST_INSERT_HEAD(&unp_shead, unp, unp_link); 544 break; 545 546 case SOCK_DGRAM: 547 LIST_INSERT_HEAD(&unp_dhead, unp, unp_link); 548 break; 549 550 case SOCK_SEQPACKET: 551 LIST_INSERT_HEAD(&unp_sphead, unp, unp_link); 552 break; 553 554 default: 555 panic("uipc_attach"); 556 } 557 558 if (locked == false) 559 UNP_LINK_WUNLOCK(); 560 561 return (0); 562 } 563 564 static int 565 uipc_bindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 566 { 567 struct sockaddr_un *soun = (struct sockaddr_un *)nam; 568 struct vattr vattr; 569 int error, namelen; 570 struct nameidata nd; 571 struct unpcb *unp; 572 struct vnode *vp; 573 struct mount *mp; 574 cap_rights_t rights; 575 char *buf; 576 mode_t mode; 577 578 if (nam->sa_family != AF_UNIX) 579 return (EAFNOSUPPORT); 580 581 unp = sotounpcb(so); 582 KASSERT(unp != NULL, ("uipc_bind: unp == NULL")); 583 584 if (soun->sun_len > sizeof(struct sockaddr_un)) 585 return (EINVAL); 586 namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); 587 if (namelen <= 0) 588 return (EINVAL); 589 590 /* 591 * We don't allow simultaneous bind() calls on a single UNIX domain 592 * socket, so flag in-progress operations, and return an error if an 593 * operation is already in progress. 594 * 595 * Historically, we have not allowed a socket to be rebound, so this 596 * also returns an error. Not allowing re-binding simplifies the 597 * implementation and avoids a great many possible failure modes. 598 */ 599 UNP_PCB_LOCK(unp); 600 if (unp->unp_vnode != NULL) { 601 UNP_PCB_UNLOCK(unp); 602 return (EINVAL); 603 } 604 if (unp->unp_flags & UNP_BINDING) { 605 UNP_PCB_UNLOCK(unp); 606 return (EALREADY); 607 } 608 unp->unp_flags |= UNP_BINDING; 609 mode = unp->unp_mode & ~td->td_proc->p_pd->pd_cmask; 610 UNP_PCB_UNLOCK(unp); 611 612 buf = malloc(namelen + 1, M_TEMP, M_WAITOK); 613 bcopy(soun->sun_path, buf, namelen); 614 buf[namelen] = 0; 615 616 restart: 617 NDINIT_ATRIGHTS(&nd, CREATE, NOFOLLOW | LOCKPARENT | NOCACHE, 618 UIO_SYSSPACE, buf, fd, cap_rights_init_one(&rights, CAP_BINDAT)); 619 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */ 620 error = namei(&nd); 621 if (error) 622 goto error; 623 vp = nd.ni_vp; 624 if (vp != NULL || vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) { 625 NDFREE_PNBUF(&nd); 626 if (nd.ni_dvp == vp) 627 vrele(nd.ni_dvp); 628 else 629 vput(nd.ni_dvp); 630 if (vp != NULL) { 631 vrele(vp); 632 error = EADDRINUSE; 633 goto error; 634 } 635 error = vn_start_write(NULL, &mp, V_XSLEEP | V_PCATCH); 636 if (error) 637 goto error; 638 goto restart; 639 } 640 VATTR_NULL(&vattr); 641 vattr.va_type = VSOCK; 642 vattr.va_mode = mode; 643 #ifdef MAC 644 error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd, 645 &vattr); 646 #endif 647 if (error == 0) { 648 /* 649 * The prior lookup may have left LK_SHARED in cn_lkflags, 650 * and VOP_CREATE technically only requires the new vnode to 651 * be locked shared. Most filesystems will return the new vnode 652 * locked exclusive regardless, but we should explicitly 653 * specify that here since we require it and assert to that 654 * effect below. 655 */ 656 nd.ni_cnd.cn_lkflags = (nd.ni_cnd.cn_lkflags & ~LK_SHARED) | 657 LK_EXCLUSIVE; 658 error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr); 659 } 660 NDFREE_PNBUF(&nd); 661 if (error) { 662 VOP_VPUT_PAIR(nd.ni_dvp, NULL, true); 663 vn_finished_write(mp); 664 if (error == ERELOOKUP) 665 goto restart; 666 goto error; 667 } 668 vp = nd.ni_vp; 669 ASSERT_VOP_ELOCKED(vp, "uipc_bind"); 670 soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK); 671 672 UNP_PCB_LOCK(unp); 673 VOP_UNP_BIND(vp, unp); 674 unp->unp_vnode = vp; 675 unp->unp_addr = soun; 676 unp->unp_flags &= ~UNP_BINDING; 677 UNP_PCB_UNLOCK(unp); 678 vref(vp); 679 VOP_VPUT_PAIR(nd.ni_dvp, &vp, true); 680 vn_finished_write(mp); 681 free(buf, M_TEMP); 682 return (0); 683 684 error: 685 UNP_PCB_LOCK(unp); 686 unp->unp_flags &= ~UNP_BINDING; 687 UNP_PCB_UNLOCK(unp); 688 free(buf, M_TEMP); 689 return (error); 690 } 691 692 static int 693 uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td) 694 { 695 696 return (uipc_bindat(AT_FDCWD, so, nam, td)); 697 } 698 699 static int 700 uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 701 { 702 int error; 703 704 KASSERT(td == curthread, ("uipc_connect: td != curthread")); 705 error = unp_connect(so, nam, td); 706 return (error); 707 } 708 709 static int 710 uipc_connectat(int fd, struct socket *so, struct sockaddr *nam, 711 struct thread *td) 712 { 713 int error; 714 715 KASSERT(td == curthread, ("uipc_connectat: td != curthread")); 716 error = unp_connectat(fd, so, nam, td, false); 717 return (error); 718 } 719 720 static void 721 uipc_close(struct socket *so) 722 { 723 struct unpcb *unp, *unp2; 724 struct vnode *vp = NULL; 725 struct mtx *vplock; 726 727 unp = sotounpcb(so); 728 KASSERT(unp != NULL, ("uipc_close: unp == NULL")); 729 730 vplock = NULL; 731 if ((vp = unp->unp_vnode) != NULL) { 732 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 733 mtx_lock(vplock); 734 } 735 UNP_PCB_LOCK(unp); 736 if (vp && unp->unp_vnode == NULL) { 737 mtx_unlock(vplock); 738 vp = NULL; 739 } 740 if (vp != NULL) { 741 VOP_UNP_DETACH(vp); 742 unp->unp_vnode = NULL; 743 } 744 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 745 unp_disconnect(unp, unp2); 746 else 747 UNP_PCB_UNLOCK(unp); 748 if (vp) { 749 mtx_unlock(vplock); 750 vrele(vp); 751 } 752 } 753 754 static int 755 uipc_chmod(struct socket *so, mode_t mode, struct ucred *cred __unused, 756 struct thread *td __unused) 757 { 758 struct unpcb *unp; 759 int error; 760 761 if ((mode & ~ACCESSPERMS) != 0) 762 return (EINVAL); 763 764 error = 0; 765 unp = sotounpcb(so); 766 UNP_PCB_LOCK(unp); 767 if (unp->unp_vnode != NULL || (unp->unp_flags & UNP_BINDING) != 0) 768 error = EINVAL; 769 else 770 unp->unp_mode = mode; 771 UNP_PCB_UNLOCK(unp); 772 return (error); 773 } 774 775 static int 776 uipc_connect2(struct socket *so1, struct socket *so2) 777 { 778 struct unpcb *unp, *unp2; 779 780 if (so1->so_type != so2->so_type) 781 return (EPROTOTYPE); 782 783 unp = so1->so_pcb; 784 KASSERT(unp != NULL, ("uipc_connect2: unp == NULL")); 785 unp2 = so2->so_pcb; 786 KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL")); 787 unp_pcb_lock_pair(unp, unp2); 788 unp_connect2(so1, so2, false); 789 unp_pcb_unlock_pair(unp, unp2); 790 791 return (0); 792 } 793 794 static void 795 maybe_schedule_gc(void) 796 { 797 if (atomic_load_int(&unp_rights) != 0) 798 taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); 799 } 800 801 static void 802 uipc_detach(struct socket *so) 803 { 804 struct unpcb *unp, *unp2; 805 struct mtx *vplock; 806 struct vnode *vp; 807 808 unp = sotounpcb(so); 809 KASSERT(unp != NULL, ("uipc_detach: unp == NULL")); 810 811 vp = NULL; 812 vplock = NULL; 813 814 if (!SOLISTENING(so)) 815 unp_dispose(so); 816 817 UNP_LINK_WLOCK(); 818 LIST_REMOVE(unp, unp_link); 819 if (unp->unp_gcflag & UNPGC_DEAD) 820 LIST_REMOVE(unp, unp_dead); 821 unp->unp_gencnt = ++unp_gencnt; 822 --unp_count; 823 UNP_LINK_WUNLOCK(); 824 825 UNP_PCB_UNLOCK_ASSERT(unp); 826 restart: 827 if ((vp = unp->unp_vnode) != NULL) { 828 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 829 mtx_lock(vplock); 830 } 831 UNP_PCB_LOCK(unp); 832 if (unp->unp_vnode != vp && unp->unp_vnode != NULL) { 833 if (vplock) 834 mtx_unlock(vplock); 835 UNP_PCB_UNLOCK(unp); 836 goto restart; 837 } 838 if ((vp = unp->unp_vnode) != NULL) { 839 VOP_UNP_DETACH(vp); 840 unp->unp_vnode = NULL; 841 } 842 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 843 unp_disconnect(unp, unp2); 844 else 845 UNP_PCB_UNLOCK(unp); 846 847 UNP_REF_LIST_LOCK(); 848 while (!LIST_EMPTY(&unp->unp_refs)) { 849 struct unpcb *ref = LIST_FIRST(&unp->unp_refs); 850 851 unp_pcb_hold(ref); 852 UNP_REF_LIST_UNLOCK(); 853 854 MPASS(ref != unp); 855 UNP_PCB_UNLOCK_ASSERT(ref); 856 unp_drop(ref); 857 UNP_REF_LIST_LOCK(); 858 } 859 UNP_REF_LIST_UNLOCK(); 860 861 UNP_PCB_LOCK(unp); 862 unp->unp_socket->so_pcb = NULL; 863 unp->unp_socket = NULL; 864 free(unp->unp_addr, M_SONAME); 865 unp->unp_addr = NULL; 866 if (!unp_pcb_rele(unp)) 867 UNP_PCB_UNLOCK(unp); 868 if (vp) { 869 mtx_unlock(vplock); 870 vrele(vp); 871 } 872 maybe_schedule_gc(); 873 874 switch (so->so_type) { 875 case SOCK_STREAM: 876 case SOCK_SEQPACKET: 877 MPASS(SOLISTENING(so) || (STAILQ_EMPTY(&so->so_rcv.uxst_mbq) && 878 so->so_rcv.uxst_peer == NULL)); 879 break; 880 case SOCK_DGRAM: 881 /* 882 * Everything should have been unlinked/freed by unp_dispose() 883 * and/or unp_disconnect(). 884 */ 885 MPASS(so->so_rcv.uxdg_peeked == NULL); 886 MPASS(STAILQ_EMPTY(&so->so_rcv.uxdg_mb)); 887 MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); 888 MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); 889 } 890 891 mtx_destroy(&so->so_snd_mtx); 892 mtx_destroy(&so->so_rcv_mtx); 893 } 894 895 static int 896 uipc_disconnect(struct socket *so) 897 { 898 struct unpcb *unp, *unp2; 899 900 unp = sotounpcb(so); 901 KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL")); 902 903 UNP_PCB_LOCK(unp); 904 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 905 unp_disconnect(unp, unp2); 906 else 907 UNP_PCB_UNLOCK(unp); 908 return (0); 909 } 910 911 static void 912 uipc_fdclose(struct socket *so __unused) 913 { 914 /* 915 * Ensure that userspace can't create orphaned file descriptors without 916 * triggering garbage collection. Triggering GC from uipc_detach() is 917 * not sufficient, since that's only closed once a socket reference 918 * count drops to zero. 919 */ 920 maybe_schedule_gc(); 921 } 922 923 static int 924 uipc_listen(struct socket *so, int backlog, struct thread *td) 925 { 926 struct unpcb *unp; 927 int error; 928 929 MPASS(so->so_type != SOCK_DGRAM); 930 931 /* 932 * Synchronize with concurrent connection attempts. 933 */ 934 error = 0; 935 unp = sotounpcb(so); 936 UNP_PCB_LOCK(unp); 937 if (unp->unp_conn != NULL || (unp->unp_flags & UNP_CONNECTING) != 0) 938 error = EINVAL; 939 else if (unp->unp_vnode == NULL) 940 error = EDESTADDRREQ; 941 if (error != 0) { 942 UNP_PCB_UNLOCK(unp); 943 return (error); 944 } 945 946 SOCK_LOCK(so); 947 error = solisten_proto_check(so); 948 if (error == 0) { 949 cru2xt(td, &unp->unp_peercred); 950 if (!SOLISTENING(so)) { 951 (void)chgsbsize(so->so_cred->cr_uidinfo, 952 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 953 (void)chgsbsize(so->so_cred->cr_uidinfo, 954 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 955 } 956 solisten_proto(so, backlog); 957 } 958 SOCK_UNLOCK(so); 959 UNP_PCB_UNLOCK(unp); 960 return (error); 961 } 962 963 static int 964 uipc_peeraddr(struct socket *so, struct sockaddr *ret) 965 { 966 struct unpcb *unp, *unp2; 967 const struct sockaddr *sa; 968 969 unp = sotounpcb(so); 970 KASSERT(unp != NULL, ("uipc_peeraddr: unp == NULL")); 971 972 UNP_PCB_LOCK(unp); 973 unp2 = unp_pcb_lock_peer(unp); 974 if (unp2 != NULL) { 975 if (unp2->unp_addr != NULL) 976 sa = (struct sockaddr *)unp2->unp_addr; 977 else 978 sa = &sun_noname; 979 bcopy(sa, ret, sa->sa_len); 980 unp_pcb_unlock_pair(unp, unp2); 981 } else { 982 UNP_PCB_UNLOCK(unp); 983 sa = &sun_noname; 984 bcopy(sa, ret, sa->sa_len); 985 } 986 return (0); 987 } 988 989 /* 990 * pr_sosend() called with mbuf instead of uio is a kernel thread. NFS, 991 * netgraph(4) and other subsystems can call into socket code. The 992 * function will condition the mbuf so that it can be safely put onto socket 993 * buffer and calculate its char count and mbuf count. 994 * 995 * Note: we don't support receiving control data from a kernel thread. Our 996 * pr_sosend methods have MPASS() to check that. This may change. 997 */ 998 static void 999 uipc_reset_kernel_mbuf(struct mbuf *m, struct mchain *mc) 1000 { 1001 1002 M_ASSERTPKTHDR(m); 1003 1004 m_clrprotoflags(m); 1005 m_tag_delete_chain(m, NULL); 1006 m->m_pkthdr.rcvif = NULL; 1007 m->m_pkthdr.flowid = 0; 1008 m->m_pkthdr.csum_flags = 0; 1009 m->m_pkthdr.fibnum = 0; 1010 m->m_pkthdr.rsstype = 0; 1011 1012 mc_init_m(mc, m); 1013 MPASS(m->m_pkthdr.len == mc->mc_len); 1014 } 1015 1016 #ifdef SOCKBUF_DEBUG 1017 static inline void 1018 uipc_stream_sbcheck(struct sockbuf *sb) 1019 { 1020 struct mbuf *d; 1021 u_int dacc, dccc, dctl, dmbcnt; 1022 bool notready = false; 1023 1024 dacc = dccc = dctl = dmbcnt = 0; 1025 STAILQ_FOREACH(d, &sb->uxst_mbq, m_stailq) { 1026 if (d == sb->uxst_fnrdy) { 1027 MPASS(d->m_flags & M_NOTREADY); 1028 notready = true; 1029 } 1030 if (d->m_type == MT_CONTROL) 1031 dctl += d->m_len; 1032 else if (d->m_type == MT_DATA) { 1033 dccc += d->m_len; 1034 if (!notready) 1035 dacc += d->m_len; 1036 } else 1037 MPASS(0); 1038 dmbcnt += MSIZE; 1039 if (d->m_flags & M_EXT) 1040 dmbcnt += d->m_ext.ext_size; 1041 if (d->m_stailq.stqe_next == NULL) 1042 MPASS(sb->uxst_mbq.stqh_last == &d->m_stailq.stqe_next); 1043 } 1044 MPASS(sb->uxst_fnrdy == NULL || notready); 1045 MPASS(dacc == sb->sb_acc); 1046 MPASS(dccc == sb->sb_ccc); 1047 MPASS(dctl == sb->sb_ctl); 1048 MPASS(dmbcnt == sb->sb_mbcnt); 1049 (void)STAILQ_EMPTY(&sb->uxst_mbq); 1050 } 1051 #define UIPC_STREAM_SBCHECK(sb) uipc_stream_sbcheck(sb) 1052 #else 1053 #define UIPC_STREAM_SBCHECK(sb) do {} while (0) 1054 #endif 1055 1056 /* 1057 * uipc_stream_sbspace() returns how much a writer can send, limited by char 1058 * count or mbuf memory use, whatever ends first. 1059 * 1060 * An obvious and legitimate reason for a socket having more data than allowed, 1061 * is lowering the limit with setsockopt(SO_RCVBUF) on already full buffer. 1062 * Also, sb_mbcnt may overcommit sb_mbmax in case if previous write observed 1063 * 'space < mbspace', but mchain allocated to hold 'space' bytes of data ended 1064 * up with 'mc_mlen > mbspace'. A typical scenario would be a full buffer with 1065 * writer trying to push in a large write, and a slow reader, that reads just 1066 * a few bytes at a time. In that case writer will keep creating new mbufs 1067 * with mc_split(). These mbufs will carry little chars, but will all point at 1068 * the same cluster, thus each adding cluster size to sb_mbcnt. This means we 1069 * will count same cluster many times potentially underutilizing socket buffer. 1070 * We aren't optimizing towards ineffective readers. Classic socket buffer had 1071 * the same "feature". 1072 */ 1073 static inline u_int 1074 uipc_stream_sbspace(struct sockbuf *sb) 1075 { 1076 u_int space, mbspace; 1077 1078 if (__predict_true(sb->sb_hiwat >= sb->sb_ccc + sb->sb_ctl)) 1079 space = sb->sb_hiwat - sb->sb_ccc - sb->sb_ctl; 1080 else 1081 return (0); 1082 if (__predict_true(sb->sb_mbmax >= sb->sb_mbcnt)) 1083 mbspace = sb->sb_mbmax - sb->sb_mbcnt; 1084 else 1085 return (0); 1086 1087 return (min(space, mbspace)); 1088 } 1089 1090 /* 1091 * UNIX version of generic sbwait() for writes. We wait on peer's receive 1092 * buffer, using our timeout. 1093 */ 1094 static int 1095 uipc_stream_sbwait(struct socket *so, sbintime_t timeo) 1096 { 1097 struct sockbuf *sb = &so->so_rcv; 1098 1099 SOCK_RECVBUF_LOCK_ASSERT(so); 1100 sb->sb_flags |= SB_WAIT; 1101 return (msleep_sbt(&sb->sb_acc, SOCK_RECVBUF_MTX(so), PSOCK | PCATCH, 1102 "sbwait", timeo, 0, 0)); 1103 } 1104 1105 static int 1106 uipc_sosend_stream_or_seqpacket(struct socket *so, struct sockaddr *addr, 1107 struct uio *uio0, struct mbuf *m, struct mbuf *c, int flags, 1108 struct thread *td) 1109 { 1110 struct unpcb *unp2; 1111 struct socket *so2; 1112 struct sockbuf *sb; 1113 struct uio *uio; 1114 struct mchain mc, cmc; 1115 size_t resid, sent; 1116 bool nonblock, eor, aio; 1117 int error, needsopts; 1118 1119 MPASS((uio0 != NULL && m == NULL) || (m != NULL && uio0 == NULL)); 1120 MPASS(m == NULL || c == NULL); 1121 1122 if (__predict_false(flags & MSG_OOB)) 1123 return (EOPNOTSUPP); 1124 1125 nonblock = (so->so_state & SS_NBIO) || 1126 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1127 eor = flags & MSG_EOR; 1128 1129 mc = MCHAIN_INITIALIZER(&mc); 1130 cmc = MCHAIN_INITIALIZER(&cmc); 1131 sent = 0; 1132 aio = false; 1133 needsopts = 0; 1134 1135 if (m == NULL) { 1136 if (c != NULL && 1137 (error = unp_internalize(c, &cmc, td, &needsopts))) 1138 goto out; 1139 /* 1140 * This function may read more data from the uio than it would 1141 * then place on socket. That would leave uio inconsistent 1142 * upon return. Normally uio is allocated on the stack of the 1143 * syscall thread and we don't care about leaving it consistent. 1144 * However, aio(9) will allocate a uio as part of job and will 1145 * use it to track progress. We detect aio(9) checking the 1146 * SB_AIO_RUNNING flag. It is safe to check it without lock 1147 * cause it is set and cleared in the same taskqueue thread. 1148 * 1149 * This check can also produce a false positive: there is 1150 * aio(9) job and also there is a syscall we are serving now. 1151 * No sane software does that, it would leave to a mess in 1152 * the socket buffer, as aio(9) doesn't grab the I/O sx(9). 1153 * But syzkaller can create this mess. For such false positive 1154 * our goal is just don't panic or leak memory. 1155 */ 1156 if (__predict_false(so->so_snd.sb_flags & SB_AIO_RUNNING)) { 1157 uio = cloneuio(uio0); 1158 aio = true; 1159 } else { 1160 uio = uio0; 1161 resid = uio->uio_resid; 1162 } 1163 /* 1164 * Optimization for a case when our send fits into the receive 1165 * buffer - do the copyin before taking any locks, sized to our 1166 * send buffer. Later copyins will also take into account 1167 * space in the peer's receive buffer. 1168 */ 1169 error = mc_uiotomc(&mc, uio, so->so_snd.sb_hiwat, 0, M_WAITOK, 1170 eor ? M_EOR : 0); 1171 if (__predict_false(error)) 1172 goto out2; 1173 } else 1174 uipc_reset_kernel_mbuf(m, &mc); 1175 1176 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 1177 if (error) 1178 goto out2; 1179 1180 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 1181 goto out3; 1182 1183 /* Check for SO_PASS* flags */ 1184 so2 = unp2->unp_socket; 1185 if ((atomic_load_int(&so2->so_options) & needsopts) != needsopts) { 1186 error = EPERM; 1187 UNP_PCB_UNLOCK(unp2); 1188 goto out3; 1189 } 1190 1191 if (unp2->unp_flags & UNP_WANTCRED_MASK) { 1192 /* 1193 * Credentials are passed only once on SOCK_STREAM and 1194 * SOCK_SEQPACKET (LOCAL_CREDS => WANTCRED_ONESHOT), or 1195 * forever (LOCAL_CREDS_PERSISTENT => WANTCRED_ALWAYS). 1196 */ 1197 unp_addsockcred(td, &cmc, unp2->unp_flags); 1198 unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT; 1199 } 1200 1201 /* 1202 * Cycle through the data to send and available space in the peer's 1203 * receive buffer. Put a reference on the peer socket, so that it 1204 * doesn't get freed while we sbwait(). If peer goes away, we will 1205 * observe the SBS_CANTRCVMORE and our sorele() will finalize peer's 1206 * socket destruction. 1207 */ 1208 soref(so2); 1209 UNP_PCB_UNLOCK(unp2); 1210 sb = &so2->so_rcv; 1211 while (mc.mc_len + cmc.mc_len > 0) { 1212 struct mchain mcnext = MCHAIN_INITIALIZER(&mcnext); 1213 u_int space; 1214 1215 SOCK_RECVBUF_LOCK(so2); 1216 restart: 1217 UIPC_STREAM_SBCHECK(sb); 1218 if (__predict_false(cmc.mc_len > sb->sb_hiwat)) { 1219 SOCK_RECVBUF_UNLOCK(so2); 1220 error = EMSGSIZE; 1221 goto out4; 1222 } 1223 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1224 SOCK_RECVBUF_UNLOCK(so2); 1225 error = EPIPE; 1226 goto out4; 1227 } 1228 /* 1229 * Wait on the peer socket receive buffer until we have enough 1230 * space to put at least control. The data is a stream and can 1231 * be put partially, but control is really a datagram. 1232 */ 1233 space = uipc_stream_sbspace(sb); 1234 if (space < sb->sb_lowat || space < cmc.mc_len) { 1235 if (nonblock) { 1236 if (aio) 1237 sb->uxst_flags |= UXST_PEER_AIO; 1238 SOCK_RECVBUF_UNLOCK(so2); 1239 if (aio) { 1240 SOCK_SENDBUF_LOCK(so); 1241 so->so_snd.sb_ccc = 1242 so->so_snd.sb_hiwat - space; 1243 SOCK_SENDBUF_UNLOCK(so); 1244 } 1245 error = EWOULDBLOCK; 1246 goto out4; 1247 } 1248 if ((error = uipc_stream_sbwait(so2, 1249 so->so_snd.sb_timeo)) != 0) { 1250 SOCK_RECVBUF_UNLOCK(so2); 1251 goto out4; 1252 } else 1253 goto restart; 1254 } 1255 MPASS(space >= cmc.mc_len); 1256 space -= cmc.mc_len; 1257 if (space == 0) { 1258 /* There is space only to send control. */ 1259 MPASS(!STAILQ_EMPTY(&cmc.mc_q)); 1260 mcnext = mc; 1261 mc = MCHAIN_INITIALIZER(&mc); 1262 } else if (space < mc.mc_len) { 1263 /* Not enough space. */ 1264 if (__predict_false(mc_split(&mc, &mcnext, space, 1265 M_NOWAIT) == ENOMEM)) { 1266 /* 1267 * If allocation failed use M_WAITOK and merge 1268 * the chain back. Next time mc_split() will 1269 * easily split at the same place. Only if we 1270 * race with setsockopt(SO_RCVBUF) shrinking 1271 * sb_hiwat can this happen more than once. 1272 */ 1273 SOCK_RECVBUF_UNLOCK(so2); 1274 (void)mc_split(&mc, &mcnext, space, M_WAITOK); 1275 mc_concat(&mc, &mcnext); 1276 SOCK_RECVBUF_LOCK(so2); 1277 goto restart; 1278 } 1279 MPASS(mc.mc_len == space); 1280 } 1281 if (!STAILQ_EMPTY(&cmc.mc_q)) { 1282 STAILQ_CONCAT(&sb->uxst_mbq, &cmc.mc_q); 1283 sb->sb_ctl += cmc.mc_len; 1284 sb->sb_mbcnt += cmc.mc_mlen; 1285 cmc.mc_len = 0; 1286 } 1287 sent += mc.mc_len; 1288 if (sb->uxst_fnrdy == NULL) 1289 sb->sb_acc += mc.mc_len; 1290 sb->sb_ccc += mc.mc_len; 1291 sb->sb_mbcnt += mc.mc_mlen; 1292 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 1293 UIPC_STREAM_SBCHECK(sb); 1294 space = uipc_stream_sbspace(sb); 1295 sorwakeup_locked(so2); 1296 if (!STAILQ_EMPTY(&mcnext.mc_q)) { 1297 /* 1298 * Such assignment is unsafe in general, but it is 1299 * safe with !STAILQ_EMPTY(&mcnext.mc_q). In C++ we 1300 * could reload = for STAILQs :) 1301 */ 1302 mc = mcnext; 1303 } else if (uio != NULL && uio->uio_resid > 0) { 1304 /* 1305 * Copyin sum of peer's receive buffer space and our 1306 * sb_hiwat, which is our virtual send buffer size. 1307 * See comment above unpst_sendspace declaration. 1308 * We are reading sb_hiwat locklessly, cause a) we 1309 * don't care about an application that does send(2) 1310 * and setsockopt(2) racing internally, and for an 1311 * application that does this in sequence we will see 1312 * the correct value cause sbsetopt() uses buffer lock 1313 * and we also have already acquired it at least once. 1314 */ 1315 error = mc_uiotomc(&mc, uio, space + 1316 atomic_load_int(&so->so_snd.sb_hiwat), 0, M_WAITOK, 1317 eor ? M_EOR : 0); 1318 if (__predict_false(error)) 1319 goto out4; 1320 } else 1321 mc = MCHAIN_INITIALIZER(&mc); 1322 } 1323 1324 MPASS(STAILQ_EMPTY(&mc.mc_q)); 1325 1326 td->td_ru.ru_msgsnd++; 1327 out4: 1328 sorele(so2); 1329 out3: 1330 SOCK_IO_SEND_UNLOCK(so); 1331 out2: 1332 if (aio) { 1333 freeuio(uio); 1334 uioadvance(uio0, sent); 1335 } else if (uio != NULL) 1336 uio->uio_resid = resid - sent; 1337 if (!mc_empty(&cmc)) 1338 unp_scan(mc_first(&cmc), unp_freerights); 1339 out: 1340 mc_freem(&mc); 1341 mc_freem(&cmc); 1342 1343 return (error); 1344 } 1345 1346 /* 1347 * Wakeup a writer, used by recv(2) and shutdown(2). 1348 * 1349 * @param so Points to a connected stream socket with receive buffer locked 1350 * 1351 * In a blocking mode peer is sleeping on our receive buffer, and we need just 1352 * wakeup(9) on it. But to wake up various event engines, we need to reach 1353 * over to peer's selinfo. This can be safely done as the socket buffer 1354 * receive lock is protecting us from the peer going away. 1355 */ 1356 static void 1357 uipc_wakeup_writer(struct socket *so) 1358 { 1359 struct sockbuf *sb = &so->so_rcv; 1360 struct selinfo *sel; 1361 1362 SOCK_RECVBUF_LOCK_ASSERT(so); 1363 MPASS(sb->uxst_peer != NULL); 1364 1365 sel = &sb->uxst_peer->so_wrsel; 1366 1367 if (sb->uxst_flags & UXST_PEER_SEL) { 1368 selwakeuppri(sel, PSOCK); 1369 /* 1370 * XXXGL: sowakeup() does SEL_WAITING() without locks. 1371 */ 1372 if (!SEL_WAITING(sel)) 1373 sb->uxst_flags &= ~UXST_PEER_SEL; 1374 } 1375 if (sb->sb_flags & SB_WAIT) { 1376 sb->sb_flags &= ~SB_WAIT; 1377 wakeup(&sb->sb_acc); 1378 } 1379 KNOTE_LOCKED(&sel->si_note, 0); 1380 SOCK_RECVBUF_UNLOCK(so); 1381 } 1382 1383 static void 1384 uipc_cantrcvmore(struct socket *so) 1385 { 1386 1387 SOCK_RECVBUF_LOCK(so); 1388 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 1389 selwakeuppri(&so->so_rdsel, PSOCK); 1390 KNOTE_LOCKED(&so->so_rdsel.si_note, 0); 1391 if (so->so_rcv.uxst_peer != NULL) 1392 uipc_wakeup_writer(so); 1393 else 1394 SOCK_RECVBUF_UNLOCK(so); 1395 } 1396 1397 static int 1398 uipc_soreceive_stream_or_seqpacket(struct socket *so, struct sockaddr **psa, 1399 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1400 { 1401 struct sockbuf *sb = &so->so_rcv; 1402 struct mbuf *control, *m, *first, *part, *next; 1403 u_int ctl, space, datalen, mbcnt, partlen; 1404 int error, flags; 1405 bool nonblock, waitall, peek; 1406 1407 MPASS(mp0 == NULL); 1408 1409 if (psa != NULL) 1410 *psa = NULL; 1411 if (controlp != NULL) 1412 *controlp = NULL; 1413 1414 flags = flagsp != NULL ? *flagsp : 0; 1415 nonblock = (so->so_state & SS_NBIO) || 1416 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1417 peek = flags & MSG_PEEK; 1418 waitall = (flags & MSG_WAITALL) && !peek; 1419 1420 /* 1421 * This check may fail only on a socket that never went through 1422 * connect(2). We can check this locklessly, cause: a) for a new born 1423 * socket we don't care about applications that may race internally 1424 * between connect(2) and recv(2), and b) for a dying socket if we 1425 * miss update by unp_sosidisconnected(), we would still get the check 1426 * correct. For dying socket we would observe SBS_CANTRCVMORE later. 1427 */ 1428 if (__predict_false((atomic_load_short(&so->so_state) & 1429 (SS_ISCONNECTED|SS_ISDISCONNECTED)) == 0)) 1430 return (ENOTCONN); 1431 1432 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 1433 if (__predict_false(error)) 1434 return (error); 1435 1436 restart: 1437 SOCK_RECVBUF_LOCK(so); 1438 UIPC_STREAM_SBCHECK(sb); 1439 while (sb->sb_acc < sb->sb_lowat && 1440 (sb->sb_ctl == 0 || controlp == NULL)) { 1441 if (so->so_error) { 1442 error = so->so_error; 1443 if (!peek) 1444 so->so_error = 0; 1445 SOCK_RECVBUF_UNLOCK(so); 1446 SOCK_IO_RECV_UNLOCK(so); 1447 return (error); 1448 } 1449 if (sb->sb_state & SBS_CANTRCVMORE) { 1450 SOCK_RECVBUF_UNLOCK(so); 1451 SOCK_IO_RECV_UNLOCK(so); 1452 return (0); 1453 } 1454 if (nonblock) { 1455 SOCK_RECVBUF_UNLOCK(so); 1456 SOCK_IO_RECV_UNLOCK(so); 1457 return (EWOULDBLOCK); 1458 } 1459 error = sbwait(so, SO_RCV); 1460 if (error) { 1461 SOCK_RECVBUF_UNLOCK(so); 1462 SOCK_IO_RECV_UNLOCK(so); 1463 return (error); 1464 } 1465 } 1466 1467 MPASS(STAILQ_FIRST(&sb->uxst_mbq)); 1468 MPASS(sb->sb_acc > 0 || sb->sb_ctl > 0); 1469 1470 mbcnt = 0; 1471 ctl = 0; 1472 first = STAILQ_FIRST(&sb->uxst_mbq); 1473 if (first->m_type == MT_CONTROL) { 1474 control = first; 1475 STAILQ_FOREACH_FROM(first, &sb->uxst_mbq, m_stailq) { 1476 if (first->m_type != MT_CONTROL) 1477 break; 1478 ctl += first->m_len; 1479 mbcnt += MSIZE; 1480 if (first->m_flags & M_EXT) 1481 mbcnt += first->m_ext.ext_size; 1482 } 1483 } else 1484 control = NULL; 1485 1486 /* 1487 * Find split point for the next copyout. On exit from the loop, 1488 * 'next' points to the new head of the buffer STAILQ and 'datalen' 1489 * contains the amount of data we will copy out at the end. The 1490 * copyout is protected by the I/O lock only, as writers can only 1491 * append to the buffer. We need to record the socket buffer state 1492 * and do all length adjustments before dropping the socket buffer lock. 1493 */ 1494 for (space = uio->uio_resid, m = next = first, part = NULL, datalen = 0; 1495 space > 0 && m != sb->uxst_fnrdy && m->m_type == MT_DATA; 1496 m = STAILQ_NEXT(m, m_stailq)) { 1497 if (space >= m->m_len) { 1498 space -= m->m_len; 1499 datalen += m->m_len; 1500 mbcnt += MSIZE; 1501 if (m->m_flags & M_EXT) 1502 mbcnt += m->m_ext.ext_size; 1503 if (m->m_flags & M_EOR) { 1504 flags |= MSG_EOR; 1505 next = STAILQ_NEXT(m, m_stailq); 1506 break; 1507 } 1508 } else { 1509 datalen += space; 1510 partlen = space; 1511 if (!peek) { 1512 m->m_len -= partlen; 1513 m->m_data += partlen; 1514 } 1515 next = part = m; 1516 break; 1517 } 1518 next = STAILQ_NEXT(m, m_stailq); 1519 } 1520 1521 if (!peek) { 1522 if (next == NULL) 1523 STAILQ_INIT(&sb->uxst_mbq); 1524 else 1525 STAILQ_FIRST(&sb->uxst_mbq) = next; 1526 MPASS(sb->sb_acc >= datalen); 1527 sb->sb_acc -= datalen; 1528 sb->sb_ccc -= datalen; 1529 MPASS(sb->sb_ctl >= ctl); 1530 sb->sb_ctl -= ctl; 1531 MPASS(sb->sb_mbcnt >= mbcnt); 1532 sb->sb_mbcnt -= mbcnt; 1533 UIPC_STREAM_SBCHECK(sb); 1534 if (__predict_true(sb->uxst_peer != NULL)) { 1535 struct unpcb *unp2; 1536 bool aio; 1537 1538 if ((aio = sb->uxst_flags & UXST_PEER_AIO)) 1539 sb->uxst_flags &= ~UXST_PEER_AIO; 1540 1541 uipc_wakeup_writer(so); 1542 /* 1543 * XXXGL: need to go through uipc_lock_peer() after 1544 * the receive buffer lock dropped, it was protecting 1545 * us from unp_soisdisconnected(). The aio workarounds 1546 * should be refactored to the aio(4) side. 1547 */ 1548 if (aio && uipc_lock_peer(so, &unp2) == 0) { 1549 struct socket *so2 = unp2->unp_socket; 1550 1551 SOCK_SENDBUF_LOCK(so2); 1552 so2->so_snd.sb_ccc -= datalen; 1553 sowakeup_aio(so2, SO_SND); 1554 SOCK_SENDBUF_UNLOCK(so2); 1555 UNP_PCB_UNLOCK(unp2); 1556 } 1557 } else 1558 SOCK_RECVBUF_UNLOCK(so); 1559 } else 1560 SOCK_RECVBUF_UNLOCK(so); 1561 1562 while (control != NULL && control->m_type == MT_CONTROL) { 1563 if (!peek) { 1564 /* 1565 * unp_externalize() failure must abort entire read(2). 1566 * Such failure should also free the problematic 1567 * control, but link back the remaining data to the head 1568 * of the buffer, so that socket is not left in a state 1569 * where it can't progress forward with reading. 1570 * Probability of such a failure is really low, so it 1571 * is fine that we need to perform pretty complex 1572 * operation here to reconstruct the buffer. 1573 */ 1574 error = unp_externalize(so, control, controlp, flags); 1575 control = m_free(control); 1576 if (__predict_false(error && control != NULL)) { 1577 struct mchain cmc; 1578 1579 mc_init_m(&cmc, control); 1580 1581 SOCK_RECVBUF_LOCK(so); 1582 if (__predict_false( 1583 (sb->sb_state & SBS_CANTRCVMORE) || 1584 cmc.mc_len + sb->sb_ccc + sb->sb_ctl > 1585 sb->sb_hiwat)) { 1586 /* 1587 * While the lock was dropped and we 1588 * were failing in unp_externalize(), 1589 * the peer could has a) disconnected, 1590 * b) filled the buffer so that we 1591 * can't prepend data back. 1592 * These are two edge conditions that 1593 * we just can't handle, so lose the 1594 * data and return the error. 1595 */ 1596 SOCK_RECVBUF_UNLOCK(so); 1597 SOCK_IO_RECV_UNLOCK(so); 1598 unp_scan(mc_first(&cmc), 1599 unp_freerights); 1600 mc_freem(&cmc); 1601 return (error); 1602 } 1603 1604 UIPC_STREAM_SBCHECK(sb); 1605 /* XXXGL: STAILQ_PREPEND */ 1606 STAILQ_CONCAT(&cmc.mc_q, &sb->uxst_mbq); 1607 STAILQ_SWAP(&cmc.mc_q, &sb->uxst_mbq, mbuf); 1608 1609 sb->sb_ctl = sb->sb_acc = sb->sb_ccc = 1610 sb->sb_mbcnt = 0; 1611 STAILQ_FOREACH(m, &sb->uxst_mbq, m_stailq) { 1612 if (m->m_type == MT_DATA) { 1613 sb->sb_acc += m->m_len; 1614 sb->sb_ccc += m->m_len; 1615 } else { 1616 sb->sb_ctl += m->m_len; 1617 } 1618 sb->sb_mbcnt += MSIZE; 1619 if (m->m_flags & M_EXT) 1620 sb->sb_mbcnt += 1621 m->m_ext.ext_size; 1622 } 1623 UIPC_STREAM_SBCHECK(sb); 1624 SOCK_RECVBUF_UNLOCK(so); 1625 SOCK_IO_RECV_UNLOCK(so); 1626 return (error); 1627 } 1628 if (controlp != NULL) { 1629 while (*controlp != NULL) 1630 controlp = &(*controlp)->m_next; 1631 } 1632 } else { 1633 /* 1634 * XXXGL 1635 * 1636 * In MSG_PEEK case control is not externalized. This 1637 * means we are leaking some kernel pointers to the 1638 * userland. They are useless to a law-abiding 1639 * application, but may be useful to a malware. This 1640 * is what the historical implementation in the 1641 * soreceive_generic() did. To be improved? 1642 */ 1643 if (controlp != NULL) { 1644 *controlp = m_copym(control, 0, control->m_len, 1645 M_WAITOK); 1646 controlp = &(*controlp)->m_next; 1647 } 1648 control = STAILQ_NEXT(control, m_stailq); 1649 } 1650 } 1651 1652 for (m = first; datalen > 0; m = next) { 1653 void *data; 1654 u_int len; 1655 1656 next = STAILQ_NEXT(m, m_stailq); 1657 if (m == part) { 1658 data = peek ? 1659 mtod(m, char *) : mtod(m, char *) - partlen; 1660 len = partlen; 1661 } else { 1662 data = mtod(m, char *); 1663 len = m->m_len; 1664 } 1665 error = uiomove(data, len, uio); 1666 if (__predict_false(error)) { 1667 if (!peek) 1668 for (; m != part && datalen > 0; m = next) { 1669 next = STAILQ_NEXT(m, m_stailq); 1670 MPASS(datalen >= m->m_len); 1671 datalen -= m->m_len; 1672 m_free(m); 1673 } 1674 SOCK_IO_RECV_UNLOCK(so); 1675 return (error); 1676 } 1677 datalen -= len; 1678 if (!peek && m != part) 1679 m_free(m); 1680 } 1681 if (waitall && !(flags & MSG_EOR) && uio->uio_resid > 0) 1682 goto restart; 1683 SOCK_IO_RECV_UNLOCK(so); 1684 1685 if (flagsp != NULL) 1686 *flagsp |= flags; 1687 1688 uio->uio_td->td_ru.ru_msgrcv++; 1689 1690 return (0); 1691 } 1692 1693 static int 1694 uipc_sopoll_stream_or_seqpacket(struct socket *so, int events, 1695 struct thread *td) 1696 { 1697 struct unpcb *unp = sotounpcb(so); 1698 int revents; 1699 1700 UNP_PCB_LOCK(unp); 1701 if (SOLISTENING(so)) { 1702 /* The above check is safe, since conversion to listening uses 1703 * both protocol and socket lock. 1704 */ 1705 SOCK_LOCK(so); 1706 if (!(events & (POLLIN | POLLRDNORM))) 1707 revents = 0; 1708 else if (!TAILQ_EMPTY(&so->sol_comp)) 1709 revents = events & (POLLIN | POLLRDNORM); 1710 else if (so->so_error) 1711 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 1712 else { 1713 selrecord(td, &so->so_rdsel); 1714 revents = 0; 1715 } 1716 SOCK_UNLOCK(so); 1717 } else { 1718 if (so->so_state & SS_ISDISCONNECTED) 1719 revents = POLLHUP; 1720 else 1721 revents = 0; 1722 if (events & (POLLIN | POLLRDNORM | POLLRDHUP)) { 1723 SOCK_RECVBUF_LOCK(so); 1724 if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat || 1725 so->so_error || so->so_rerror) 1726 revents |= events & (POLLIN | POLLRDNORM); 1727 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1728 revents |= events & 1729 (POLLIN | POLLRDNORM | POLLRDHUP); 1730 if (!(revents & (POLLIN | POLLRDNORM | POLLRDHUP))) { 1731 selrecord(td, &so->so_rdsel); 1732 so->so_rcv.sb_flags |= SB_SEL; 1733 } 1734 SOCK_RECVBUF_UNLOCK(so); 1735 } 1736 if (events & (POLLOUT | POLLWRNORM)) { 1737 struct socket *so2 = so->so_rcv.uxst_peer; 1738 1739 if (so2 != NULL) { 1740 struct sockbuf *sb = &so2->so_rcv; 1741 1742 SOCK_RECVBUF_LOCK(so2); 1743 if (uipc_stream_sbspace(sb) >= sb->sb_lowat) 1744 revents |= events & 1745 (POLLOUT | POLLWRNORM); 1746 if (sb->sb_state & SBS_CANTRCVMORE) 1747 revents |= POLLHUP; 1748 if (!(revents & (POLLOUT | POLLWRNORM))) { 1749 so2->so_rcv.uxst_flags |= UXST_PEER_SEL; 1750 selrecord(td, &so->so_wrsel); 1751 } 1752 SOCK_RECVBUF_UNLOCK(so2); 1753 } else 1754 selrecord(td, &so->so_wrsel); 1755 } 1756 } 1757 UNP_PCB_UNLOCK(unp); 1758 return (revents); 1759 } 1760 1761 static void 1762 uipc_wrknl_lock(void *arg) 1763 { 1764 struct socket *so = arg; 1765 struct unpcb *unp = sotounpcb(so); 1766 1767 retry: 1768 if (SOLISTENING(so)) { 1769 SOLISTEN_LOCK(so); 1770 } else { 1771 UNP_PCB_LOCK(unp); 1772 if (__predict_false(SOLISTENING(so))) { 1773 UNP_PCB_UNLOCK(unp); 1774 goto retry; 1775 } 1776 if (so->so_rcv.uxst_peer != NULL) 1777 SOCK_RECVBUF_LOCK(so->so_rcv.uxst_peer); 1778 } 1779 } 1780 1781 static void 1782 uipc_wrknl_unlock(void *arg) 1783 { 1784 struct socket *so = arg; 1785 struct unpcb *unp = sotounpcb(so); 1786 1787 if (SOLISTENING(so)) 1788 SOLISTEN_UNLOCK(so); 1789 else { 1790 if (so->so_rcv.uxst_peer != NULL) 1791 SOCK_RECVBUF_UNLOCK(so->so_rcv.uxst_peer); 1792 UNP_PCB_UNLOCK(unp); 1793 } 1794 } 1795 1796 static void 1797 uipc_wrknl_assert_lock(void *arg, int what) 1798 { 1799 struct socket *so = arg; 1800 1801 if (SOLISTENING(so)) { 1802 if (what == LA_LOCKED) 1803 SOLISTEN_LOCK_ASSERT(so); 1804 else 1805 SOLISTEN_UNLOCK_ASSERT(so); 1806 } else { 1807 /* 1808 * The pr_soreceive method will put a note without owning the 1809 * unp lock, so we can't assert it here. But we can safely 1810 * dereference uxst_peer pointer, since receive buffer lock 1811 * is assumed to be held here. 1812 */ 1813 if (what == LA_LOCKED && so->so_rcv.uxst_peer != NULL) 1814 SOCK_RECVBUF_LOCK_ASSERT(so->so_rcv.uxst_peer); 1815 } 1816 } 1817 1818 static void 1819 uipc_filt_sowdetach(struct knote *kn) 1820 { 1821 struct socket *so = kn->kn_fp->f_data; 1822 1823 uipc_wrknl_lock(so); 1824 knlist_remove(&so->so_wrsel.si_note, kn, 1); 1825 uipc_wrknl_unlock(so); 1826 } 1827 1828 static int 1829 uipc_filt_sowrite(struct knote *kn, long hint) 1830 { 1831 struct socket *so = kn->kn_fp->f_data, *so2; 1832 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1833 1834 if (SOLISTENING(so)) 1835 return (0); 1836 1837 if (unp2 == NULL) { 1838 if (so->so_state & SS_ISDISCONNECTED) { 1839 kn->kn_flags |= EV_EOF; 1840 kn->kn_fflags = so->so_error; 1841 return (1); 1842 } else 1843 return (0); 1844 } 1845 1846 so2 = unp2->unp_socket; 1847 SOCK_RECVBUF_LOCK_ASSERT(so2); 1848 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1849 1850 if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { 1851 kn->kn_flags |= EV_EOF; 1852 return (1); 1853 } else if (kn->kn_sfflags & NOTE_LOWAT) 1854 return (kn->kn_data >= kn->kn_sdata); 1855 else 1856 return (kn->kn_data >= so2->so_rcv.sb_lowat); 1857 } 1858 1859 static int 1860 uipc_filt_soempty(struct knote *kn, long hint) 1861 { 1862 struct socket *so = kn->kn_fp->f_data, *so2; 1863 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1864 1865 if (SOLISTENING(so) || unp2 == NULL) 1866 return (1); 1867 1868 so2 = unp2->unp_socket; 1869 SOCK_RECVBUF_LOCK_ASSERT(so2); 1870 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1871 1872 return (kn->kn_data == 0 ? 1 : 0); 1873 } 1874 1875 static const struct filterops uipc_write_filtops = { 1876 .f_isfd = 1, 1877 .f_detach = uipc_filt_sowdetach, 1878 .f_event = uipc_filt_sowrite, 1879 .f_copy = knote_triv_copy, 1880 }; 1881 static const struct filterops uipc_empty_filtops = { 1882 .f_isfd = 1, 1883 .f_detach = uipc_filt_sowdetach, 1884 .f_event = uipc_filt_soempty, 1885 .f_copy = knote_triv_copy, 1886 }; 1887 1888 static int 1889 uipc_kqfilter_stream_or_seqpacket(struct socket *so, struct knote *kn) 1890 { 1891 struct unpcb *unp = sotounpcb(so); 1892 struct knlist *knl; 1893 1894 switch (kn->kn_filter) { 1895 case EVFILT_READ: 1896 return (sokqfilter_generic(so, kn)); 1897 case EVFILT_WRITE: 1898 kn->kn_fop = &uipc_write_filtops; 1899 break; 1900 case EVFILT_EMPTY: 1901 kn->kn_fop = &uipc_empty_filtops; 1902 break; 1903 default: 1904 return (EINVAL); 1905 } 1906 1907 knl = &so->so_wrsel.si_note; 1908 UNP_PCB_LOCK(unp); 1909 if (SOLISTENING(so)) { 1910 SOLISTEN_LOCK(so); 1911 knlist_add(knl, kn, 1); 1912 SOLISTEN_UNLOCK(so); 1913 } else { 1914 struct socket *so2 = so->so_rcv.uxst_peer; 1915 1916 if (so2 != NULL) 1917 SOCK_RECVBUF_LOCK(so2); 1918 knlist_add(knl, kn, 1); 1919 if (so2 != NULL) 1920 SOCK_RECVBUF_UNLOCK(so2); 1921 } 1922 UNP_PCB_UNLOCK(unp); 1923 return (0); 1924 } 1925 1926 /* PF_UNIX/SOCK_DGRAM version of sbspace() */ 1927 static inline bool 1928 uipc_dgram_sbspace(struct sockbuf *sb, u_int cc, u_int mbcnt) 1929 { 1930 u_int bleft, mleft; 1931 1932 /* 1933 * Negative space may happen if send(2) is followed by 1934 * setsockopt(SO_SNDBUF/SO_RCVBUF) that shrinks maximum. 1935 */ 1936 if (__predict_false(sb->sb_hiwat < sb->uxdg_cc || 1937 sb->sb_mbmax < sb->uxdg_mbcnt)) 1938 return (false); 1939 1940 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) 1941 return (false); 1942 1943 bleft = sb->sb_hiwat - sb->uxdg_cc; 1944 mleft = sb->sb_mbmax - sb->uxdg_mbcnt; 1945 1946 return (bleft >= cc && mleft >= mbcnt); 1947 } 1948 1949 /* 1950 * PF_UNIX/SOCK_DGRAM send 1951 * 1952 * Allocate a record consisting of 3 mbufs in the sequence of 1953 * from -> control -> data and append it to the socket buffer. 1954 * 1955 * The first mbuf carries sender's name and is a pkthdr that stores 1956 * overall length of datagram, its memory consumption and control length. 1957 */ 1958 #define ctllen PH_loc.thirtytwo[1] 1959 _Static_assert(offsetof(struct pkthdr, memlen) + sizeof(u_int) <= 1960 offsetof(struct pkthdr, ctllen), "unix/dgram can not store ctllen"); 1961 static int 1962 uipc_sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1963 struct mbuf *m, struct mbuf *c, int flags, struct thread *td) 1964 { 1965 struct unpcb *unp, *unp2; 1966 const struct sockaddr *from; 1967 struct socket *so2; 1968 struct sockbuf *sb; 1969 struct mchain cmc = MCHAIN_INITIALIZER(&cmc); 1970 struct mbuf *f; 1971 u_int cc, ctl, mbcnt; 1972 u_int dcc __diagused, dctl __diagused, dmbcnt __diagused; 1973 int error, needsopts; 1974 1975 MPASS((uio != NULL && m == NULL) || (m != NULL && uio == NULL)); 1976 1977 error = needsopts = 0; 1978 f = NULL; 1979 1980 if (__predict_false(flags & MSG_OOB)) { 1981 error = EOPNOTSUPP; 1982 goto out; 1983 } 1984 if (m == NULL) { 1985 if (__predict_false(uio->uio_resid > unpdg_maxdgram)) { 1986 error = EMSGSIZE; 1987 goto out; 1988 } 1989 m = m_uiotombuf(uio, M_WAITOK, 0, max_hdr, M_PKTHDR); 1990 if (__predict_false(m == NULL)) { 1991 error = EFAULT; 1992 goto out; 1993 } 1994 f = m_gethdr(M_WAITOK, MT_SONAME); 1995 cc = m->m_pkthdr.len; 1996 mbcnt = MSIZE + m->m_pkthdr.memlen; 1997 if (c != NULL && 1998 (error = unp_internalize(c, &cmc, td, &needsopts))) 1999 goto out; 2000 } else { 2001 struct mchain mc; 2002 2003 uipc_reset_kernel_mbuf(m, &mc); 2004 cc = mc.mc_len; 2005 mbcnt = mc.mc_mlen; 2006 if (__predict_false(m->m_pkthdr.len > unpdg_maxdgram)) { 2007 error = EMSGSIZE; 2008 goto out; 2009 } 2010 if ((f = m_gethdr(M_NOWAIT, MT_SONAME)) == NULL) { 2011 error = ENOBUFS; 2012 goto out; 2013 } 2014 } 2015 2016 unp = sotounpcb(so); 2017 MPASS(unp); 2018 2019 /* 2020 * XXXGL: would be cool to fully remove so_snd out of the equation 2021 * and avoid this lock, which is not only extraneous, but also being 2022 * released, thus still leaving possibility for a race. We can easily 2023 * handle SBS_CANTSENDMORE/SS_ISCONNECTED complement in unpcb, but it 2024 * is more difficult to invent something to handle so_error. 2025 */ 2026 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2027 if (error) 2028 goto out2; 2029 SOCK_SENDBUF_LOCK(so); 2030 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2031 SOCK_SENDBUF_UNLOCK(so); 2032 error = EPIPE; 2033 goto out3; 2034 } 2035 if (so->so_error != 0) { 2036 error = so->so_error; 2037 so->so_error = 0; 2038 SOCK_SENDBUF_UNLOCK(so); 2039 goto out3; 2040 } 2041 if (((so->so_state & SS_ISCONNECTED) == 0) && addr == NULL) { 2042 SOCK_SENDBUF_UNLOCK(so); 2043 error = EDESTADDRREQ; 2044 goto out3; 2045 } 2046 SOCK_SENDBUF_UNLOCK(so); 2047 2048 if (addr != NULL) { 2049 if ((error = unp_connectat(AT_FDCWD, so, addr, td, true))) 2050 goto out3; 2051 UNP_PCB_LOCK_ASSERT(unp); 2052 unp2 = unp->unp_conn; 2053 UNP_PCB_LOCK_ASSERT(unp2); 2054 } else { 2055 UNP_PCB_LOCK(unp); 2056 unp2 = unp_pcb_lock_peer(unp); 2057 if (unp2 == NULL) { 2058 UNP_PCB_UNLOCK(unp); 2059 error = ENOTCONN; 2060 goto out3; 2061 } 2062 } 2063 2064 /* Check for SO_PASS* flags */ 2065 so2 = unp2->unp_socket; 2066 if ((atomic_load_int(&so2->so_options) & needsopts) != needsopts) { 2067 error = EPERM; 2068 goto out4; 2069 } 2070 2071 if (unp2->unp_flags & UNP_WANTCRED_MASK) 2072 unp_addsockcred(td, &cmc, unp2->unp_flags); 2073 if (unp->unp_addr != NULL) 2074 from = (struct sockaddr *)unp->unp_addr; 2075 else 2076 from = &sun_noname; 2077 f->m_len = from->sa_len; 2078 MPASS(from->sa_len <= MLEN); 2079 bcopy(from, mtod(f, void *), from->sa_len); 2080 2081 /* 2082 * Concatenate mbufs: from -> control -> data. 2083 * Save overall cc and mbcnt in "from" mbuf. 2084 */ 2085 if (!STAILQ_EMPTY(&cmc.mc_q)) { 2086 f->m_next = mc_first(&cmc); 2087 mc_last(&cmc)->m_next = m; 2088 /* XXXGL: This is dirty as well as rollback after ENOBUFS. */ 2089 STAILQ_INIT(&cmc.mc_q); 2090 } else 2091 f->m_next = m; 2092 m = NULL; 2093 ctl = f->m_len + cmc.mc_len; 2094 mbcnt += cmc.mc_mlen; 2095 #ifdef INVARIANTS 2096 dcc = dctl = dmbcnt = 0; 2097 for (struct mbuf *mb = f; mb != NULL; mb = mb->m_next) { 2098 if (mb->m_type == MT_DATA) 2099 dcc += mb->m_len; 2100 else 2101 dctl += mb->m_len; 2102 dmbcnt += MSIZE; 2103 if (mb->m_flags & M_EXT) 2104 dmbcnt += mb->m_ext.ext_size; 2105 } 2106 MPASS(dcc == cc); 2107 MPASS(dctl == ctl); 2108 MPASS(dmbcnt == mbcnt); 2109 #endif 2110 f->m_pkthdr.len = cc + ctl; 2111 f->m_pkthdr.memlen = mbcnt; 2112 f->m_pkthdr.ctllen = ctl; 2113 2114 /* 2115 * Destination socket buffer selection. 2116 * 2117 * Unconnected sends, when !(so->so_state & SS_ISCONNECTED) and the 2118 * destination address is supplied, create a temporary connection for 2119 * the run time of the function (see call to unp_connectat() above and 2120 * to unp_disconnect() below). We distinguish them by condition of 2121 * (addr != NULL). We intentionally avoid adding 'bool connected' for 2122 * that condition, since, again, through the run time of this code we 2123 * are always connected. For such "unconnected" sends, the destination 2124 * buffer would be the receive buffer of destination socket so2. 2125 * 2126 * For connected sends, data lands on the send buffer of the sender's 2127 * socket "so". Then, if we just added the very first datagram 2128 * on this send buffer, we need to add the send buffer on to the 2129 * receiving socket's buffer list. We put ourselves on top of the 2130 * list. Such logic gives infrequent senders priority over frequent 2131 * senders. 2132 * 2133 * Note on byte count management. As long as event methods kevent(2), 2134 * select(2) are not protocol specific (yet), we need to maintain 2135 * meaningful values on the receive buffer. So, the receive buffer 2136 * would accumulate counters from all connected buffers potentially 2137 * having sb_ccc > sb_hiwat or sb_mbcnt > sb_mbmax. 2138 */ 2139 sb = (addr == NULL) ? &so->so_snd : &so2->so_rcv; 2140 SOCK_RECVBUF_LOCK(so2); 2141 if (uipc_dgram_sbspace(sb, cc + ctl, mbcnt)) { 2142 if (addr == NULL && STAILQ_EMPTY(&sb->uxdg_mb)) 2143 TAILQ_INSERT_HEAD(&so2->so_rcv.uxdg_conns, &so->so_snd, 2144 uxdg_clist); 2145 STAILQ_INSERT_TAIL(&sb->uxdg_mb, f, m_stailqpkt); 2146 sb->uxdg_cc += cc + ctl; 2147 sb->uxdg_ctl += ctl; 2148 sb->uxdg_mbcnt += mbcnt; 2149 so2->so_rcv.sb_acc += cc + ctl; 2150 so2->so_rcv.sb_ccc += cc + ctl; 2151 so2->so_rcv.sb_ctl += ctl; 2152 so2->so_rcv.sb_mbcnt += mbcnt; 2153 sorwakeup_locked(so2); 2154 f = NULL; 2155 } else { 2156 soroverflow_locked(so2); 2157 error = ENOBUFS; 2158 if (f->m_next->m_type == MT_CONTROL) { 2159 STAILQ_FIRST(&cmc.mc_q) = f->m_next; 2160 f->m_next = NULL; 2161 } 2162 } 2163 2164 out4: 2165 if (addr != NULL) 2166 unp_disconnect(unp, unp2); 2167 else 2168 unp_pcb_unlock_pair(unp, unp2); 2169 2170 td->td_ru.ru_msgsnd++; 2171 2172 out3: 2173 SOCK_IO_SEND_UNLOCK(so); 2174 out2: 2175 if (!mc_empty(&cmc)) 2176 unp_scan(mc_first(&cmc), unp_freerights); 2177 out: 2178 if (f) 2179 m_freem(f); 2180 mc_freem(&cmc); 2181 if (m) 2182 m_freem(m); 2183 2184 return (error); 2185 } 2186 2187 /* 2188 * PF_UNIX/SOCK_DGRAM receive with MSG_PEEK. 2189 * The mbuf has already been unlinked from the uxdg_mb of socket buffer 2190 * and needs to be linked onto uxdg_peeked of receive socket buffer. 2191 */ 2192 static int 2193 uipc_peek_dgram(struct socket *so, struct mbuf *m, struct sockaddr **psa, 2194 struct uio *uio, struct mbuf **controlp, int *flagsp) 2195 { 2196 ssize_t len = 0; 2197 int error; 2198 2199 so->so_rcv.uxdg_peeked = m; 2200 so->so_rcv.uxdg_cc += m->m_pkthdr.len; 2201 so->so_rcv.uxdg_ctl += m->m_pkthdr.ctllen; 2202 so->so_rcv.uxdg_mbcnt += m->m_pkthdr.memlen; 2203 SOCK_RECVBUF_UNLOCK(so); 2204 2205 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2206 if (psa != NULL) 2207 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2208 2209 m = m->m_next; 2210 KASSERT(m, ("%s: no data or control after soname", __func__)); 2211 2212 /* 2213 * With MSG_PEEK the control isn't executed, just copied. 2214 */ 2215 while (m != NULL && m->m_type == MT_CONTROL) { 2216 if (controlp != NULL) { 2217 *controlp = m_copym(m, 0, m->m_len, M_WAITOK); 2218 controlp = &(*controlp)->m_next; 2219 } 2220 m = m->m_next; 2221 } 2222 KASSERT(m == NULL || m->m_type == MT_DATA, 2223 ("%s: not MT_DATA mbuf %p", __func__, m)); 2224 while (m != NULL && uio->uio_resid > 0) { 2225 len = uio->uio_resid; 2226 if (len > m->m_len) 2227 len = m->m_len; 2228 error = uiomove(mtod(m, char *), (int)len, uio); 2229 if (error) { 2230 SOCK_IO_RECV_UNLOCK(so); 2231 return (error); 2232 } 2233 if (len == m->m_len) 2234 m = m->m_next; 2235 } 2236 SOCK_IO_RECV_UNLOCK(so); 2237 2238 if (flagsp != NULL) { 2239 if (m != NULL) { 2240 if (*flagsp & MSG_TRUNC) { 2241 /* Report real length of the packet */ 2242 uio->uio_resid -= m_length(m, NULL) - len; 2243 } 2244 *flagsp |= MSG_TRUNC; 2245 } else 2246 *flagsp &= ~MSG_TRUNC; 2247 } 2248 2249 return (0); 2250 } 2251 2252 /* 2253 * PF_UNIX/SOCK_DGRAM receive 2254 */ 2255 static int 2256 uipc_soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2257 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2258 { 2259 struct sockbuf *sb = NULL; 2260 struct mbuf *m; 2261 int flags, error; 2262 ssize_t len = 0; 2263 bool nonblock; 2264 2265 MPASS(mp0 == NULL); 2266 2267 if (psa != NULL) 2268 *psa = NULL; 2269 if (controlp != NULL) 2270 *controlp = NULL; 2271 2272 flags = flagsp != NULL ? *flagsp : 0; 2273 nonblock = (so->so_state & SS_NBIO) || 2274 (flags & (MSG_DONTWAIT | MSG_NBIO)); 2275 2276 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 2277 if (__predict_false(error)) 2278 return (error); 2279 2280 /* 2281 * Loop blocking while waiting for a datagram. Prioritize connected 2282 * peers over unconnected sends. Set sb to selected socket buffer 2283 * containing an mbuf on exit from the wait loop. A datagram that 2284 * had already been peeked at has top priority. 2285 */ 2286 SOCK_RECVBUF_LOCK(so); 2287 while ((m = so->so_rcv.uxdg_peeked) == NULL && 2288 (sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) == NULL && 2289 (m = STAILQ_FIRST(&so->so_rcv.uxdg_mb)) == NULL) { 2290 if (so->so_error) { 2291 error = so->so_error; 2292 if (!(flags & MSG_PEEK)) 2293 so->so_error = 0; 2294 SOCK_RECVBUF_UNLOCK(so); 2295 SOCK_IO_RECV_UNLOCK(so); 2296 return (error); 2297 } 2298 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2299 uio->uio_resid == 0) { 2300 SOCK_RECVBUF_UNLOCK(so); 2301 SOCK_IO_RECV_UNLOCK(so); 2302 return (0); 2303 } 2304 if (nonblock) { 2305 SOCK_RECVBUF_UNLOCK(so); 2306 SOCK_IO_RECV_UNLOCK(so); 2307 return (EWOULDBLOCK); 2308 } 2309 error = sbwait(so, SO_RCV); 2310 if (error) { 2311 SOCK_RECVBUF_UNLOCK(so); 2312 SOCK_IO_RECV_UNLOCK(so); 2313 return (error); 2314 } 2315 } 2316 2317 if (sb == NULL) 2318 sb = &so->so_rcv; 2319 else if (m == NULL) 2320 m = STAILQ_FIRST(&sb->uxdg_mb); 2321 else 2322 MPASS(m == so->so_rcv.uxdg_peeked); 2323 2324 MPASS(sb->uxdg_cc > 0); 2325 M_ASSERTPKTHDR(m); 2326 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2327 2328 if (uio->uio_td) 2329 uio->uio_td->td_ru.ru_msgrcv++; 2330 2331 if (__predict_true(m != so->so_rcv.uxdg_peeked)) { 2332 STAILQ_REMOVE_HEAD(&sb->uxdg_mb, m_stailqpkt); 2333 if (STAILQ_EMPTY(&sb->uxdg_mb) && sb != &so->so_rcv) 2334 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 2335 } else 2336 so->so_rcv.uxdg_peeked = NULL; 2337 2338 sb->uxdg_cc -= m->m_pkthdr.len; 2339 sb->uxdg_ctl -= m->m_pkthdr.ctllen; 2340 sb->uxdg_mbcnt -= m->m_pkthdr.memlen; 2341 2342 if (__predict_false(flags & MSG_PEEK)) 2343 return (uipc_peek_dgram(so, m, psa, uio, controlp, flagsp)); 2344 2345 so->so_rcv.sb_acc -= m->m_pkthdr.len; 2346 so->so_rcv.sb_ccc -= m->m_pkthdr.len; 2347 so->so_rcv.sb_ctl -= m->m_pkthdr.ctllen; 2348 so->so_rcv.sb_mbcnt -= m->m_pkthdr.memlen; 2349 SOCK_RECVBUF_UNLOCK(so); 2350 2351 if (psa != NULL) 2352 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2353 m = m_free(m); 2354 KASSERT(m, ("%s: no data or control after soname", __func__)); 2355 2356 /* 2357 * Packet to copyout() is now in 'm' and it is disconnected from the 2358 * queue. 2359 * 2360 * Process one or more MT_CONTROL mbufs present before any data mbufs 2361 * in the first mbuf chain on the socket buffer. We call into the 2362 * unp_externalize() to perform externalization (or freeing if 2363 * controlp == NULL). In some cases there can be only MT_CONTROL mbufs 2364 * without MT_DATA mbufs. 2365 */ 2366 while (m != NULL && m->m_type == MT_CONTROL) { 2367 error = unp_externalize(so, m, controlp, flags); 2368 m = m_free(m); 2369 if (error != 0) { 2370 SOCK_IO_RECV_UNLOCK(so); 2371 unp_scan(m, unp_freerights); 2372 m_freem(m); 2373 return (error); 2374 } 2375 if (controlp != NULL) { 2376 while (*controlp != NULL) 2377 controlp = &(*controlp)->m_next; 2378 } 2379 } 2380 KASSERT(m == NULL || m->m_type == MT_DATA, 2381 ("%s: not MT_DATA mbuf %p", __func__, m)); 2382 while (m != NULL && uio->uio_resid > 0) { 2383 len = uio->uio_resid; 2384 if (len > m->m_len) 2385 len = m->m_len; 2386 error = uiomove(mtod(m, char *), (int)len, uio); 2387 if (error) { 2388 SOCK_IO_RECV_UNLOCK(so); 2389 m_freem(m); 2390 return (error); 2391 } 2392 if (len == m->m_len) 2393 m = m_free(m); 2394 else { 2395 m->m_data += len; 2396 m->m_len -= len; 2397 } 2398 } 2399 SOCK_IO_RECV_UNLOCK(so); 2400 2401 if (m != NULL) { 2402 if (flagsp != NULL) { 2403 if (flags & MSG_TRUNC) { 2404 /* Report real length of the packet */ 2405 uio->uio_resid -= m_length(m, NULL); 2406 } 2407 *flagsp |= MSG_TRUNC; 2408 } 2409 m_freem(m); 2410 } else if (flagsp != NULL) 2411 *flagsp &= ~MSG_TRUNC; 2412 2413 return (0); 2414 } 2415 2416 static int 2417 uipc_sendfile_wait(struct socket *so, off_t need, int *space) 2418 { 2419 struct unpcb *unp2; 2420 struct socket *so2; 2421 struct sockbuf *sb; 2422 bool nonblock, sockref; 2423 int error; 2424 2425 MPASS(so->so_type == SOCK_STREAM); 2426 MPASS(need > 0); 2427 MPASS(space != NULL); 2428 2429 nonblock = so->so_state & SS_NBIO; 2430 sockref = false; 2431 2432 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) 2433 return (ENOTCONN); 2434 2435 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2436 return (error); 2437 2438 so2 = unp2->unp_socket; 2439 sb = &so2->so_rcv; 2440 SOCK_RECVBUF_LOCK(so2); 2441 UNP_PCB_UNLOCK(unp2); 2442 while ((*space = uipc_stream_sbspace(sb)) < need && 2443 (*space < so->so_snd.sb_hiwat / 2)) { 2444 UIPC_STREAM_SBCHECK(sb); 2445 if (nonblock) { 2446 SOCK_RECVBUF_UNLOCK(so2); 2447 return (EAGAIN); 2448 } 2449 if (!sockref) { 2450 soref(so2); 2451 sockref = true; 2452 } 2453 error = uipc_stream_sbwait(so2, so->so_snd.sb_timeo); 2454 if (error == 0 && 2455 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) 2456 error = EPIPE; 2457 if (error) { 2458 SOCK_RECVBUF_UNLOCK(so2); 2459 sorele(so2); 2460 return (error); 2461 } 2462 } 2463 UIPC_STREAM_SBCHECK(sb); 2464 SOCK_RECVBUF_UNLOCK(so2); 2465 if (sockref) 2466 sorele(so2); 2467 2468 return (0); 2469 } 2470 2471 /* 2472 * Although this is a pr_send method, for unix(4) it is called only via 2473 * sendfile(2) path. This means we can be sure that mbufs are clear of 2474 * any extra flags and don't require any conditioning. 2475 */ 2476 static int 2477 uipc_sendfile(struct socket *so, int flags, struct mbuf *m, 2478 struct sockaddr *from, struct mbuf *control, struct thread *td) 2479 { 2480 struct mchain mc; 2481 struct unpcb *unp2; 2482 struct socket *so2; 2483 struct sockbuf *sb; 2484 bool notready, wakeup; 2485 int error; 2486 2487 MPASS(so->so_type == SOCK_STREAM); 2488 MPASS(from == NULL && control == NULL); 2489 KASSERT(!(m->m_flags & M_EXTPG), 2490 ("unix(4): TLS sendfile(2) not supported")); 2491 2492 notready = flags & PRUS_NOTREADY; 2493 2494 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) { 2495 error = ENOTCONN; 2496 goto out; 2497 } 2498 2499 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2500 goto out; 2501 2502 mc_init_m(&mc, m); 2503 2504 so2 = unp2->unp_socket; 2505 sb = &so2->so_rcv; 2506 SOCK_RECVBUF_LOCK(so2); 2507 UNP_PCB_UNLOCK(unp2); 2508 UIPC_STREAM_SBCHECK(sb); 2509 sb->sb_ccc += mc.mc_len; 2510 sb->sb_mbcnt += mc.mc_mlen; 2511 if (sb->uxst_fnrdy == NULL) { 2512 if (notready) { 2513 wakeup = false; 2514 STAILQ_FOREACH(m, &mc.mc_q, m_stailq) { 2515 if (m->m_flags & M_NOTREADY) { 2516 sb->uxst_fnrdy = m; 2517 break; 2518 } else { 2519 sb->sb_acc += m->m_len; 2520 wakeup = true; 2521 } 2522 } 2523 } else { 2524 wakeup = true; 2525 sb->sb_acc += mc.mc_len; 2526 } 2527 } else { 2528 wakeup = false; 2529 } 2530 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 2531 UIPC_STREAM_SBCHECK(sb); 2532 if (wakeup) 2533 sorwakeup_locked(so2); 2534 else 2535 SOCK_RECVBUF_UNLOCK(so2); 2536 2537 return (0); 2538 out: 2539 /* 2540 * In case of not ready data, uipc_ready() is responsible 2541 * for freeing memory. 2542 */ 2543 if (m != NULL && !notready) 2544 m_freem(m); 2545 2546 return (error); 2547 } 2548 2549 static int 2550 uipc_sbready(struct sockbuf *sb, struct mbuf *m, int count) 2551 { 2552 bool blocker; 2553 2554 /* assert locked */ 2555 2556 blocker = (sb->uxst_fnrdy == m); 2557 STAILQ_FOREACH_FROM(m, &sb->uxst_mbq, m_stailq) { 2558 if (count > 0) { 2559 MPASS(m->m_flags & M_NOTREADY); 2560 m->m_flags &= ~M_NOTREADY; 2561 if (blocker) 2562 sb->sb_acc += m->m_len; 2563 count--; 2564 } else if (m->m_flags & M_NOTREADY) 2565 break; 2566 else if (blocker) 2567 sb->sb_acc += m->m_len; 2568 } 2569 if (blocker) { 2570 sb->uxst_fnrdy = m; 2571 return (0); 2572 } else 2573 return (EINPROGRESS); 2574 } 2575 2576 static bool 2577 uipc_ready_scan(struct socket *so, struct mbuf *m, int count, int *errorp) 2578 { 2579 struct mbuf *mb; 2580 struct sockbuf *sb; 2581 2582 SOCK_LOCK(so); 2583 if (SOLISTENING(so)) { 2584 SOCK_UNLOCK(so); 2585 return (false); 2586 } 2587 mb = NULL; 2588 sb = &so->so_rcv; 2589 SOCK_RECVBUF_LOCK(so); 2590 if (sb->uxst_fnrdy != NULL) { 2591 STAILQ_FOREACH(mb, &sb->uxst_mbq, m_stailq) { 2592 if (mb == m) { 2593 *errorp = uipc_sbready(sb, m, count); 2594 break; 2595 } 2596 } 2597 } 2598 SOCK_RECVBUF_UNLOCK(so); 2599 SOCK_UNLOCK(so); 2600 return (mb != NULL); 2601 } 2602 2603 static int 2604 uipc_ready(struct socket *so, struct mbuf *m, int count) 2605 { 2606 struct unpcb *unp, *unp2; 2607 int error; 2608 2609 MPASS(so->so_type == SOCK_STREAM); 2610 2611 if (__predict_true(uipc_lock_peer(so, &unp2) == 0)) { 2612 struct socket *so2; 2613 struct sockbuf *sb; 2614 2615 so2 = unp2->unp_socket; 2616 sb = &so2->so_rcv; 2617 SOCK_RECVBUF_LOCK(so2); 2618 UNP_PCB_UNLOCK(unp2); 2619 UIPC_STREAM_SBCHECK(sb); 2620 error = uipc_sbready(sb, m, count); 2621 UIPC_STREAM_SBCHECK(sb); 2622 if (error == 0) 2623 sorwakeup_locked(so2); 2624 else 2625 SOCK_RECVBUF_UNLOCK(so2); 2626 } else { 2627 /* 2628 * The receiving socket has been disconnected, but may still 2629 * be valid. In this case, the not-ready mbufs are still 2630 * present in its socket buffer, so perform an exhaustive 2631 * search before giving up and freeing the mbufs. 2632 */ 2633 UNP_LINK_RLOCK(); 2634 LIST_FOREACH(unp, &unp_shead, unp_link) { 2635 if (uipc_ready_scan(unp->unp_socket, m, count, &error)) 2636 break; 2637 } 2638 UNP_LINK_RUNLOCK(); 2639 2640 if (unp == NULL) { 2641 for (int i = 0; i < count; i++) 2642 m = m_free(m); 2643 return (ECONNRESET); 2644 } 2645 } 2646 return (error); 2647 } 2648 2649 static int 2650 uipc_sense(struct socket *so, struct stat *sb) 2651 { 2652 struct unpcb *unp; 2653 2654 unp = sotounpcb(so); 2655 KASSERT(unp != NULL, ("uipc_sense: unp == NULL")); 2656 2657 sb->st_blksize = so->so_snd.sb_hiwat; 2658 sb->st_dev = NODEV; 2659 sb->st_ino = unp->unp_ino; 2660 return (0); 2661 } 2662 2663 static int 2664 uipc_shutdown(struct socket *so, enum shutdown_how how) 2665 { 2666 struct unpcb *unp = sotounpcb(so); 2667 int error; 2668 2669 SOCK_LOCK(so); 2670 if (SOLISTENING(so)) { 2671 if (how != SHUT_WR) { 2672 so->so_error = ECONNABORTED; 2673 solisten_wakeup(so); /* unlocks so */ 2674 } else 2675 SOCK_UNLOCK(so); 2676 return (ENOTCONN); 2677 } else if ((so->so_state & 2678 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2679 /* 2680 * POSIX mandates us to just return ENOTCONN when shutdown(2) is 2681 * invoked on a datagram sockets, however historically we would 2682 * actually tear socket down. This is known to be leveraged by 2683 * some applications to unblock process waiting in recv(2) by 2684 * other process that it shares that socket with. Try to meet 2685 * both backward-compatibility and POSIX requirements by forcing 2686 * ENOTCONN but still flushing buffers and performing wakeup(9). 2687 * 2688 * XXXGL: it remains unknown what applications expect this 2689 * behavior and is this isolated to unix/dgram or inet/dgram or 2690 * both. See: D10351, D3039. 2691 */ 2692 error = ENOTCONN; 2693 if (so->so_type != SOCK_DGRAM) { 2694 SOCK_UNLOCK(so); 2695 return (error); 2696 } 2697 } else 2698 error = 0; 2699 SOCK_UNLOCK(so); 2700 2701 switch (how) { 2702 case SHUT_RD: 2703 if (so->so_type == SOCK_DGRAM) 2704 socantrcvmore(so); 2705 else 2706 uipc_cantrcvmore(so); 2707 unp_dispose(so); 2708 break; 2709 case SHUT_RDWR: 2710 if (so->so_type == SOCK_DGRAM) 2711 socantrcvmore(so); 2712 else 2713 uipc_cantrcvmore(so); 2714 unp_dispose(so); 2715 /* FALLTHROUGH */ 2716 case SHUT_WR: 2717 if (so->so_type == SOCK_DGRAM) { 2718 socantsendmore(so); 2719 } else { 2720 UNP_PCB_LOCK(unp); 2721 if (unp->unp_conn != NULL) 2722 uipc_cantrcvmore(unp->unp_conn->unp_socket); 2723 UNP_PCB_UNLOCK(unp); 2724 } 2725 } 2726 wakeup(&so->so_timeo); 2727 2728 return (error); 2729 } 2730 2731 static int 2732 uipc_sockaddr(struct socket *so, struct sockaddr *ret) 2733 { 2734 struct unpcb *unp; 2735 const struct sockaddr *sa; 2736 2737 unp = sotounpcb(so); 2738 KASSERT(unp != NULL, ("uipc_sockaddr: unp == NULL")); 2739 2740 UNP_PCB_LOCK(unp); 2741 if (unp->unp_addr != NULL) 2742 sa = (struct sockaddr *) unp->unp_addr; 2743 else 2744 sa = &sun_noname; 2745 bcopy(sa, ret, sa->sa_len); 2746 UNP_PCB_UNLOCK(unp); 2747 return (0); 2748 } 2749 2750 static int 2751 uipc_ctloutput(struct socket *so, struct sockopt *sopt) 2752 { 2753 struct unpcb *unp; 2754 struct xucred xu; 2755 int error, optval; 2756 2757 if (sopt->sopt_level != SOL_LOCAL) 2758 return (EINVAL); 2759 2760 unp = sotounpcb(so); 2761 KASSERT(unp != NULL, ("uipc_ctloutput: unp == NULL")); 2762 error = 0; 2763 switch (sopt->sopt_dir) { 2764 case SOPT_GET: 2765 switch (sopt->sopt_name) { 2766 case LOCAL_PEERCRED: 2767 UNP_PCB_LOCK(unp); 2768 if (unp->unp_flags & UNP_HAVEPC) 2769 xu = unp->unp_peercred; 2770 else { 2771 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2772 error = ENOTCONN; 2773 else 2774 error = EINVAL; 2775 } 2776 UNP_PCB_UNLOCK(unp); 2777 if (error != 0) 2778 break; 2779 #ifdef COMPAT_FREEBSD32 2780 if (sopt->sopt_td && 2781 SV_PROC_FLAG(sopt->sopt_td->td_proc, SV_ILP32)) 2782 { 2783 struct xucred32 xu32 = {}; 2784 int i; 2785 2786 xu32.cr_version = xu.cr_version; 2787 xu32.cr_uid = xu.cr_uid; 2788 xu32.cr_ngroups = xu.cr_ngroups; 2789 for (i = 0; i < XU_NGROUPS; i++) 2790 xu32.cr_groups[i] = xu.cr_groups[i]; 2791 xu32.cr_pid = xu.cr_pid; 2792 error = sooptcopyout(sopt, &xu32, sizeof(xu32)); 2793 break; 2794 } 2795 #endif 2796 error = sooptcopyout(sopt, &xu, sizeof(xu)); 2797 break; 2798 2799 case LOCAL_CREDS: 2800 /* Unlocked read. */ 2801 optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0; 2802 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2803 break; 2804 2805 case LOCAL_CREDS_PERSISTENT: 2806 /* Unlocked read. */ 2807 optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0; 2808 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2809 break; 2810 2811 default: 2812 error = EOPNOTSUPP; 2813 break; 2814 } 2815 break; 2816 2817 case SOPT_SET: 2818 switch (sopt->sopt_name) { 2819 case LOCAL_CREDS: 2820 case LOCAL_CREDS_PERSISTENT: 2821 error = sooptcopyin(sopt, &optval, sizeof(optval), 2822 sizeof(optval)); 2823 if (error) 2824 break; 2825 2826 #define OPTSET(bit, exclusive) do { \ 2827 UNP_PCB_LOCK(unp); \ 2828 if (optval) { \ 2829 if ((unp->unp_flags & (exclusive)) != 0) { \ 2830 UNP_PCB_UNLOCK(unp); \ 2831 error = EINVAL; \ 2832 break; \ 2833 } \ 2834 unp->unp_flags |= (bit); \ 2835 } else \ 2836 unp->unp_flags &= ~(bit); \ 2837 UNP_PCB_UNLOCK(unp); \ 2838 } while (0) 2839 2840 switch (sopt->sopt_name) { 2841 case LOCAL_CREDS: 2842 OPTSET(UNP_WANTCRED_ONESHOT, UNP_WANTCRED_ALWAYS); 2843 break; 2844 2845 case LOCAL_CREDS_PERSISTENT: 2846 OPTSET(UNP_WANTCRED_ALWAYS, UNP_WANTCRED_ONESHOT); 2847 break; 2848 2849 default: 2850 break; 2851 } 2852 break; 2853 #undef OPTSET 2854 default: 2855 error = ENOPROTOOPT; 2856 break; 2857 } 2858 break; 2859 2860 default: 2861 error = EOPNOTSUPP; 2862 break; 2863 } 2864 return (error); 2865 } 2866 2867 static int 2868 unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 2869 { 2870 2871 return (unp_connectat(AT_FDCWD, so, nam, td, false)); 2872 } 2873 2874 static int 2875 unp_connectat(int fd, struct socket *so, struct sockaddr *nam, 2876 struct thread *td, bool return_locked) 2877 { 2878 struct mtx *vplock; 2879 struct sockaddr_un *soun; 2880 struct vnode *vp; 2881 struct socket *so2; 2882 struct unpcb *unp, *unp2, *unp3; 2883 struct nameidata nd; 2884 char buf[SOCK_MAXADDRLEN]; 2885 struct sockaddr *sa; 2886 cap_rights_t rights; 2887 int error, len; 2888 bool connreq; 2889 2890 CURVNET_ASSERT_SET(); 2891 2892 if (nam->sa_family != AF_UNIX) 2893 return (EAFNOSUPPORT); 2894 if (nam->sa_len > sizeof(struct sockaddr_un)) 2895 return (EINVAL); 2896 len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); 2897 if (len <= 0) 2898 return (EINVAL); 2899 soun = (struct sockaddr_un *)nam; 2900 bcopy(soun->sun_path, buf, len); 2901 buf[len] = 0; 2902 2903 error = 0; 2904 unp = sotounpcb(so); 2905 UNP_PCB_LOCK(unp); 2906 for (;;) { 2907 /* 2908 * Wait for connection state to stabilize. If a connection 2909 * already exists, give up. For datagram sockets, which permit 2910 * multiple consecutive connect(2) calls, upper layers are 2911 * responsible for disconnecting in advance of a subsequent 2912 * connect(2), but this is not synchronized with PCB connection 2913 * state. 2914 * 2915 * Also make sure that no threads are currently attempting to 2916 * lock the peer socket, to ensure that unp_conn cannot 2917 * transition between two valid sockets while locks are dropped. 2918 */ 2919 if (SOLISTENING(so)) 2920 error = EOPNOTSUPP; 2921 else if (unp->unp_conn != NULL) 2922 error = EISCONN; 2923 else if ((unp->unp_flags & UNP_CONNECTING) != 0) { 2924 error = EALREADY; 2925 } 2926 if (error != 0) { 2927 UNP_PCB_UNLOCK(unp); 2928 return (error); 2929 } 2930 if (unp->unp_pairbusy > 0) { 2931 unp->unp_flags |= UNP_WAITING; 2932 mtx_sleep(unp, UNP_PCB_LOCKPTR(unp), 0, "unpeer", 0); 2933 continue; 2934 } 2935 break; 2936 } 2937 unp->unp_flags |= UNP_CONNECTING; 2938 UNP_PCB_UNLOCK(unp); 2939 2940 connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0; 2941 if (connreq) 2942 sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK); 2943 else 2944 sa = NULL; 2945 NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | 2946 (fd == AT_FDCWD ? 0 : EMPTYPATH), UIO_SYSSPACE, buf, fd, 2947 cap_rights_init_one(&rights, CAP_CONNECTAT)); 2948 error = namei(&nd); 2949 if (error) 2950 vp = NULL; 2951 else 2952 vp = nd.ni_vp; 2953 ASSERT_VOP_LOCKED(vp, "unp_connect"); 2954 if (error) 2955 goto bad; 2956 NDFREE_PNBUF(&nd); 2957 2958 if (vp->v_type != VSOCK) { 2959 error = ENOTSOCK; 2960 goto bad; 2961 } 2962 #ifdef MAC 2963 error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD); 2964 if (error) 2965 goto bad; 2966 #endif 2967 error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td); 2968 if (error) 2969 goto bad; 2970 2971 unp = sotounpcb(so); 2972 KASSERT(unp != NULL, ("unp_connect: unp == NULL")); 2973 2974 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 2975 mtx_lock(vplock); 2976 VOP_UNP_CONNECT(vp, &unp2); 2977 if (unp2 == NULL) { 2978 error = ECONNREFUSED; 2979 goto bad2; 2980 } 2981 so2 = unp2->unp_socket; 2982 if (so->so_type != so2->so_type) { 2983 error = EPROTOTYPE; 2984 goto bad2; 2985 } 2986 if (connreq) { 2987 if (SOLISTENING(so2)) 2988 so2 = solisten_clone(so2); 2989 else 2990 so2 = NULL; 2991 if (so2 == NULL) { 2992 error = ECONNREFUSED; 2993 goto bad2; 2994 } 2995 if ((error = uipc_attach(so2, 0, NULL)) != 0) { 2996 sodealloc(so2); 2997 goto bad2; 2998 } 2999 unp3 = sotounpcb(so2); 3000 unp_pcb_lock_pair(unp2, unp3); 3001 if (unp2->unp_addr != NULL) { 3002 bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); 3003 unp3->unp_addr = (struct sockaddr_un *) sa; 3004 sa = NULL; 3005 } 3006 3007 unp_copy_peercred(td, unp3, unp, unp2); 3008 3009 UNP_PCB_UNLOCK(unp2); 3010 unp2 = unp3; 3011 3012 /* 3013 * It is safe to block on the PCB lock here since unp2 is 3014 * nascent and cannot be connected to any other sockets. 3015 */ 3016 UNP_PCB_LOCK(unp); 3017 #ifdef MAC 3018 mac_socketpeer_set_from_socket(so, so2); 3019 mac_socketpeer_set_from_socket(so2, so); 3020 #endif 3021 } else { 3022 unp_pcb_lock_pair(unp, unp2); 3023 } 3024 KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && 3025 sotounpcb(so2) == unp2, 3026 ("%s: unp2 %p so2 %p", __func__, unp2, so2)); 3027 unp_connect2(so, so2, connreq); 3028 if (connreq) 3029 (void)solisten_enqueue(so2, SS_ISCONNECTED); 3030 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 3031 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 3032 unp->unp_flags &= ~UNP_CONNECTING; 3033 if (!return_locked) 3034 unp_pcb_unlock_pair(unp, unp2); 3035 bad2: 3036 mtx_unlock(vplock); 3037 bad: 3038 if (vp != NULL) { 3039 /* 3040 * If we are returning locked (called via uipc_sosend_dgram()), 3041 * we need to be sure that vput() won't sleep. This is 3042 * guaranteed by VOP_UNP_CONNECT() call above and unp2 lock. 3043 * SOCK_STREAM/SEQPACKET can't request return_locked (yet). 3044 */ 3045 MPASS(!(return_locked && connreq)); 3046 vput(vp); 3047 } 3048 free(sa, M_SONAME); 3049 if (__predict_false(error)) { 3050 UNP_PCB_LOCK(unp); 3051 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 3052 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 3053 unp->unp_flags &= ~UNP_CONNECTING; 3054 UNP_PCB_UNLOCK(unp); 3055 } 3056 return (error); 3057 } 3058 3059 /* 3060 * Set socket peer credentials at connection time. 3061 * 3062 * The client's PCB credentials are copied from its process structure. The 3063 * server's PCB credentials are copied from the socket on which it called 3064 * listen(2). uipc_listen cached that process's credentials at the time. 3065 */ 3066 void 3067 unp_copy_peercred(struct thread *td, struct unpcb *client_unp, 3068 struct unpcb *server_unp, struct unpcb *listen_unp) 3069 { 3070 cru2xt(td, &client_unp->unp_peercred); 3071 client_unp->unp_flags |= UNP_HAVEPC; 3072 3073 memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred, 3074 sizeof(server_unp->unp_peercred)); 3075 server_unp->unp_flags |= UNP_HAVEPC; 3076 client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK); 3077 } 3078 3079 /* 3080 * unix/stream & unix/seqpacket version of soisconnected(). 3081 * 3082 * The crucial thing we are doing here is setting up the uxst_peer linkage, 3083 * holding unp and receive buffer locks of the both sockets. The disconnect 3084 * procedure does the same. This gives as a safe way to access the peer in the 3085 * send(2) and recv(2) during the socket lifetime. 3086 * 3087 * The less important thing is event notification of the fact that a socket is 3088 * now connected. It is unusual for a software to put a socket into event 3089 * mechanism before connect(2), but is supposed to be supported. Note that 3090 * there can not be any sleeping I/O on the socket, yet, only presence in the 3091 * select/poll/kevent. 3092 * 3093 * This function can be called via two call paths: 3094 * 1) socketpair(2) - in this case socket has not been yet reported to userland 3095 * and just can't have any event notifications mechanisms set up. The 3096 * 'wakeup' boolean is always false. 3097 * 2) connect(2) of existing socket to a recent clone of a listener: 3098 * 2.1) Socket that connect(2)s will have 'wakeup' true. An application 3099 * could have already put it into event mechanism, is it shall be 3100 * reported as readable and as writable. 3101 * 2.2) Socket that was just cloned with solisten_clone(). Same as 1). 3102 */ 3103 static void 3104 unp_soisconnected(struct socket *so, bool wakeup) 3105 { 3106 struct socket *so2 = sotounpcb(so)->unp_conn->unp_socket; 3107 struct sockbuf *sb; 3108 3109 SOCK_LOCK_ASSERT(so); 3110 UNP_PCB_LOCK_ASSERT(sotounpcb(so)); 3111 UNP_PCB_LOCK_ASSERT(sotounpcb(so2)); 3112 SOCK_RECVBUF_LOCK_ASSERT(so); 3113 SOCK_RECVBUF_LOCK_ASSERT(so2); 3114 3115 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3116 MPASS((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 3117 SS_ISDISCONNECTING)) == 0); 3118 MPASS(so->so_qstate == SQ_NONE); 3119 3120 so->so_state &= ~SS_ISDISCONNECTED; 3121 so->so_state |= SS_ISCONNECTED; 3122 3123 sb = &so2->so_rcv; 3124 sb->uxst_peer = so; 3125 3126 if (wakeup) { 3127 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); 3128 sb = &so->so_rcv; 3129 selwakeuppri(sb->sb_sel, PSOCK); 3130 SOCK_SENDBUF_LOCK_ASSERT(so); 3131 sb = &so->so_snd; 3132 selwakeuppri(sb->sb_sel, PSOCK); 3133 SOCK_SENDBUF_UNLOCK(so); 3134 } 3135 } 3136 3137 static void 3138 unp_connect2(struct socket *so, struct socket *so2, bool wakeup) 3139 { 3140 struct unpcb *unp; 3141 struct unpcb *unp2; 3142 3143 MPASS(so2->so_type == so->so_type); 3144 unp = sotounpcb(so); 3145 KASSERT(unp != NULL, ("unp_connect2: unp == NULL")); 3146 unp2 = sotounpcb(so2); 3147 KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL")); 3148 3149 UNP_PCB_LOCK_ASSERT(unp); 3150 UNP_PCB_LOCK_ASSERT(unp2); 3151 KASSERT(unp->unp_conn == NULL, 3152 ("%s: socket %p is already connected", __func__, unp)); 3153 3154 unp->unp_conn = unp2; 3155 unp_pcb_hold(unp2); 3156 unp_pcb_hold(unp); 3157 switch (so->so_type) { 3158 case SOCK_DGRAM: 3159 UNP_REF_LIST_LOCK(); 3160 LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); 3161 UNP_REF_LIST_UNLOCK(); 3162 soisconnected(so); 3163 break; 3164 3165 case SOCK_STREAM: 3166 case SOCK_SEQPACKET: 3167 KASSERT(unp2->unp_conn == NULL, 3168 ("%s: socket %p is already connected", __func__, unp2)); 3169 unp2->unp_conn = unp; 3170 SOCK_LOCK(so); 3171 SOCK_LOCK(so2); 3172 if (wakeup) /* Avoid LOR with receive buffer lock. */ 3173 SOCK_SENDBUF_LOCK(so); 3174 SOCK_RECVBUF_LOCK(so); 3175 SOCK_RECVBUF_LOCK(so2); 3176 unp_soisconnected(so, wakeup); /* Will unlock send buffer. */ 3177 unp_soisconnected(so2, false); 3178 SOCK_RECVBUF_UNLOCK(so); 3179 SOCK_RECVBUF_UNLOCK(so2); 3180 SOCK_UNLOCK(so); 3181 SOCK_UNLOCK(so2); 3182 break; 3183 3184 default: 3185 panic("unp_connect2"); 3186 } 3187 } 3188 3189 static void 3190 unp_soisdisconnected(struct socket *so) 3191 { 3192 SOCK_LOCK_ASSERT(so); 3193 SOCK_RECVBUF_LOCK_ASSERT(so); 3194 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3195 MPASS(!SOLISTENING(so)); 3196 MPASS((so->so_state & (SS_ISCONNECTING | SS_ISDISCONNECTING | 3197 SS_ISDISCONNECTED)) == 0); 3198 MPASS(so->so_state & SS_ISCONNECTED); 3199 3200 so->so_state |= SS_ISDISCONNECTED; 3201 so->so_state &= ~SS_ISCONNECTED; 3202 so->so_rcv.uxst_peer = NULL; 3203 selwakeuppri(&so->so_wrsel, PSOCK); 3204 KNOTE_LOCKED(&so->so_snd.sb_sel->si_note, 0); 3205 socantrcvmore_locked(so); 3206 } 3207 3208 static void 3209 unp_disconnect(struct unpcb *unp, struct unpcb *unp2) 3210 { 3211 struct socket *so, *so2; 3212 struct mbuf *m = NULL; 3213 #ifdef INVARIANTS 3214 struct unpcb *unptmp; 3215 #endif 3216 3217 UNP_PCB_LOCK_ASSERT(unp); 3218 UNP_PCB_LOCK_ASSERT(unp2); 3219 KASSERT(unp->unp_conn == unp2, 3220 ("%s: unpcb %p is not connected to %p", __func__, unp, unp2)); 3221 3222 unp->unp_conn = NULL; 3223 so = unp->unp_socket; 3224 so2 = unp2->unp_socket; 3225 switch (unp->unp_socket->so_type) { 3226 case SOCK_DGRAM: 3227 /* 3228 * Remove our send socket buffer from the peer's receive buffer. 3229 * Move the data to the receive buffer only if it is empty. 3230 * This is a protection against a scenario where a peer 3231 * connects, floods and disconnects, effectively blocking 3232 * sendto() from unconnected sockets. 3233 */ 3234 SOCK_RECVBUF_LOCK(so2); 3235 if (!STAILQ_EMPTY(&so->so_snd.uxdg_mb)) { 3236 TAILQ_REMOVE(&so2->so_rcv.uxdg_conns, &so->so_snd, 3237 uxdg_clist); 3238 if (__predict_true((so2->so_rcv.sb_state & 3239 SBS_CANTRCVMORE) == 0) && 3240 STAILQ_EMPTY(&so2->so_rcv.uxdg_mb)) { 3241 STAILQ_CONCAT(&so2->so_rcv.uxdg_mb, 3242 &so->so_snd.uxdg_mb); 3243 so2->so_rcv.uxdg_cc += so->so_snd.uxdg_cc; 3244 so2->so_rcv.uxdg_ctl += so->so_snd.uxdg_ctl; 3245 so2->so_rcv.uxdg_mbcnt += so->so_snd.uxdg_mbcnt; 3246 } else { 3247 m = STAILQ_FIRST(&so->so_snd.uxdg_mb); 3248 STAILQ_INIT(&so->so_snd.uxdg_mb); 3249 so2->so_rcv.sb_acc -= so->so_snd.uxdg_cc; 3250 so2->so_rcv.sb_ccc -= so->so_snd.uxdg_cc; 3251 so2->so_rcv.sb_ctl -= so->so_snd.uxdg_ctl; 3252 so2->so_rcv.sb_mbcnt -= so->so_snd.uxdg_mbcnt; 3253 } 3254 /* Note: so may reconnect. */ 3255 so->so_snd.uxdg_cc = 0; 3256 so->so_snd.uxdg_ctl = 0; 3257 so->so_snd.uxdg_mbcnt = 0; 3258 } 3259 SOCK_RECVBUF_UNLOCK(so2); 3260 UNP_REF_LIST_LOCK(); 3261 #ifdef INVARIANTS 3262 LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) { 3263 if (unptmp == unp) 3264 break; 3265 } 3266 KASSERT(unptmp != NULL, 3267 ("%s: %p not found in reflist of %p", __func__, unp, unp2)); 3268 #endif 3269 LIST_REMOVE(unp, unp_reflink); 3270 UNP_REF_LIST_UNLOCK(); 3271 SOCK_LOCK(so); 3272 so->so_state &= ~SS_ISCONNECTED; 3273 SOCK_UNLOCK(so); 3274 break; 3275 3276 case SOCK_STREAM: 3277 case SOCK_SEQPACKET: 3278 SOCK_LOCK(so); 3279 SOCK_LOCK(so2); 3280 SOCK_RECVBUF_LOCK(so); 3281 SOCK_RECVBUF_LOCK(so2); 3282 unp_soisdisconnected(so); 3283 MPASS(unp2->unp_conn == unp); 3284 unp2->unp_conn = NULL; 3285 unp_soisdisconnected(so2); 3286 SOCK_UNLOCK(so); 3287 SOCK_UNLOCK(so2); 3288 break; 3289 } 3290 3291 if (unp == unp2) { 3292 unp_pcb_rele_notlast(unp); 3293 if (!unp_pcb_rele(unp)) 3294 UNP_PCB_UNLOCK(unp); 3295 } else { 3296 if (!unp_pcb_rele(unp)) 3297 UNP_PCB_UNLOCK(unp); 3298 if (!unp_pcb_rele(unp2)) 3299 UNP_PCB_UNLOCK(unp2); 3300 } 3301 3302 if (m != NULL) { 3303 unp_scan(m, unp_freerights); 3304 m_freemp(m); 3305 } 3306 } 3307 3308 /* 3309 * unp_pcblist() walks the global list of struct unpcb's to generate a 3310 * pointer list, bumping the refcount on each unpcb. It then copies them out 3311 * sequentially, validating the generation number on each to see if it has 3312 * been detached. All of this is necessary because copyout() may sleep on 3313 * disk I/O. 3314 */ 3315 static int 3316 unp_pcblist(SYSCTL_HANDLER_ARGS) 3317 { 3318 struct unpcb *unp, **unp_list; 3319 unp_gen_t gencnt; 3320 struct xunpgen *xug; 3321 struct unp_head *head; 3322 struct xunpcb *xu; 3323 u_int i; 3324 int error, n; 3325 3326 switch ((intptr_t)arg1) { 3327 case SOCK_STREAM: 3328 head = &unp_shead; 3329 break; 3330 3331 case SOCK_DGRAM: 3332 head = &unp_dhead; 3333 break; 3334 3335 case SOCK_SEQPACKET: 3336 head = &unp_sphead; 3337 break; 3338 3339 default: 3340 panic("unp_pcblist: arg1 %d", (int)(intptr_t)arg1); 3341 } 3342 3343 /* 3344 * The process of preparing the PCB list is too time-consuming and 3345 * resource-intensive to repeat twice on every request. 3346 */ 3347 if (req->oldptr == NULL) { 3348 n = unp_count; 3349 req->oldidx = 2 * (sizeof *xug) 3350 + (n + n/8) * sizeof(struct xunpcb); 3351 return (0); 3352 } 3353 3354 if (req->newptr != NULL) 3355 return (EPERM); 3356 3357 /* 3358 * OK, now we're committed to doing something. 3359 */ 3360 xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK | M_ZERO); 3361 UNP_LINK_RLOCK(); 3362 gencnt = unp_gencnt; 3363 n = unp_count; 3364 UNP_LINK_RUNLOCK(); 3365 3366 xug->xug_len = sizeof *xug; 3367 xug->xug_count = n; 3368 xug->xug_gen = gencnt; 3369 xug->xug_sogen = so_gencnt; 3370 error = SYSCTL_OUT(req, xug, sizeof *xug); 3371 if (error) { 3372 free(xug, M_TEMP); 3373 return (error); 3374 } 3375 3376 unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK); 3377 3378 UNP_LINK_RLOCK(); 3379 for (unp = LIST_FIRST(head), i = 0; unp && i < n; 3380 unp = LIST_NEXT(unp, unp_link)) { 3381 UNP_PCB_LOCK(unp); 3382 if (unp->unp_gencnt <= gencnt) { 3383 if (cr_cansee(req->td->td_ucred, 3384 unp->unp_socket->so_cred)) { 3385 UNP_PCB_UNLOCK(unp); 3386 continue; 3387 } 3388 unp_list[i++] = unp; 3389 unp_pcb_hold(unp); 3390 } 3391 UNP_PCB_UNLOCK(unp); 3392 } 3393 UNP_LINK_RUNLOCK(); 3394 n = i; /* In case we lost some during malloc. */ 3395 3396 error = 0; 3397 xu = malloc(sizeof(*xu), M_TEMP, M_WAITOK | M_ZERO); 3398 for (i = 0; i < n; i++) { 3399 unp = unp_list[i]; 3400 UNP_PCB_LOCK(unp); 3401 if (unp_pcb_rele(unp)) 3402 continue; 3403 3404 if (unp->unp_gencnt <= gencnt) { 3405 xu->xu_len = sizeof *xu; 3406 xu->xu_unpp = (uintptr_t)unp; 3407 /* 3408 * XXX - need more locking here to protect against 3409 * connect/disconnect races for SMP. 3410 */ 3411 if (unp->unp_addr != NULL) 3412 bcopy(unp->unp_addr, &xu->xu_addr, 3413 unp->unp_addr->sun_len); 3414 else 3415 bzero(&xu->xu_addr, sizeof(xu->xu_addr)); 3416 if (unp->unp_conn != NULL && 3417 unp->unp_conn->unp_addr != NULL) 3418 bcopy(unp->unp_conn->unp_addr, 3419 &xu->xu_caddr, 3420 unp->unp_conn->unp_addr->sun_len); 3421 else 3422 bzero(&xu->xu_caddr, sizeof(xu->xu_caddr)); 3423 xu->unp_vnode = (uintptr_t)unp->unp_vnode; 3424 xu->unp_conn = (uintptr_t)unp->unp_conn; 3425 xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs); 3426 xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink); 3427 xu->unp_gencnt = unp->unp_gencnt; 3428 sotoxsocket(unp->unp_socket, &xu->xu_socket); 3429 UNP_PCB_UNLOCK(unp); 3430 error = SYSCTL_OUT(req, xu, sizeof *xu); 3431 } else { 3432 UNP_PCB_UNLOCK(unp); 3433 } 3434 } 3435 free(xu, M_TEMP); 3436 if (!error) { 3437 /* 3438 * Give the user an updated idea of our state. If the 3439 * generation differs from what we told her before, she knows 3440 * that something happened while we were processing this 3441 * request, and it might be necessary to retry. 3442 */ 3443 xug->xug_gen = unp_gencnt; 3444 xug->xug_sogen = so_gencnt; 3445 xug->xug_count = unp_count; 3446 error = SYSCTL_OUT(req, xug, sizeof *xug); 3447 } 3448 free(unp_list, M_TEMP); 3449 free(xug, M_TEMP); 3450 return (error); 3451 } 3452 3453 SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, 3454 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3455 (void *)(intptr_t)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb", 3456 "List of active local datagram sockets"); 3457 SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, 3458 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3459 (void *)(intptr_t)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb", 3460 "List of active local stream sockets"); 3461 SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist, 3462 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3463 (void *)(intptr_t)SOCK_SEQPACKET, 0, unp_pcblist, "S,xunpcb", 3464 "List of active local seqpacket sockets"); 3465 3466 static void 3467 unp_drop(struct unpcb *unp) 3468 { 3469 struct socket *so; 3470 struct unpcb *unp2; 3471 3472 /* 3473 * Regardless of whether the socket's peer dropped the connection 3474 * with this socket by aborting or disconnecting, POSIX requires 3475 * that ECONNRESET is returned on next connected send(2) in case of 3476 * a SOCK_DGRAM socket and EPIPE for SOCK_STREAM. 3477 */ 3478 UNP_PCB_LOCK(unp); 3479 if ((so = unp->unp_socket) != NULL) 3480 so->so_error = 3481 so->so_proto->pr_type == SOCK_DGRAM ? ECONNRESET : EPIPE; 3482 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) { 3483 /* Last reference dropped in unp_disconnect(). */ 3484 unp_pcb_rele_notlast(unp); 3485 unp_disconnect(unp, unp2); 3486 } else if (!unp_pcb_rele(unp)) { 3487 UNP_PCB_UNLOCK(unp); 3488 } 3489 } 3490 3491 static void 3492 unp_freerights(struct filedescent **fdep, int fdcount) 3493 { 3494 struct file *fp; 3495 int i; 3496 3497 KASSERT(fdcount > 0, ("%s: fdcount %d", __func__, fdcount)); 3498 3499 for (i = 0; i < fdcount; i++) { 3500 fp = fdep[i]->fde_file; 3501 filecaps_free(&fdep[i]->fde_caps); 3502 unp_discard(fp); 3503 } 3504 free(fdep[0], M_FILECAPS); 3505 } 3506 3507 /* 3508 * Flags to set on the receiving side when externalizing a file descriptor. 3509 * When transferring fds between jails, ensure that the receiver cannot use 3510 * a dirfd to escape the jail chroot. 3511 */ 3512 static int 3513 externalize_fdflags(struct filedescent *fde, struct thread *td) 3514 { 3515 struct prison *prison1, *prison2; 3516 3517 if ((fde->fde_flags & UF_RESOLVE_BENEATH) != 0) 3518 return (O_RESOLVE_BENEATH); 3519 prison1 = fde->fde_file->f_cred->cr_prison; 3520 prison2 = td->td_ucred->cr_prison; 3521 if (prison1 != prison2 && prison1->pr_root != prison2->pr_root && 3522 prison2 != &prison0) 3523 return (O_RESOLVE_BENEATH); 3524 else 3525 return (0); 3526 } 3527 3528 static int 3529 unp_externalize(const struct socket *so, struct mbuf *control, 3530 struct mbuf **controlp, int flags) 3531 { 3532 struct thread *td = curthread; /* XXX */ 3533 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 3534 int *fdp; 3535 struct filedesc *fdesc = td->td_proc->p_fd; 3536 struct filedescent **fdep; 3537 void *data; 3538 socklen_t clen = control->m_len, datalen; 3539 int error, fdflags, newfds; 3540 u_int newlen; 3541 3542 UNP_LINK_UNLOCK_ASSERT(); 3543 3544 fdflags = ((flags & MSG_CMSG_CLOEXEC) ? O_CLOEXEC : 0) | 3545 ((flags & MSG_CMSG_CLOFORK) ? O_CLOFORK : 0); 3546 3547 error = 0; 3548 if (controlp != NULL) /* controlp == NULL => free control messages */ 3549 *controlp = NULL; 3550 while (cm != NULL) { 3551 MPASS(clen >= sizeof(*cm) && clen >= cm->cmsg_len); 3552 3553 data = CMSG_DATA(cm); 3554 datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; 3555 if (cm->cmsg_level == SOL_SOCKET 3556 && cm->cmsg_type == SCM_RIGHTS) { 3557 newfds = datalen / sizeof(*fdep); 3558 if (newfds == 0) 3559 goto next; 3560 fdep = data; 3561 3562 /* 3563 * If we're not outputting the descriptors, free them. 3564 * 3565 * In the case of having revoked SCM_PASSRIGHTS, the 3566 * receiver must have toggled it before trying to 3567 * receive control messages- we'll take that as a signal 3568 * that they didn't want these, but they raced against 3569 * the sender trying to pass files anyways. 3570 */ 3571 if (error || controlp == NULL || 3572 (atomic_load_int(&so->so_options) & 3573 SO_PASSRIGHTS) == 0) { 3574 unp_freerights(fdep, newfds); 3575 goto next; 3576 } 3577 FILEDESC_XLOCK(fdesc); 3578 3579 /* 3580 * Now change each pointer to an fd in the global 3581 * table to an integer that is the index to the local 3582 * fd table entry that we set up to point to the 3583 * global one we are transferring. 3584 */ 3585 newlen = newfds * sizeof(int); 3586 *controlp = sbcreatecontrol(NULL, newlen, 3587 SCM_RIGHTS, SOL_SOCKET, M_WAITOK); 3588 3589 fdp = (int *) 3590 CMSG_DATA(mtod(*controlp, struct cmsghdr *)); 3591 if ((error = fdallocn(td, 0, fdp, newfds))) { 3592 FILEDESC_XUNLOCK(fdesc); 3593 unp_freerights(fdep, newfds); 3594 m_freem(*controlp); 3595 *controlp = NULL; 3596 goto next; 3597 } 3598 for (int i = 0; i < newfds; i++, fdp++) { 3599 struct file *fp; 3600 3601 fp = fdep[i]->fde_file; 3602 _finstall(fdesc, fp, *fdp, 3603 fdflags | externalize_fdflags(fdep[i], td), 3604 &fdep[i]->fde_caps); 3605 unp_externalize_fp(fp); 3606 } 3607 3608 /* 3609 * The new type indicates that the mbuf data refers to 3610 * kernel resources that may need to be released before 3611 * the mbuf is freed. 3612 */ 3613 m_chtype(*controlp, MT_EXTCONTROL); 3614 FILEDESC_XUNLOCK(fdesc); 3615 free(fdep[0], M_FILECAPS); 3616 } else { 3617 /* We can just copy anything else across. */ 3618 if (error || controlp == NULL) 3619 goto next; 3620 *controlp = sbcreatecontrol(NULL, datalen, 3621 cm->cmsg_type, cm->cmsg_level, M_WAITOK); 3622 bcopy(data, 3623 CMSG_DATA(mtod(*controlp, struct cmsghdr *)), 3624 datalen); 3625 } 3626 controlp = &(*controlp)->m_next; 3627 3628 next: 3629 if (CMSG_SPACE(datalen) < clen) { 3630 clen -= CMSG_SPACE(datalen); 3631 cm = (struct cmsghdr *) 3632 ((caddr_t)cm + CMSG_SPACE(datalen)); 3633 } else { 3634 clen = 0; 3635 cm = NULL; 3636 } 3637 } 3638 3639 return (error); 3640 } 3641 3642 static void 3643 unp_zone_change(void *tag) 3644 { 3645 3646 uma_zone_set_max(unp_zone, maxsockets); 3647 } 3648 3649 #ifdef INVARIANTS 3650 static void 3651 unp_zdtor(void *mem, int size __unused, void *arg __unused) 3652 { 3653 struct unpcb *unp; 3654 3655 unp = mem; 3656 3657 KASSERT(LIST_EMPTY(&unp->unp_refs), 3658 ("%s: unpcb %p has lingering refs", __func__, unp)); 3659 KASSERT(unp->unp_socket == NULL, 3660 ("%s: unpcb %p has socket backpointer", __func__, unp)); 3661 KASSERT(unp->unp_vnode == NULL, 3662 ("%s: unpcb %p has vnode references", __func__, unp)); 3663 KASSERT(unp->unp_conn == NULL, 3664 ("%s: unpcb %p is still connected", __func__, unp)); 3665 KASSERT(unp->unp_addr == NULL, 3666 ("%s: unpcb %p has leaked addr", __func__, unp)); 3667 } 3668 #endif 3669 3670 static void 3671 unp_init(void *arg __unused) 3672 { 3673 uma_dtor dtor; 3674 3675 #ifdef INVARIANTS 3676 dtor = unp_zdtor; 3677 #else 3678 dtor = NULL; 3679 #endif 3680 unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, dtor, 3681 NULL, NULL, UMA_ALIGN_CACHE, 0); 3682 uma_zone_set_max(unp_zone, maxsockets); 3683 uma_zone_set_warning(unp_zone, "kern.ipc.maxsockets limit reached"); 3684 EVENTHANDLER_REGISTER(maxsockets_change, unp_zone_change, 3685 NULL, EVENTHANDLER_PRI_ANY); 3686 LIST_INIT(&unp_dhead); 3687 LIST_INIT(&unp_shead); 3688 LIST_INIT(&unp_sphead); 3689 SLIST_INIT(&unp_defers); 3690 TIMEOUT_TASK_INIT(taskqueue_thread, &unp_gc_task, 0, unp_gc, NULL); 3691 TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL); 3692 UNP_LINK_LOCK_INIT(); 3693 UNP_DEFERRED_LOCK_INIT(); 3694 unp_vp_mtxpool = mtx_pool_create("unp vp mtxpool", 32, MTX_DEF); 3695 } 3696 SYSINIT(unp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_SECOND, unp_init, NULL); 3697 3698 static void 3699 unp_internalize_cleanup_rights(struct mbuf *control) 3700 { 3701 struct cmsghdr *cp; 3702 struct mbuf *m; 3703 void *data; 3704 socklen_t datalen; 3705 3706 for (m = control; m != NULL; m = m->m_next) { 3707 cp = mtod(m, struct cmsghdr *); 3708 if (cp->cmsg_level != SOL_SOCKET || 3709 cp->cmsg_type != SCM_RIGHTS) 3710 continue; 3711 data = CMSG_DATA(cp); 3712 datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data; 3713 unp_freerights(data, datalen / sizeof(struct filedesc *)); 3714 } 3715 } 3716 3717 static int 3718 unp_internalize(struct mbuf *control, struct mchain *mc, struct thread *td, 3719 int *needsopts) 3720 { 3721 struct proc *p; 3722 struct filedesc *fdesc; 3723 struct bintime *bt; 3724 struct cmsghdr *cm; 3725 struct cmsgcred *cmcred; 3726 struct mbuf *m; 3727 struct filedescent *fde, **fdep, *fdev; 3728 struct file *fp; 3729 struct timeval *tv; 3730 struct timespec *ts; 3731 void *data; 3732 socklen_t clen, datalen; 3733 int i, j, error, *fdp, oldfds; 3734 u_int newlen; 3735 3736 MPASS(control->m_next == NULL); /* COMPAT_OLDSOCK may violate */ 3737 UNP_LINK_UNLOCK_ASSERT(); 3738 3739 p = td->td_proc; 3740 fdesc = p->p_fd; 3741 error = 0; 3742 *mc = MCHAIN_INITIALIZER(mc); 3743 for (clen = control->m_len, cm = mtod(control, struct cmsghdr *), 3744 data = CMSG_DATA(cm); 3745 3746 clen >= sizeof(*cm) && cm->cmsg_level == SOL_SOCKET && 3747 clen >= cm->cmsg_len && cm->cmsg_len >= sizeof(*cm) && 3748 (char *)cm + cm->cmsg_len >= (char *)data; 3749 3750 clen -= min(CMSG_SPACE(datalen), clen), 3751 cm = (struct cmsghdr *) ((char *)cm + CMSG_SPACE(datalen)), 3752 data = CMSG_DATA(cm)) { 3753 datalen = (char *)cm + cm->cmsg_len - (char *)data; 3754 switch (cm->cmsg_type) { 3755 case SCM_CREDS: 3756 m = sbcreatecontrol(NULL, sizeof(*cmcred), SCM_CREDS, 3757 SOL_SOCKET, M_WAITOK); 3758 cmcred = (struct cmsgcred *) 3759 CMSG_DATA(mtod(m, struct cmsghdr *)); 3760 cmcred->cmcred_pid = p->p_pid; 3761 cmcred->cmcred_uid = td->td_ucred->cr_ruid; 3762 cmcred->cmcred_gid = td->td_ucred->cr_rgid; 3763 cmcred->cmcred_euid = td->td_ucred->cr_uid; 3764 _Static_assert(CMGROUP_MAX >= 1, 3765 "Room needed for the effective GID."); 3766 cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups + 1, 3767 CMGROUP_MAX); 3768 cmcred->cmcred_groups[0] = td->td_ucred->cr_gid; 3769 for (i = 1; i < cmcred->cmcred_ngroups; i++) 3770 cmcred->cmcred_groups[i] = 3771 td->td_ucred->cr_groups[i - 1]; 3772 break; 3773 3774 case SCM_RIGHTS: 3775 *needsopts |= SO_PASSRIGHTS; 3776 oldfds = datalen / sizeof (int); 3777 if (oldfds == 0) 3778 continue; 3779 /* On some machines sizeof pointer is bigger than 3780 * sizeof int, so we need to check if data fits into 3781 * single mbuf. We could allocate several mbufs, and 3782 * unp_externalize() should even properly handle that. 3783 * But it is not worth to complicate the code for an 3784 * insane scenario of passing over 200 file descriptors 3785 * at once. 3786 */ 3787 newlen = oldfds * sizeof(fdep[0]); 3788 if (CMSG_SPACE(newlen) > MCLBYTES) { 3789 error = EMSGSIZE; 3790 goto out; 3791 } 3792 /* 3793 * Check that all the FDs passed in refer to legal 3794 * files. If not, reject the entire operation. 3795 */ 3796 fdp = data; 3797 FILEDESC_SLOCK(fdesc); 3798 for (i = 0; i < oldfds; i++, fdp++) { 3799 fp = fget_noref(fdesc, *fdp); 3800 if (fp == NULL) { 3801 FILEDESC_SUNLOCK(fdesc); 3802 error = EBADF; 3803 goto out; 3804 } 3805 if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) { 3806 FILEDESC_SUNLOCK(fdesc); 3807 error = EOPNOTSUPP; 3808 goto out; 3809 } 3810 } 3811 3812 /* 3813 * Now replace the integer FDs with pointers to the 3814 * file structure and capability rights. 3815 */ 3816 m = sbcreatecontrol(NULL, newlen, SCM_RIGHTS, 3817 SOL_SOCKET, M_WAITOK); 3818 fdp = data; 3819 for (i = 0; i < oldfds; i++, fdp++) { 3820 if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) { 3821 fdp = data; 3822 for (j = 0; j < i; j++, fdp++) { 3823 fdrop(fdesc->fd_ofiles[*fdp]. 3824 fde_file, td); 3825 } 3826 FILEDESC_SUNLOCK(fdesc); 3827 error = EBADF; 3828 goto out; 3829 } 3830 } 3831 fdp = data; 3832 fdep = (struct filedescent **) 3833 CMSG_DATA(mtod(m, struct cmsghdr *)); 3834 fdev = malloc(sizeof(*fdev) * oldfds, M_FILECAPS, 3835 M_WAITOK); 3836 for (i = 0; i < oldfds; i++, fdev++, fdp++) { 3837 fde = &fdesc->fd_ofiles[*fdp]; 3838 fdep[i] = fdev; 3839 fdep[i]->fde_file = fde->fde_file; 3840 filecaps_copy(&fde->fde_caps, 3841 &fdep[i]->fde_caps, true); 3842 fdep[i]->fde_flags = fde->fde_flags; 3843 unp_internalize_fp(fdep[i]->fde_file); 3844 } 3845 FILEDESC_SUNLOCK(fdesc); 3846 break; 3847 3848 case SCM_TIMESTAMP: 3849 m = sbcreatecontrol(NULL, sizeof(*tv), SCM_TIMESTAMP, 3850 SOL_SOCKET, M_WAITOK); 3851 tv = (struct timeval *) 3852 CMSG_DATA(mtod(m, struct cmsghdr *)); 3853 microtime(tv); 3854 break; 3855 3856 case SCM_BINTIME: 3857 m = sbcreatecontrol(NULL, sizeof(*bt), SCM_BINTIME, 3858 SOL_SOCKET, M_WAITOK); 3859 bt = (struct bintime *) 3860 CMSG_DATA(mtod(m, struct cmsghdr *)); 3861 bintime(bt); 3862 break; 3863 3864 case SCM_REALTIME: 3865 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_REALTIME, 3866 SOL_SOCKET, M_WAITOK); 3867 ts = (struct timespec *) 3868 CMSG_DATA(mtod(m, struct cmsghdr *)); 3869 nanotime(ts); 3870 break; 3871 3872 case SCM_MONOTONIC: 3873 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_MONOTONIC, 3874 SOL_SOCKET, M_WAITOK); 3875 ts = (struct timespec *) 3876 CMSG_DATA(mtod(m, struct cmsghdr *)); 3877 nanouptime(ts); 3878 break; 3879 3880 default: 3881 error = EINVAL; 3882 goto out; 3883 } 3884 3885 mc_append(mc, m); 3886 } 3887 if (clen > 0) 3888 error = EINVAL; 3889 3890 out: 3891 if (error != 0) 3892 unp_internalize_cleanup_rights(mc_first(mc)); 3893 m_freem(control); 3894 return (error); 3895 } 3896 3897 static void 3898 unp_addsockcred(struct thread *td, struct mchain *mc, int mode) 3899 { 3900 struct mbuf *m, *n, *n_prev; 3901 const struct cmsghdr *cm; 3902 int ngroups, i, cmsgtype; 3903 size_t ctrlsz; 3904 3905 ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX); 3906 if (mode & UNP_WANTCRED_ALWAYS) { 3907 ctrlsz = SOCKCRED2SIZE(ngroups); 3908 cmsgtype = SCM_CREDS2; 3909 } else { 3910 ctrlsz = SOCKCREDSIZE(ngroups); 3911 cmsgtype = SCM_CREDS; 3912 } 3913 3914 /* XXXGL: uipc_sosend_*() need to be improved so that we can M_WAITOK */ 3915 m = sbcreatecontrol(NULL, ctrlsz, cmsgtype, SOL_SOCKET, M_NOWAIT); 3916 if (m == NULL) 3917 return; 3918 MPASS((m->m_flags & M_EXT) == 0 && m->m_next == NULL); 3919 3920 if (mode & UNP_WANTCRED_ALWAYS) { 3921 struct sockcred2 *sc; 3922 3923 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3924 sc->sc_version = 0; 3925 sc->sc_pid = td->td_proc->p_pid; 3926 sc->sc_uid = td->td_ucred->cr_ruid; 3927 sc->sc_euid = td->td_ucred->cr_uid; 3928 sc->sc_gid = td->td_ucred->cr_rgid; 3929 sc->sc_egid = td->td_ucred->cr_gid; 3930 sc->sc_ngroups = ngroups; 3931 for (i = 0; i < sc->sc_ngroups; i++) 3932 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3933 } else { 3934 struct sockcred *sc; 3935 3936 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3937 sc->sc_uid = td->td_ucred->cr_ruid; 3938 sc->sc_euid = td->td_ucred->cr_uid; 3939 sc->sc_gid = td->td_ucred->cr_rgid; 3940 sc->sc_egid = td->td_ucred->cr_gid; 3941 sc->sc_ngroups = ngroups; 3942 for (i = 0; i < sc->sc_ngroups; i++) 3943 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3944 } 3945 3946 /* 3947 * Unlink SCM_CREDS control messages (struct cmsgcred), since just 3948 * created SCM_CREDS control message (struct sockcred) has another 3949 * format. 3950 */ 3951 if (!STAILQ_EMPTY(&mc->mc_q) && cmsgtype == SCM_CREDS) 3952 STAILQ_FOREACH_SAFE(n, &mc->mc_q, m_stailq, n_prev) { 3953 cm = mtod(n, struct cmsghdr *); 3954 if (cm->cmsg_level == SOL_SOCKET && 3955 cm->cmsg_type == SCM_CREDS) { 3956 mc_remove(mc, n); 3957 m_free(n); 3958 } 3959 } 3960 3961 /* Prepend it to the head. */ 3962 mc_prepend(mc, m); 3963 } 3964 3965 static struct unpcb * 3966 fptounp(struct file *fp) 3967 { 3968 struct socket *so; 3969 3970 if (fp->f_type != DTYPE_SOCKET) 3971 return (NULL); 3972 if ((so = fp->f_data) == NULL) 3973 return (NULL); 3974 if (so->so_proto->pr_domain != &localdomain) 3975 return (NULL); 3976 return sotounpcb(so); 3977 } 3978 3979 static void 3980 unp_discard(struct file *fp) 3981 { 3982 struct unp_defer *dr; 3983 3984 if (unp_externalize_fp(fp)) { 3985 dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK); 3986 dr->ud_fp = fp; 3987 UNP_DEFERRED_LOCK(); 3988 SLIST_INSERT_HEAD(&unp_defers, dr, ud_link); 3989 UNP_DEFERRED_UNLOCK(); 3990 atomic_add_int(&unp_defers_count, 1); 3991 taskqueue_enqueue(taskqueue_thread, &unp_defer_task); 3992 } else 3993 closef_nothread(fp); 3994 } 3995 3996 static void 3997 unp_process_defers(void *arg __unused, int pending) 3998 { 3999 struct unp_defer *dr; 4000 SLIST_HEAD(, unp_defer) drl; 4001 int count; 4002 4003 SLIST_INIT(&drl); 4004 for (;;) { 4005 UNP_DEFERRED_LOCK(); 4006 if (SLIST_FIRST(&unp_defers) == NULL) { 4007 UNP_DEFERRED_UNLOCK(); 4008 break; 4009 } 4010 SLIST_SWAP(&unp_defers, &drl, unp_defer); 4011 UNP_DEFERRED_UNLOCK(); 4012 count = 0; 4013 while ((dr = SLIST_FIRST(&drl)) != NULL) { 4014 SLIST_REMOVE_HEAD(&drl, ud_link); 4015 closef_nothread(dr->ud_fp); 4016 free(dr, M_TEMP); 4017 count++; 4018 } 4019 atomic_add_int(&unp_defers_count, -count); 4020 } 4021 } 4022 4023 static void 4024 unp_internalize_fp(struct file *fp) 4025 { 4026 struct unpcb *unp; 4027 4028 UNP_LINK_WLOCK(); 4029 if ((unp = fptounp(fp)) != NULL) { 4030 unp->unp_file = fp; 4031 unp->unp_msgcount++; 4032 } 4033 unp_rights++; 4034 UNP_LINK_WUNLOCK(); 4035 } 4036 4037 static int 4038 unp_externalize_fp(struct file *fp) 4039 { 4040 struct unpcb *unp; 4041 int ret; 4042 4043 UNP_LINK_WLOCK(); 4044 if ((unp = fptounp(fp)) != NULL) { 4045 unp->unp_msgcount--; 4046 ret = 1; 4047 } else 4048 ret = 0; 4049 unp_rights--; 4050 UNP_LINK_WUNLOCK(); 4051 return (ret); 4052 } 4053 4054 /* 4055 * unp_defer indicates whether additional work has been defered for a future 4056 * pass through unp_gc(). It is thread local and does not require explicit 4057 * synchronization. 4058 */ 4059 static int unp_marked; 4060 4061 static void 4062 unp_remove_dead_ref(struct filedescent **fdep, int fdcount) 4063 { 4064 struct unpcb *unp; 4065 struct file *fp; 4066 int i; 4067 4068 /* 4069 * This function can only be called from the gc task. 4070 */ 4071 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4072 ("%s: not on gc callout", __func__)); 4073 UNP_LINK_LOCK_ASSERT(); 4074 4075 for (i = 0; i < fdcount; i++) { 4076 fp = fdep[i]->fde_file; 4077 if ((unp = fptounp(fp)) == NULL) 4078 continue; 4079 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4080 continue; 4081 unp->unp_gcrefs--; 4082 } 4083 } 4084 4085 static void 4086 unp_restore_undead_ref(struct filedescent **fdep, int fdcount) 4087 { 4088 struct unpcb *unp; 4089 struct file *fp; 4090 int i; 4091 4092 /* 4093 * This function can only be called from the gc task. 4094 */ 4095 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4096 ("%s: not on gc callout", __func__)); 4097 UNP_LINK_LOCK_ASSERT(); 4098 4099 for (i = 0; i < fdcount; i++) { 4100 fp = fdep[i]->fde_file; 4101 if ((unp = fptounp(fp)) == NULL) 4102 continue; 4103 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4104 continue; 4105 unp->unp_gcrefs++; 4106 unp_marked++; 4107 } 4108 } 4109 4110 static void 4111 unp_scan_socket(struct socket *so, void (*op)(struct filedescent **, int)) 4112 { 4113 struct sockbuf *sb; 4114 4115 SOCK_LOCK_ASSERT(so); 4116 4117 if (sotounpcb(so)->unp_gcflag & UNPGC_IGNORE_RIGHTS) 4118 return; 4119 4120 SOCK_RECVBUF_LOCK(so); 4121 switch (so->so_type) { 4122 case SOCK_DGRAM: 4123 unp_scan(STAILQ_FIRST(&so->so_rcv.uxdg_mb), op); 4124 unp_scan(so->so_rcv.uxdg_peeked, op); 4125 TAILQ_FOREACH(sb, &so->so_rcv.uxdg_conns, uxdg_clist) 4126 unp_scan(STAILQ_FIRST(&sb->uxdg_mb), op); 4127 break; 4128 case SOCK_STREAM: 4129 case SOCK_SEQPACKET: 4130 unp_scan(STAILQ_FIRST(&so->so_rcv.uxst_mbq), op); 4131 break; 4132 } 4133 SOCK_RECVBUF_UNLOCK(so); 4134 } 4135 4136 static void 4137 unp_gc_scan(struct unpcb *unp, void (*op)(struct filedescent **, int)) 4138 { 4139 struct socket *so, *soa; 4140 4141 so = unp->unp_socket; 4142 SOCK_LOCK(so); 4143 if (SOLISTENING(so)) { 4144 /* 4145 * Mark all sockets in our accept queue. 4146 */ 4147 TAILQ_FOREACH(soa, &so->sol_comp, so_list) 4148 unp_scan_socket(soa, op); 4149 } else { 4150 /* 4151 * Mark all sockets we reference with RIGHTS. 4152 */ 4153 unp_scan_socket(so, op); 4154 } 4155 SOCK_UNLOCK(so); 4156 } 4157 4158 static int unp_recycled; 4159 SYSCTL_INT(_net_local, OID_AUTO, recycled, CTLFLAG_RD, &unp_recycled, 0, 4160 "Number of unreachable sockets claimed by the garbage collector."); 4161 4162 static int unp_taskcount; 4163 SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0, 4164 "Number of times the garbage collector has run."); 4165 4166 SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0, 4167 "Number of active local sockets."); 4168 4169 static void 4170 unp_gc(__unused void *arg, int pending) 4171 { 4172 struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead, 4173 NULL }; 4174 struct unp_head **head; 4175 struct unp_head unp_deadhead; /* List of potentially-dead sockets. */ 4176 struct file *f, **unref; 4177 struct unpcb *unp, *unptmp; 4178 int i, total, unp_unreachable; 4179 4180 LIST_INIT(&unp_deadhead); 4181 unp_taskcount++; 4182 UNP_LINK_RLOCK(); 4183 /* 4184 * First determine which sockets may be in cycles. 4185 */ 4186 unp_unreachable = 0; 4187 4188 for (head = heads; *head != NULL; head++) 4189 LIST_FOREACH(unp, *head, unp_link) { 4190 KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0, 4191 ("%s: unp %p has unexpected gc flags 0x%x", 4192 __func__, unp, (unsigned int)unp->unp_gcflag)); 4193 4194 f = unp->unp_file; 4195 4196 /* 4197 * Check for an unreachable socket potentially in a 4198 * cycle. It must be in a queue as indicated by 4199 * msgcount, and this must equal the file reference 4200 * count. Note that when msgcount is 0 the file is 4201 * NULL. 4202 */ 4203 if (f != NULL && unp->unp_msgcount != 0 && 4204 refcount_load(&f->f_count) == unp->unp_msgcount) { 4205 LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead); 4206 unp->unp_gcflag |= UNPGC_DEAD; 4207 unp->unp_gcrefs = unp->unp_msgcount; 4208 unp_unreachable++; 4209 } 4210 } 4211 4212 /* 4213 * Scan all sockets previously marked as potentially being in a cycle 4214 * and remove the references each socket holds on any UNPGC_DEAD 4215 * sockets in its queue. After this step, all remaining references on 4216 * sockets marked UNPGC_DEAD should not be part of any cycle. 4217 */ 4218 LIST_FOREACH(unp, &unp_deadhead, unp_dead) 4219 unp_gc_scan(unp, unp_remove_dead_ref); 4220 4221 /* 4222 * If a socket still has a non-negative refcount, it cannot be in a 4223 * cycle. In this case increment refcount of all children iteratively. 4224 * Stop the scan once we do a complete loop without discovering 4225 * a new reachable socket. 4226 */ 4227 do { 4228 unp_marked = 0; 4229 LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp) 4230 if (unp->unp_gcrefs > 0) { 4231 unp->unp_gcflag &= ~UNPGC_DEAD; 4232 LIST_REMOVE(unp, unp_dead); 4233 KASSERT(unp_unreachable > 0, 4234 ("%s: unp_unreachable underflow.", 4235 __func__)); 4236 unp_unreachable--; 4237 unp_gc_scan(unp, unp_restore_undead_ref); 4238 } 4239 } while (unp_marked); 4240 4241 UNP_LINK_RUNLOCK(); 4242 4243 if (unp_unreachable == 0) 4244 return; 4245 4246 /* 4247 * Allocate space for a local array of dead unpcbs. 4248 * TODO: can this path be simplified by instead using the local 4249 * dead list at unp_deadhead, after taking out references 4250 * on the file object and/or unpcb and dropping the link lock? 4251 */ 4252 unref = malloc(unp_unreachable * sizeof(struct file *), 4253 M_TEMP, M_WAITOK); 4254 4255 /* 4256 * Iterate looking for sockets which have been specifically marked 4257 * as unreachable and store them locally. 4258 */ 4259 UNP_LINK_RLOCK(); 4260 total = 0; 4261 LIST_FOREACH(unp, &unp_deadhead, unp_dead) { 4262 KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0, 4263 ("%s: unp %p not marked UNPGC_DEAD", __func__, unp)); 4264 unp->unp_gcflag &= ~UNPGC_DEAD; 4265 f = unp->unp_file; 4266 if (unp->unp_msgcount == 0 || f == NULL || 4267 refcount_load(&f->f_count) != unp->unp_msgcount || 4268 !fhold(f)) 4269 continue; 4270 unref[total++] = f; 4271 KASSERT(total <= unp_unreachable, 4272 ("%s: incorrect unreachable count.", __func__)); 4273 } 4274 UNP_LINK_RUNLOCK(); 4275 4276 /* 4277 * Now flush all sockets, free'ing rights. This will free the 4278 * struct files associated with these sockets but leave each socket 4279 * with one remaining ref. 4280 */ 4281 for (i = 0; i < total; i++) { 4282 struct socket *so; 4283 4284 so = unref[i]->f_data; 4285 if (!SOLISTENING(so)) { 4286 CURVNET_SET(so->so_vnet); 4287 socantrcvmore(so); 4288 unp_dispose(so); 4289 CURVNET_RESTORE(); 4290 } 4291 } 4292 4293 /* 4294 * And finally release the sockets so they can be reclaimed. 4295 */ 4296 for (i = 0; i < total; i++) 4297 fdrop(unref[i], NULL); 4298 unp_recycled += total; 4299 free(unref, M_TEMP); 4300 } 4301 4302 /* 4303 * Synchronize against unp_gc, which can trip over data as we are freeing it. 4304 */ 4305 static void 4306 unp_dispose(struct socket *so) 4307 { 4308 struct sockbuf *sb; 4309 struct unpcb *unp; 4310 struct mbuf *m; 4311 int error __diagused; 4312 4313 MPASS(!SOLISTENING(so)); 4314 4315 unp = sotounpcb(so); 4316 UNP_LINK_WLOCK(); 4317 unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS; 4318 UNP_LINK_WUNLOCK(); 4319 4320 /* 4321 * Grab our special mbufs before calling sbrelease(). 4322 */ 4323 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 4324 MPASS(!error); 4325 SOCK_RECVBUF_LOCK(so); 4326 switch (so->so_type) { 4327 case SOCK_DGRAM: 4328 while ((sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) != NULL) { 4329 STAILQ_CONCAT(&so->so_rcv.uxdg_mb, &sb->uxdg_mb); 4330 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 4331 /* Note: socket of sb may reconnect. */ 4332 sb->uxdg_cc = sb->uxdg_ctl = sb->uxdg_mbcnt = 0; 4333 } 4334 sb = &so->so_rcv; 4335 if (sb->uxdg_peeked != NULL) { 4336 STAILQ_INSERT_HEAD(&sb->uxdg_mb, sb->uxdg_peeked, 4337 m_stailqpkt); 4338 sb->uxdg_peeked = NULL; 4339 } 4340 m = STAILQ_FIRST(&sb->uxdg_mb); 4341 STAILQ_INIT(&sb->uxdg_mb); 4342 break; 4343 case SOCK_STREAM: 4344 case SOCK_SEQPACKET: 4345 sb = &so->so_rcv; 4346 m = STAILQ_FIRST(&sb->uxst_mbq); 4347 STAILQ_INIT(&sb->uxst_mbq); 4348 sb->sb_acc = sb->sb_ccc = sb->sb_ctl = sb->sb_mbcnt = 0; 4349 /* 4350 * Trim M_NOTREADY buffers from the free list. They are 4351 * referenced by the I/O thread. 4352 */ 4353 if (sb->uxst_fnrdy != NULL) { 4354 struct mbuf *n, *prev; 4355 4356 while (m != NULL && m->m_flags & M_NOTREADY) 4357 m = m->m_next; 4358 for (prev = n = m; n != NULL; n = n->m_next) { 4359 if (n->m_flags & M_NOTREADY) 4360 prev->m_next = n->m_next; 4361 else 4362 prev = n; 4363 } 4364 sb->uxst_fnrdy = NULL; 4365 } 4366 break; 4367 } 4368 /* 4369 * Mark sb with SBS_CANTRCVMORE. This is needed to prevent 4370 * uipc_sosend_*() or unp_disconnect() adding more data to the socket. 4371 * We came here either through shutdown(2) or from the final sofree(). 4372 * The sofree() case is simple as it guarantees that no more sends will 4373 * happen, however we can race with unp_disconnect() from our peer. 4374 * The shutdown(2) case is more exotic. It would call into 4375 * unp_dispose() only if socket is SS_ISCONNECTED. This is possible if 4376 * we did connect(2) on this socket and we also had it bound with 4377 * bind(2) and receive connections from other sockets. Because 4378 * uipc_shutdown() violates POSIX (see comment there) this applies to 4379 * SOCK_DGRAM as well. For SOCK_DGRAM this SBS_CANTRCVMORE will have 4380 * affect not only on the peer we connect(2)ed to, but also on all of 4381 * the peers who had connect(2)ed to us. Their sends would end up 4382 * with ENOBUFS. 4383 */ 4384 sb->sb_state |= SBS_CANTRCVMORE; 4385 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 4386 RLIM_INFINITY); 4387 SOCK_RECVBUF_UNLOCK(so); 4388 SOCK_IO_RECV_UNLOCK(so); 4389 4390 if (m != NULL) { 4391 unp_scan(m, unp_freerights); 4392 m_freemp(m); 4393 } 4394 } 4395 4396 static void 4397 unp_scan(struct mbuf *m0, void (*op)(struct filedescent **, int)) 4398 { 4399 struct mbuf *m; 4400 struct cmsghdr *cm; 4401 void *data; 4402 socklen_t clen, datalen; 4403 4404 while (m0 != NULL) { 4405 for (m = m0; m; m = m->m_next) { 4406 if (m->m_type != MT_CONTROL) 4407 continue; 4408 4409 cm = mtod(m, struct cmsghdr *); 4410 clen = m->m_len; 4411 4412 while (cm != NULL) { 4413 if (sizeof(*cm) > clen || cm->cmsg_len > clen) 4414 break; 4415 4416 data = CMSG_DATA(cm); 4417 datalen = (caddr_t)cm + cm->cmsg_len 4418 - (caddr_t)data; 4419 4420 if (cm->cmsg_level == SOL_SOCKET && 4421 cm->cmsg_type == SCM_RIGHTS) { 4422 (*op)(data, datalen / 4423 sizeof(struct filedescent *)); 4424 } 4425 4426 if (CMSG_SPACE(datalen) < clen) { 4427 clen -= CMSG_SPACE(datalen); 4428 cm = (struct cmsghdr *) 4429 ((caddr_t)cm + CMSG_SPACE(datalen)); 4430 } else { 4431 clen = 0; 4432 cm = NULL; 4433 } 4434 } 4435 } 4436 m0 = m0->m_nextpkt; 4437 } 4438 } 4439 4440 /* 4441 * Definitions of protocols supported in the LOCAL domain. 4442 */ 4443 static struct protosw streamproto = { 4444 .pr_type = SOCK_STREAM, 4445 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4446 .pr_ctloutput = &uipc_ctloutput, 4447 .pr_abort = uipc_abort, 4448 .pr_accept = uipc_peeraddr, 4449 .pr_attach = uipc_attach, 4450 .pr_bind = uipc_bind, 4451 .pr_bindat = uipc_bindat, 4452 .pr_connect = uipc_connect, 4453 .pr_connectat = uipc_connectat, 4454 .pr_connect2 = uipc_connect2, 4455 .pr_detach = uipc_detach, 4456 .pr_disconnect = uipc_disconnect, 4457 .pr_fdclose = uipc_fdclose, 4458 .pr_listen = uipc_listen, 4459 .pr_peeraddr = uipc_peeraddr, 4460 .pr_send = uipc_sendfile, 4461 .pr_sendfile_wait = uipc_sendfile_wait, 4462 .pr_ready = uipc_ready, 4463 .pr_sense = uipc_sense, 4464 .pr_shutdown = uipc_shutdown, 4465 .pr_sockaddr = uipc_sockaddr, 4466 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4467 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4468 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4469 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4470 .pr_close = uipc_close, 4471 .pr_chmod = uipc_chmod, 4472 }; 4473 4474 static struct protosw dgramproto = { 4475 .pr_type = SOCK_DGRAM, 4476 .pr_flags = PR_ATOMIC | PR_ADDR | PR_CAPATTACH | PR_SOCKBUF, 4477 .pr_ctloutput = &uipc_ctloutput, 4478 .pr_abort = uipc_abort, 4479 .pr_accept = uipc_peeraddr, 4480 .pr_attach = uipc_attach, 4481 .pr_bind = uipc_bind, 4482 .pr_bindat = uipc_bindat, 4483 .pr_connect = uipc_connect, 4484 .pr_connectat = uipc_connectat, 4485 .pr_connect2 = uipc_connect2, 4486 .pr_detach = uipc_detach, 4487 .pr_disconnect = uipc_disconnect, 4488 .pr_fdclose = uipc_fdclose, 4489 .pr_peeraddr = uipc_peeraddr, 4490 .pr_sosend = uipc_sosend_dgram, 4491 .pr_sense = uipc_sense, 4492 .pr_shutdown = uipc_shutdown, 4493 .pr_sockaddr = uipc_sockaddr, 4494 .pr_soreceive = uipc_soreceive_dgram, 4495 .pr_close = uipc_close, 4496 .pr_chmod = uipc_chmod, 4497 }; 4498 4499 static struct protosw seqpacketproto = { 4500 .pr_type = SOCK_SEQPACKET, 4501 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4502 .pr_ctloutput = &uipc_ctloutput, 4503 .pr_abort = uipc_abort, 4504 .pr_accept = uipc_peeraddr, 4505 .pr_attach = uipc_attach, 4506 .pr_bind = uipc_bind, 4507 .pr_bindat = uipc_bindat, 4508 .pr_connect = uipc_connect, 4509 .pr_connectat = uipc_connectat, 4510 .pr_connect2 = uipc_connect2, 4511 .pr_detach = uipc_detach, 4512 .pr_disconnect = uipc_disconnect, 4513 .pr_fdclose = uipc_fdclose, 4514 .pr_listen = uipc_listen, 4515 .pr_peeraddr = uipc_peeraddr, 4516 .pr_sense = uipc_sense, 4517 .pr_shutdown = uipc_shutdown, 4518 .pr_sockaddr = uipc_sockaddr, 4519 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4520 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4521 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4522 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4523 .pr_close = uipc_close, 4524 .pr_chmod = uipc_chmod, 4525 }; 4526 4527 static struct domain localdomain = { 4528 .dom_family = AF_LOCAL, 4529 .dom_name = "local", 4530 .dom_nprotosw = 3, 4531 .dom_protosw = { 4532 &streamproto, 4533 &dgramproto, 4534 &seqpacketproto, 4535 } 4536 }; 4537 DOMAIN_SET(local); 4538 4539 /* 4540 * A helper function called by VFS before socket-type vnode reclamation. 4541 * For an active vnode it clears unp_vnode pointer and decrements unp_vnode 4542 * use count. 4543 */ 4544 void 4545 vfs_unp_reclaim(struct vnode *vp) 4546 { 4547 struct unpcb *unp; 4548 int active; 4549 struct mtx *vplock; 4550 4551 ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim"); 4552 KASSERT(vp->v_type == VSOCK, 4553 ("vfs_unp_reclaim: vp->v_type != VSOCK")); 4554 4555 active = 0; 4556 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 4557 mtx_lock(vplock); 4558 VOP_UNP_CONNECT(vp, &unp); 4559 if (unp == NULL) 4560 goto done; 4561 UNP_PCB_LOCK(unp); 4562 if (unp->unp_vnode == vp) { 4563 VOP_UNP_DETACH(vp); 4564 unp->unp_vnode = NULL; 4565 active = 1; 4566 } 4567 UNP_PCB_UNLOCK(unp); 4568 done: 4569 mtx_unlock(vplock); 4570 if (active) 4571 vunref(vp); 4572 } 4573 4574 #ifdef DDB 4575 static void 4576 db_print_indent(int indent) 4577 { 4578 int i; 4579 4580 for (i = 0; i < indent; i++) 4581 db_printf(" "); 4582 } 4583 4584 static void 4585 db_print_unpflags(int unp_flags) 4586 { 4587 int comma; 4588 4589 comma = 0; 4590 if (unp_flags & UNP_HAVEPC) { 4591 db_printf("%sUNP_HAVEPC", comma ? ", " : ""); 4592 comma = 1; 4593 } 4594 if (unp_flags & UNP_WANTCRED_ALWAYS) { 4595 db_printf("%sUNP_WANTCRED_ALWAYS", comma ? ", " : ""); 4596 comma = 1; 4597 } 4598 if (unp_flags & UNP_WANTCRED_ONESHOT) { 4599 db_printf("%sUNP_WANTCRED_ONESHOT", comma ? ", " : ""); 4600 comma = 1; 4601 } 4602 if (unp_flags & UNP_CONNECTING) { 4603 db_printf("%sUNP_CONNECTING", comma ? ", " : ""); 4604 comma = 1; 4605 } 4606 if (unp_flags & UNP_BINDING) { 4607 db_printf("%sUNP_BINDING", comma ? ", " : ""); 4608 comma = 1; 4609 } 4610 } 4611 4612 static void 4613 db_print_xucred(int indent, struct xucred *xu) 4614 { 4615 int comma, i; 4616 4617 db_print_indent(indent); 4618 db_printf("cr_version: %u cr_uid: %u cr_pid: %d cr_ngroups: %d\n", 4619 xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups); 4620 db_print_indent(indent); 4621 db_printf("cr_groups: "); 4622 comma = 0; 4623 for (i = 0; i < xu->cr_ngroups; i++) { 4624 db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]); 4625 comma = 1; 4626 } 4627 db_printf("\n"); 4628 } 4629 4630 static void 4631 db_print_unprefs(int indent, struct unp_head *uh) 4632 { 4633 struct unpcb *unp; 4634 int counter; 4635 4636 counter = 0; 4637 LIST_FOREACH(unp, uh, unp_reflink) { 4638 if (counter % 4 == 0) 4639 db_print_indent(indent); 4640 db_printf("%p ", unp); 4641 if (counter % 4 == 3) 4642 db_printf("\n"); 4643 counter++; 4644 } 4645 if (counter != 0 && counter % 4 != 0) 4646 db_printf("\n"); 4647 } 4648 4649 DB_SHOW_COMMAND(unpcb, db_show_unpcb) 4650 { 4651 struct unpcb *unp; 4652 4653 if (!have_addr) { 4654 db_printf("usage: show unpcb <addr>\n"); 4655 return; 4656 } 4657 unp = (struct unpcb *)addr; 4658 4659 db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket, 4660 unp->unp_vnode); 4661 4662 db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino, 4663 unp->unp_conn); 4664 4665 db_printf("unp_refs:\n"); 4666 db_print_unprefs(2, &unp->unp_refs); 4667 4668 /* XXXRW: Would be nice to print the full address, if any. */ 4669 db_printf("unp_addr: %p\n", unp->unp_addr); 4670 4671 db_printf("unp_gencnt: %llu\n", 4672 (unsigned long long)unp->unp_gencnt); 4673 4674 db_printf("unp_flags: %x (", unp->unp_flags); 4675 db_print_unpflags(unp->unp_flags); 4676 db_printf(")\n"); 4677 4678 db_printf("unp_peercred:\n"); 4679 db_print_xucred(2, &unp->unp_peercred); 4680 4681 db_printf("unp_refcount: %u\n", unp->unp_refcount); 4682 } 4683 #endif 4684