1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1989, 1991, 1993 5 * The Regents of the University of California. All Rights Reserved. 6 * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved. 7 * Copyright (c) 2018 Matthew Macy 8 * Copyright (c) 2022-2025 Gleb Smirnoff <glebius@FreeBSD.org> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * UNIX Domain (Local) Sockets 37 * 38 * This is an implementation of UNIX (local) domain sockets. Each socket has 39 * an associated struct unpcb (UNIX protocol control block). Stream sockets 40 * may be connected to 0 or 1 other socket. Datagram sockets may be 41 * connected to 0, 1, or many other sockets. Sockets may be created and 42 * connected in pairs (socketpair(2)), or bound/connected to using the file 43 * system name space. For most purposes, only the receive socket buffer is 44 * used, as sending on one socket delivers directly to the receive socket 45 * buffer of a second socket. 46 * 47 * The implementation is substantially complicated by the fact that 48 * "ancillary data", such as file descriptors or credentials, may be passed 49 * across UNIX domain sockets. The potential for passing UNIX domain sockets 50 * over other UNIX domain sockets requires the implementation of a simple 51 * garbage collector to find and tear down cycles of disconnected sockets. 52 * 53 * TODO: 54 * RDM 55 * rethink name space problems 56 * need a proper out-of-band 57 */ 58 59 #include "opt_ddb.h" 60 61 #include <sys/param.h> 62 #include <sys/capsicum.h> 63 #include <sys/domain.h> 64 #include <sys/eventhandler.h> 65 #include <sys/fcntl.h> 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/jail.h> 69 #include <sys/kernel.h> 70 #include <sys/lock.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/mount.h> 74 #include <sys/mutex.h> 75 #include <sys/namei.h> 76 #include <sys/poll.h> 77 #include <sys/proc.h> 78 #include <sys/protosw.h> 79 #include <sys/queue.h> 80 #include <sys/resourcevar.h> 81 #include <sys/rwlock.h> 82 #include <sys/socket.h> 83 #include <sys/socketvar.h> 84 #include <sys/signalvar.h> 85 #include <sys/stat.h> 86 #include <sys/sysent.h> 87 #include <sys/sx.h> 88 #include <sys/sysctl.h> 89 #include <sys/systm.h> 90 #include <sys/taskqueue.h> 91 #include <sys/un.h> 92 #include <sys/unpcb.h> 93 #include <sys/vnode.h> 94 95 #include <net/vnet.h> 96 97 #ifdef DDB 98 #include <ddb/ddb.h> 99 #endif 100 101 #include <security/mac/mac_framework.h> 102 103 #include <vm/uma.h> 104 105 MALLOC_DECLARE(M_FILECAPS); 106 107 static struct domain localdomain; 108 109 static uma_zone_t unp_zone; 110 static unp_gen_t unp_gencnt; /* (l) */ 111 static u_int unp_count; /* (l) Count of local sockets. */ 112 static ino_t unp_ino; /* Prototype for fake inode numbers. */ 113 static int unp_rights; /* (g) File descriptors in flight. */ 114 static struct unp_head unp_shead; /* (l) List of stream sockets. */ 115 static struct unp_head unp_dhead; /* (l) List of datagram sockets. */ 116 static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */ 117 static struct mtx_pool *unp_vp_mtxpool; 118 119 struct unp_defer { 120 SLIST_ENTRY(unp_defer) ud_link; 121 struct file *ud_fp; 122 }; 123 static SLIST_HEAD(, unp_defer) unp_defers; 124 static int unp_defers_count; 125 126 static const struct sockaddr sun_noname = { 127 .sa_len = sizeof(sun_noname), 128 .sa_family = AF_LOCAL, 129 }; 130 131 /* 132 * Garbage collection of cyclic file descriptor/socket references occurs 133 * asynchronously in a taskqueue context in order to avoid recursion and 134 * reentrance in the UNIX domain socket, file descriptor, and socket layer 135 * code. See unp_gc() for a full description. 136 */ 137 static struct timeout_task unp_gc_task; 138 139 /* 140 * The close of unix domain sockets attached as SCM_RIGHTS is 141 * postponed to the taskqueue, to avoid arbitrary recursion depth. 142 * The attached sockets might have another sockets attached. 143 */ 144 static struct task unp_defer_task; 145 146 /* 147 * SOCK_STREAM and SOCK_SEQPACKET unix(4) sockets fully bypass the send buffer, 148 * however the notion of send buffer still makes sense with them. Its size is 149 * the amount of space that a send(2) syscall may copyin(9) before checking 150 * with the receive buffer of a peer. Although not linked anywhere yet, 151 * pointed to by a stack variable, effectively it is a buffer that needs to be 152 * sized. 153 * 154 * SOCK_DGRAM sockets really use the sendspace as the maximum datagram size, 155 * and don't really want to reserve the sendspace. Their recvspace should be 156 * large enough for at least one max-size datagram plus address. 157 */ 158 static u_long unpst_sendspace = 64*1024; 159 static u_long unpst_recvspace = 64*1024; 160 static u_long unpdg_maxdgram = 8*1024; /* support 8KB syslog msgs */ 161 static u_long unpdg_recvspace = 16*1024; 162 static u_long unpsp_sendspace = 64*1024; 163 static u_long unpsp_recvspace = 64*1024; 164 165 static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 166 "Local domain"); 167 static SYSCTL_NODE(_net_local, SOCK_STREAM, stream, 168 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 169 "SOCK_STREAM"); 170 static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, 171 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 172 "SOCK_DGRAM"); 173 static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket, 174 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 175 "SOCK_SEQPACKET"); 176 177 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW, 178 &unpst_sendspace, 0, "Default stream send space."); 179 SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW, 180 &unpst_recvspace, 0, "Default stream receive space."); 181 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW, 182 &unpdg_maxdgram, 0, "Maximum datagram size."); 183 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW, 184 &unpdg_recvspace, 0, "Default datagram receive space."); 185 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW, 186 &unpsp_sendspace, 0, "Default seqpacket send space."); 187 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW, 188 &unpsp_recvspace, 0, "Default seqpacket receive space."); 189 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, 190 "File descriptors in flight."); 191 SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, 192 &unp_defers_count, 0, 193 "File descriptors deferred to taskqueue for close."); 194 195 /* 196 * Locking and synchronization: 197 * 198 * Several types of locks exist in the local domain socket implementation: 199 * - a global linkage lock 200 * - a global connection list lock 201 * - the mtxpool lock 202 * - per-unpcb mutexes 203 * 204 * The linkage lock protects the global socket lists, the generation number 205 * counter and garbage collector state. 206 * 207 * The connection list lock protects the list of referring sockets in a datagram 208 * socket PCB. This lock is also overloaded to protect a global list of 209 * sockets whose buffers contain socket references in the form of SCM_RIGHTS 210 * messages. To avoid recursion, such references are released by a dedicated 211 * thread. 212 * 213 * The mtxpool lock protects the vnode from being modified while referenced. 214 * Lock ordering rules require that it be acquired before any PCB locks. 215 * 216 * The unpcb lock (unp_mtx) protects the most commonly referenced fields in the 217 * unpcb. This includes the unp_conn field, which either links two connected 218 * PCBs together (for connected socket types) or points at the destination 219 * socket (for connectionless socket types). The operations of creating or 220 * destroying a connection therefore involve locking multiple PCBs. To avoid 221 * lock order reversals, in some cases this involves dropping a PCB lock and 222 * using a reference counter to maintain liveness. 223 * 224 * UNIX domain sockets each have an unpcb hung off of their so_pcb pointer, 225 * allocated in pr_attach() and freed in pr_detach(). The validity of that 226 * pointer is an invariant, so no lock is required to dereference the so_pcb 227 * pointer if a valid socket reference is held by the caller. In practice, 228 * this is always true during operations performed on a socket. Each unpcb 229 * has a back-pointer to its socket, unp_socket, which will be stable under 230 * the same circumstances. 231 * 232 * This pointer may only be safely dereferenced as long as a valid reference 233 * to the unpcb is held. Typically, this reference will be from the socket, 234 * or from another unpcb when the referring unpcb's lock is held (in order 235 * that the reference not be invalidated during use). For example, to follow 236 * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee 237 * that detach is not run clearing unp_socket. 238 * 239 * Blocking with UNIX domain sockets is a tricky issue: unlike most network 240 * protocols, bind() is a non-atomic operation, and connect() requires 241 * potential sleeping in the protocol, due to potentially waiting on local or 242 * distributed file systems. We try to separate "lookup" operations, which 243 * may sleep, and the IPC operations themselves, which typically can occur 244 * with relative atomicity as locks can be held over the entire operation. 245 * 246 * Another tricky issue is simultaneous multi-threaded or multi-process 247 * access to a single UNIX domain socket. These are handled by the flags 248 * UNP_CONNECTING and UNP_BINDING, which prevent concurrent connecting or 249 * binding, both of which involve dropping UNIX domain socket locks in order 250 * to perform namei() and other file system operations. 251 */ 252 static struct rwlock unp_link_rwlock; 253 static struct mtx unp_defers_lock; 254 255 #define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \ 256 "unp_link_rwlock") 257 258 #define UNP_LINK_LOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 259 RA_LOCKED) 260 #define UNP_LINK_UNLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 261 RA_UNLOCKED) 262 263 #define UNP_LINK_RLOCK() rw_rlock(&unp_link_rwlock) 264 #define UNP_LINK_RUNLOCK() rw_runlock(&unp_link_rwlock) 265 #define UNP_LINK_WLOCK() rw_wlock(&unp_link_rwlock) 266 #define UNP_LINK_WUNLOCK() rw_wunlock(&unp_link_rwlock) 267 #define UNP_LINK_WLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 268 RA_WLOCKED) 269 #define UNP_LINK_WOWNED() rw_wowned(&unp_link_rwlock) 270 271 #define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \ 272 "unp_defer", NULL, MTX_DEF) 273 #define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock) 274 #define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock) 275 276 #define UNP_REF_LIST_LOCK() UNP_DEFERRED_LOCK(); 277 #define UNP_REF_LIST_UNLOCK() UNP_DEFERRED_UNLOCK(); 278 279 #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \ 280 "unp", "unp", \ 281 MTX_DUPOK|MTX_DEF) 282 #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx) 283 #define UNP_PCB_LOCKPTR(unp) (&(unp)->unp_mtx) 284 #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx) 285 #define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx) 286 #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx) 287 #define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx) 288 #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED) 289 #define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED) 290 291 static int uipc_connect2(struct socket *, struct socket *); 292 static int uipc_ctloutput(struct socket *, struct sockopt *); 293 static int unp_connect(struct socket *, struct sockaddr *, 294 struct thread *); 295 static int unp_connectat(int, struct socket *, struct sockaddr *, 296 struct thread *, bool); 297 static void unp_connect2(struct socket *, struct socket *, bool); 298 static void unp_disconnect(struct unpcb *unp, struct unpcb *unp2); 299 static void unp_dispose(struct socket *so); 300 static void unp_drop(struct unpcb *); 301 static void unp_gc(__unused void *, int); 302 static void unp_scan(struct mbuf *, void (*)(struct filedescent **, int)); 303 static void unp_discard(struct file *); 304 static void unp_freerights(struct filedescent **, int); 305 static int unp_internalize(struct mbuf *, struct mchain *, 306 struct thread *); 307 static void unp_internalize_fp(struct file *); 308 static int unp_externalize(struct mbuf *, struct mbuf **, int); 309 static int unp_externalize_fp(struct file *); 310 static void unp_addsockcred(struct thread *, struct mchain *, int); 311 static void unp_process_defers(void * __unused, int); 312 313 static void uipc_wrknl_lock(void *); 314 static void uipc_wrknl_unlock(void *); 315 static void uipc_wrknl_assert_lock(void *, int); 316 317 static void 318 unp_pcb_hold(struct unpcb *unp) 319 { 320 u_int old __unused; 321 322 old = refcount_acquire(&unp->unp_refcount); 323 KASSERT(old > 0, ("%s: unpcb %p has no references", __func__, unp)); 324 } 325 326 static __result_use_check bool 327 unp_pcb_rele(struct unpcb *unp) 328 { 329 bool ret; 330 331 UNP_PCB_LOCK_ASSERT(unp); 332 333 if ((ret = refcount_release(&unp->unp_refcount))) { 334 UNP_PCB_UNLOCK(unp); 335 UNP_PCB_LOCK_DESTROY(unp); 336 uma_zfree(unp_zone, unp); 337 } 338 return (ret); 339 } 340 341 static void 342 unp_pcb_rele_notlast(struct unpcb *unp) 343 { 344 bool ret __unused; 345 346 ret = refcount_release(&unp->unp_refcount); 347 KASSERT(!ret, ("%s: unpcb %p has no references", __func__, unp)); 348 } 349 350 static void 351 unp_pcb_lock_pair(struct unpcb *unp, struct unpcb *unp2) 352 { 353 UNP_PCB_UNLOCK_ASSERT(unp); 354 UNP_PCB_UNLOCK_ASSERT(unp2); 355 356 if (unp == unp2) { 357 UNP_PCB_LOCK(unp); 358 } else if ((uintptr_t)unp2 > (uintptr_t)unp) { 359 UNP_PCB_LOCK(unp); 360 UNP_PCB_LOCK(unp2); 361 } else { 362 UNP_PCB_LOCK(unp2); 363 UNP_PCB_LOCK(unp); 364 } 365 } 366 367 static void 368 unp_pcb_unlock_pair(struct unpcb *unp, struct unpcb *unp2) 369 { 370 UNP_PCB_UNLOCK(unp); 371 if (unp != unp2) 372 UNP_PCB_UNLOCK(unp2); 373 } 374 375 /* 376 * Try to lock the connected peer of an already locked socket. In some cases 377 * this requires that we unlock the current socket. The pairbusy counter is 378 * used to block concurrent connection attempts while the lock is dropped. The 379 * caller must be careful to revalidate PCB state. 380 */ 381 static struct unpcb * 382 unp_pcb_lock_peer(struct unpcb *unp) 383 { 384 struct unpcb *unp2; 385 386 UNP_PCB_LOCK_ASSERT(unp); 387 unp2 = unp->unp_conn; 388 if (unp2 == NULL) 389 return (NULL); 390 if (__predict_false(unp == unp2)) 391 return (unp); 392 393 UNP_PCB_UNLOCK_ASSERT(unp2); 394 395 if (__predict_true(UNP_PCB_TRYLOCK(unp2))) 396 return (unp2); 397 if ((uintptr_t)unp2 > (uintptr_t)unp) { 398 UNP_PCB_LOCK(unp2); 399 return (unp2); 400 } 401 unp->unp_pairbusy++; 402 unp_pcb_hold(unp2); 403 UNP_PCB_UNLOCK(unp); 404 405 UNP_PCB_LOCK(unp2); 406 UNP_PCB_LOCK(unp); 407 KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL, 408 ("%s: socket %p was reconnected", __func__, unp)); 409 if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) { 410 unp->unp_flags &= ~UNP_WAITING; 411 wakeup(unp); 412 } 413 if (unp_pcb_rele(unp2)) { 414 /* unp2 is unlocked. */ 415 return (NULL); 416 } 417 if (unp->unp_conn == NULL) { 418 UNP_PCB_UNLOCK(unp2); 419 return (NULL); 420 } 421 return (unp2); 422 } 423 424 /* 425 * Try to lock peer of our socket for purposes of sending data to it. 426 */ 427 static int 428 uipc_lock_peer(struct socket *so, struct unpcb **unp2) 429 { 430 struct unpcb *unp; 431 int error; 432 433 unp = sotounpcb(so); 434 UNP_PCB_LOCK(unp); 435 *unp2 = unp_pcb_lock_peer(unp); 436 if (__predict_false(so->so_error != 0)) { 437 error = so->so_error; 438 so->so_error = 0; 439 UNP_PCB_UNLOCK(unp); 440 if (*unp2 != NULL) 441 UNP_PCB_UNLOCK(*unp2); 442 return (error); 443 } 444 if (__predict_false(*unp2 == NULL)) { 445 /* 446 * Different error code for a previously connected socket and 447 * a never connected one. The SS_ISDISCONNECTED is set in the 448 * unp_soisdisconnected() and is synchronized by the pcb lock. 449 */ 450 error = so->so_state & SS_ISDISCONNECTED ? EPIPE : ENOTCONN; 451 UNP_PCB_UNLOCK(unp); 452 return (error); 453 } 454 UNP_PCB_UNLOCK(unp); 455 456 return (0); 457 } 458 459 static void 460 uipc_abort(struct socket *so) 461 { 462 struct unpcb *unp, *unp2; 463 464 unp = sotounpcb(so); 465 KASSERT(unp != NULL, ("uipc_abort: unp == NULL")); 466 UNP_PCB_UNLOCK_ASSERT(unp); 467 468 UNP_PCB_LOCK(unp); 469 unp2 = unp->unp_conn; 470 if (unp2 != NULL) { 471 unp_pcb_hold(unp2); 472 UNP_PCB_UNLOCK(unp); 473 unp_drop(unp2); 474 } else 475 UNP_PCB_UNLOCK(unp); 476 } 477 478 static int 479 uipc_attach(struct socket *so, int proto, struct thread *td) 480 { 481 u_long sendspace, recvspace; 482 struct unpcb *unp; 483 int error, rcvmtxopts; 484 bool locked; 485 486 KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); 487 switch (so->so_type) { 488 case SOCK_DGRAM: 489 STAILQ_INIT(&so->so_rcv.uxdg_mb); 490 STAILQ_INIT(&so->so_snd.uxdg_mb); 491 TAILQ_INIT(&so->so_rcv.uxdg_conns); 492 /* 493 * Since send buffer is either bypassed or is a part 494 * of one-to-many receive buffer, we assign both space 495 * limits to unpdg_recvspace. 496 */ 497 sendspace = recvspace = unpdg_recvspace; 498 rcvmtxopts = 0; 499 break; 500 501 case SOCK_STREAM: 502 sendspace = unpst_sendspace; 503 recvspace = unpst_recvspace; 504 goto common; 505 506 case SOCK_SEQPACKET: 507 sendspace = unpsp_sendspace; 508 recvspace = unpsp_recvspace; 509 common: 510 rcvmtxopts = MTX_DUPOK; 511 knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, 512 uipc_wrknl_unlock, uipc_wrknl_assert_lock); 513 STAILQ_INIT(&so->so_rcv.uxst_mbq); 514 break; 515 default: 516 panic("uipc_attach"); 517 } 518 mtx_init(&so->so_rcv_mtx, "unix so_rcv", NULL, MTX_DEF | rcvmtxopts); 519 mtx_init(&so->so_snd_mtx, "unix so_snd", NULL, MTX_DEF); 520 error = soreserve(so, sendspace, recvspace); 521 if (error) 522 return (error); 523 unp = uma_zalloc(unp_zone, M_NOWAIT | M_ZERO); 524 if (unp == NULL) 525 return (ENOBUFS); 526 LIST_INIT(&unp->unp_refs); 527 UNP_PCB_LOCK_INIT(unp); 528 unp->unp_socket = so; 529 so->so_pcb = unp; 530 refcount_init(&unp->unp_refcount, 1); 531 unp->unp_mode = ACCESSPERMS; 532 533 if ((locked = UNP_LINK_WOWNED()) == false) 534 UNP_LINK_WLOCK(); 535 536 unp->unp_gencnt = ++unp_gencnt; 537 unp->unp_ino = ++unp_ino; 538 unp_count++; 539 switch (so->so_type) { 540 case SOCK_STREAM: 541 LIST_INSERT_HEAD(&unp_shead, unp, unp_link); 542 break; 543 544 case SOCK_DGRAM: 545 LIST_INSERT_HEAD(&unp_dhead, unp, unp_link); 546 break; 547 548 case SOCK_SEQPACKET: 549 LIST_INSERT_HEAD(&unp_sphead, unp, unp_link); 550 break; 551 552 default: 553 panic("uipc_attach"); 554 } 555 556 if (locked == false) 557 UNP_LINK_WUNLOCK(); 558 559 return (0); 560 } 561 562 static int 563 uipc_bindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 564 { 565 struct sockaddr_un *soun = (struct sockaddr_un *)nam; 566 struct vattr vattr; 567 int error, namelen; 568 struct nameidata nd; 569 struct unpcb *unp; 570 struct vnode *vp; 571 struct mount *mp; 572 cap_rights_t rights; 573 char *buf; 574 mode_t mode; 575 576 if (nam->sa_family != AF_UNIX) 577 return (EAFNOSUPPORT); 578 579 unp = sotounpcb(so); 580 KASSERT(unp != NULL, ("uipc_bind: unp == NULL")); 581 582 if (soun->sun_len > sizeof(struct sockaddr_un)) 583 return (EINVAL); 584 namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); 585 if (namelen <= 0) 586 return (EINVAL); 587 588 /* 589 * We don't allow simultaneous bind() calls on a single UNIX domain 590 * socket, so flag in-progress operations, and return an error if an 591 * operation is already in progress. 592 * 593 * Historically, we have not allowed a socket to be rebound, so this 594 * also returns an error. Not allowing re-binding simplifies the 595 * implementation and avoids a great many possible failure modes. 596 */ 597 UNP_PCB_LOCK(unp); 598 if (unp->unp_vnode != NULL) { 599 UNP_PCB_UNLOCK(unp); 600 return (EINVAL); 601 } 602 if (unp->unp_flags & UNP_BINDING) { 603 UNP_PCB_UNLOCK(unp); 604 return (EALREADY); 605 } 606 unp->unp_flags |= UNP_BINDING; 607 mode = unp->unp_mode & ~td->td_proc->p_pd->pd_cmask; 608 UNP_PCB_UNLOCK(unp); 609 610 buf = malloc(namelen + 1, M_TEMP, M_WAITOK); 611 bcopy(soun->sun_path, buf, namelen); 612 buf[namelen] = 0; 613 614 restart: 615 NDINIT_ATRIGHTS(&nd, CREATE, NOFOLLOW | LOCKPARENT | NOCACHE, 616 UIO_SYSSPACE, buf, fd, cap_rights_init_one(&rights, CAP_BINDAT)); 617 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */ 618 error = namei(&nd); 619 if (error) 620 goto error; 621 vp = nd.ni_vp; 622 if (vp != NULL || vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) { 623 NDFREE_PNBUF(&nd); 624 if (nd.ni_dvp == vp) 625 vrele(nd.ni_dvp); 626 else 627 vput(nd.ni_dvp); 628 if (vp != NULL) { 629 vrele(vp); 630 error = EADDRINUSE; 631 goto error; 632 } 633 error = vn_start_write(NULL, &mp, V_XSLEEP | V_PCATCH); 634 if (error) 635 goto error; 636 goto restart; 637 } 638 VATTR_NULL(&vattr); 639 vattr.va_type = VSOCK; 640 vattr.va_mode = mode; 641 #ifdef MAC 642 error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd, 643 &vattr); 644 #endif 645 if (error == 0) { 646 /* 647 * The prior lookup may have left LK_SHARED in cn_lkflags, 648 * and VOP_CREATE technically only requires the new vnode to 649 * be locked shared. Most filesystems will return the new vnode 650 * locked exclusive regardless, but we should explicitly 651 * specify that here since we require it and assert to that 652 * effect below. 653 */ 654 nd.ni_cnd.cn_lkflags = (nd.ni_cnd.cn_lkflags & ~LK_SHARED) | 655 LK_EXCLUSIVE; 656 error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr); 657 } 658 NDFREE_PNBUF(&nd); 659 if (error) { 660 VOP_VPUT_PAIR(nd.ni_dvp, NULL, true); 661 vn_finished_write(mp); 662 if (error == ERELOOKUP) 663 goto restart; 664 goto error; 665 } 666 vp = nd.ni_vp; 667 ASSERT_VOP_ELOCKED(vp, "uipc_bind"); 668 soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK); 669 670 UNP_PCB_LOCK(unp); 671 VOP_UNP_BIND(vp, unp); 672 unp->unp_vnode = vp; 673 unp->unp_addr = soun; 674 unp->unp_flags &= ~UNP_BINDING; 675 UNP_PCB_UNLOCK(unp); 676 vref(vp); 677 VOP_VPUT_PAIR(nd.ni_dvp, &vp, true); 678 vn_finished_write(mp); 679 free(buf, M_TEMP); 680 return (0); 681 682 error: 683 UNP_PCB_LOCK(unp); 684 unp->unp_flags &= ~UNP_BINDING; 685 UNP_PCB_UNLOCK(unp); 686 free(buf, M_TEMP); 687 return (error); 688 } 689 690 static int 691 uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td) 692 { 693 694 return (uipc_bindat(AT_FDCWD, so, nam, td)); 695 } 696 697 static int 698 uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 699 { 700 int error; 701 702 KASSERT(td == curthread, ("uipc_connect: td != curthread")); 703 error = unp_connect(so, nam, td); 704 return (error); 705 } 706 707 static int 708 uipc_connectat(int fd, struct socket *so, struct sockaddr *nam, 709 struct thread *td) 710 { 711 int error; 712 713 KASSERT(td == curthread, ("uipc_connectat: td != curthread")); 714 error = unp_connectat(fd, so, nam, td, false); 715 return (error); 716 } 717 718 static void 719 uipc_close(struct socket *so) 720 { 721 struct unpcb *unp, *unp2; 722 struct vnode *vp = NULL; 723 struct mtx *vplock; 724 725 unp = sotounpcb(so); 726 KASSERT(unp != NULL, ("uipc_close: unp == NULL")); 727 728 vplock = NULL; 729 if ((vp = unp->unp_vnode) != NULL) { 730 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 731 mtx_lock(vplock); 732 } 733 UNP_PCB_LOCK(unp); 734 if (vp && unp->unp_vnode == NULL) { 735 mtx_unlock(vplock); 736 vp = NULL; 737 } 738 if (vp != NULL) { 739 VOP_UNP_DETACH(vp); 740 unp->unp_vnode = NULL; 741 } 742 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 743 unp_disconnect(unp, unp2); 744 else 745 UNP_PCB_UNLOCK(unp); 746 if (vp) { 747 mtx_unlock(vplock); 748 vrele(vp); 749 } 750 } 751 752 static int 753 uipc_chmod(struct socket *so, mode_t mode, struct ucred *cred __unused, 754 struct thread *td __unused) 755 { 756 struct unpcb *unp; 757 int error; 758 759 if ((mode & ~ACCESSPERMS) != 0) 760 return (EINVAL); 761 762 error = 0; 763 unp = sotounpcb(so); 764 UNP_PCB_LOCK(unp); 765 if (unp->unp_vnode != NULL || (unp->unp_flags & UNP_BINDING) != 0) 766 error = EINVAL; 767 else 768 unp->unp_mode = mode; 769 UNP_PCB_UNLOCK(unp); 770 return (error); 771 } 772 773 static int 774 uipc_connect2(struct socket *so1, struct socket *so2) 775 { 776 struct unpcb *unp, *unp2; 777 778 if (so1->so_type != so2->so_type) 779 return (EPROTOTYPE); 780 781 unp = so1->so_pcb; 782 KASSERT(unp != NULL, ("uipc_connect2: unp == NULL")); 783 unp2 = so2->so_pcb; 784 KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL")); 785 unp_pcb_lock_pair(unp, unp2); 786 unp_connect2(so1, so2, false); 787 unp_pcb_unlock_pair(unp, unp2); 788 789 return (0); 790 } 791 792 static void 793 maybe_schedule_gc(void) 794 { 795 if (atomic_load_int(&unp_rights) != 0) 796 taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); 797 } 798 799 static void 800 uipc_detach(struct socket *so) 801 { 802 struct unpcb *unp, *unp2; 803 struct mtx *vplock; 804 struct vnode *vp; 805 806 unp = sotounpcb(so); 807 KASSERT(unp != NULL, ("uipc_detach: unp == NULL")); 808 809 vp = NULL; 810 vplock = NULL; 811 812 if (!SOLISTENING(so)) 813 unp_dispose(so); 814 815 UNP_LINK_WLOCK(); 816 LIST_REMOVE(unp, unp_link); 817 if (unp->unp_gcflag & UNPGC_DEAD) 818 LIST_REMOVE(unp, unp_dead); 819 unp->unp_gencnt = ++unp_gencnt; 820 --unp_count; 821 UNP_LINK_WUNLOCK(); 822 823 UNP_PCB_UNLOCK_ASSERT(unp); 824 restart: 825 if ((vp = unp->unp_vnode) != NULL) { 826 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 827 mtx_lock(vplock); 828 } 829 UNP_PCB_LOCK(unp); 830 if (unp->unp_vnode != vp && unp->unp_vnode != NULL) { 831 if (vplock) 832 mtx_unlock(vplock); 833 UNP_PCB_UNLOCK(unp); 834 goto restart; 835 } 836 if ((vp = unp->unp_vnode) != NULL) { 837 VOP_UNP_DETACH(vp); 838 unp->unp_vnode = NULL; 839 } 840 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 841 unp_disconnect(unp, unp2); 842 else 843 UNP_PCB_UNLOCK(unp); 844 845 UNP_REF_LIST_LOCK(); 846 while (!LIST_EMPTY(&unp->unp_refs)) { 847 struct unpcb *ref = LIST_FIRST(&unp->unp_refs); 848 849 unp_pcb_hold(ref); 850 UNP_REF_LIST_UNLOCK(); 851 852 MPASS(ref != unp); 853 UNP_PCB_UNLOCK_ASSERT(ref); 854 unp_drop(ref); 855 UNP_REF_LIST_LOCK(); 856 } 857 UNP_REF_LIST_UNLOCK(); 858 859 UNP_PCB_LOCK(unp); 860 unp->unp_socket->so_pcb = NULL; 861 unp->unp_socket = NULL; 862 free(unp->unp_addr, M_SONAME); 863 unp->unp_addr = NULL; 864 if (!unp_pcb_rele(unp)) 865 UNP_PCB_UNLOCK(unp); 866 if (vp) { 867 mtx_unlock(vplock); 868 vrele(vp); 869 } 870 maybe_schedule_gc(); 871 872 switch (so->so_type) { 873 case SOCK_STREAM: 874 case SOCK_SEQPACKET: 875 MPASS(SOLISTENING(so) || (STAILQ_EMPTY(&so->so_rcv.uxst_mbq) && 876 so->so_rcv.uxst_peer == NULL)); 877 break; 878 case SOCK_DGRAM: 879 /* 880 * Everything should have been unlinked/freed by unp_dispose() 881 * and/or unp_disconnect(). 882 */ 883 MPASS(so->so_rcv.uxdg_peeked == NULL); 884 MPASS(STAILQ_EMPTY(&so->so_rcv.uxdg_mb)); 885 MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); 886 MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); 887 } 888 889 mtx_destroy(&so->so_snd_mtx); 890 mtx_destroy(&so->so_rcv_mtx); 891 } 892 893 static int 894 uipc_disconnect(struct socket *so) 895 { 896 struct unpcb *unp, *unp2; 897 898 unp = sotounpcb(so); 899 KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL")); 900 901 UNP_PCB_LOCK(unp); 902 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 903 unp_disconnect(unp, unp2); 904 else 905 UNP_PCB_UNLOCK(unp); 906 return (0); 907 } 908 909 static void 910 uipc_fdclose(struct socket *so __unused) 911 { 912 /* 913 * Ensure that userspace can't create orphaned file descriptors without 914 * triggering garbage collection. Triggering GC from uipc_detach() is 915 * not sufficient, since that's only closed once a socket reference 916 * count drops to zero. 917 */ 918 maybe_schedule_gc(); 919 } 920 921 static int 922 uipc_listen(struct socket *so, int backlog, struct thread *td) 923 { 924 struct unpcb *unp; 925 int error; 926 927 MPASS(so->so_type != SOCK_DGRAM); 928 929 /* 930 * Synchronize with concurrent connection attempts. 931 */ 932 error = 0; 933 unp = sotounpcb(so); 934 UNP_PCB_LOCK(unp); 935 if (unp->unp_conn != NULL || (unp->unp_flags & UNP_CONNECTING) != 0) 936 error = EINVAL; 937 else if (unp->unp_vnode == NULL) 938 error = EDESTADDRREQ; 939 if (error != 0) { 940 UNP_PCB_UNLOCK(unp); 941 return (error); 942 } 943 944 SOCK_LOCK(so); 945 error = solisten_proto_check(so); 946 if (error == 0) { 947 cru2xt(td, &unp->unp_peercred); 948 if (!SOLISTENING(so)) { 949 (void)chgsbsize(so->so_cred->cr_uidinfo, 950 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 951 (void)chgsbsize(so->so_cred->cr_uidinfo, 952 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 953 } 954 solisten_proto(so, backlog); 955 } 956 SOCK_UNLOCK(so); 957 UNP_PCB_UNLOCK(unp); 958 return (error); 959 } 960 961 static int 962 uipc_peeraddr(struct socket *so, struct sockaddr *ret) 963 { 964 struct unpcb *unp, *unp2; 965 const struct sockaddr *sa; 966 967 unp = sotounpcb(so); 968 KASSERT(unp != NULL, ("uipc_peeraddr: unp == NULL")); 969 970 UNP_PCB_LOCK(unp); 971 unp2 = unp_pcb_lock_peer(unp); 972 if (unp2 != NULL) { 973 if (unp2->unp_addr != NULL) 974 sa = (struct sockaddr *)unp2->unp_addr; 975 else 976 sa = &sun_noname; 977 bcopy(sa, ret, sa->sa_len); 978 unp_pcb_unlock_pair(unp, unp2); 979 } else { 980 UNP_PCB_UNLOCK(unp); 981 sa = &sun_noname; 982 bcopy(sa, ret, sa->sa_len); 983 } 984 return (0); 985 } 986 987 /* 988 * pr_sosend() called with mbuf instead of uio is a kernel thread. NFS, 989 * netgraph(4) and other subsystems can call into socket code. The 990 * function will condition the mbuf so that it can be safely put onto socket 991 * buffer and calculate its char count and mbuf count. 992 * 993 * Note: we don't support receiving control data from a kernel thread. Our 994 * pr_sosend methods have MPASS() to check that. This may change. 995 */ 996 static void 997 uipc_reset_kernel_mbuf(struct mbuf *m, struct mchain *mc) 998 { 999 1000 M_ASSERTPKTHDR(m); 1001 1002 m_clrprotoflags(m); 1003 m_tag_delete_chain(m, NULL); 1004 m->m_pkthdr.rcvif = NULL; 1005 m->m_pkthdr.flowid = 0; 1006 m->m_pkthdr.csum_flags = 0; 1007 m->m_pkthdr.fibnum = 0; 1008 m->m_pkthdr.rsstype = 0; 1009 1010 mc_init_m(mc, m); 1011 MPASS(m->m_pkthdr.len == mc->mc_len); 1012 } 1013 1014 #ifdef SOCKBUF_DEBUG 1015 static inline void 1016 uipc_stream_sbcheck(struct sockbuf *sb) 1017 { 1018 struct mbuf *d; 1019 u_int dacc, dccc, dctl, dmbcnt; 1020 bool notready = false; 1021 1022 dacc = dccc = dctl = dmbcnt = 0; 1023 STAILQ_FOREACH(d, &sb->uxst_mbq, m_stailq) { 1024 if (d == sb->uxst_fnrdy) { 1025 MPASS(d->m_flags & M_NOTREADY); 1026 notready = true; 1027 } 1028 if (d->m_type == MT_CONTROL) 1029 dctl += d->m_len; 1030 else if (d->m_type == MT_DATA) { 1031 dccc += d->m_len; 1032 if (!notready) 1033 dacc += d->m_len; 1034 } else 1035 MPASS(0); 1036 dmbcnt += MSIZE; 1037 if (d->m_flags & M_EXT) 1038 dmbcnt += d->m_ext.ext_size; 1039 if (d->m_stailq.stqe_next == NULL) 1040 MPASS(sb->uxst_mbq.stqh_last == &d->m_stailq.stqe_next); 1041 } 1042 MPASS(sb->uxst_fnrdy == NULL || notready); 1043 MPASS(dacc == sb->sb_acc); 1044 MPASS(dccc == sb->sb_ccc); 1045 MPASS(dctl == sb->sb_ctl); 1046 MPASS(dmbcnt == sb->sb_mbcnt); 1047 (void)STAILQ_EMPTY(&sb->uxst_mbq); 1048 } 1049 #define UIPC_STREAM_SBCHECK(sb) uipc_stream_sbcheck(sb) 1050 #else 1051 #define UIPC_STREAM_SBCHECK(sb) do {} while (0) 1052 #endif 1053 1054 /* 1055 * uipc_stream_sbspace() returns how much a writer can send, limited by char 1056 * count or mbuf memory use, whatever ends first. 1057 * 1058 * An obvious and legitimate reason for a socket having more data than allowed, 1059 * is lowering the limit with setsockopt(SO_RCVBUF) on already full buffer. 1060 * Also, sb_mbcnt may overcommit sb_mbmax in case if previous write observed 1061 * 'space < mbspace', but mchain allocated to hold 'space' bytes of data ended 1062 * up with 'mc_mlen > mbspace'. A typical scenario would be a full buffer with 1063 * writer trying to push in a large write, and a slow reader, that reads just 1064 * a few bytes at a time. In that case writer will keep creating new mbufs 1065 * with mc_split(). These mbufs will carry little chars, but will all point at 1066 * the same cluster, thus each adding cluster size to sb_mbcnt. This means we 1067 * will count same cluster many times potentially underutilizing socket buffer. 1068 * We aren't optimizing towards ineffective readers. Classic socket buffer had 1069 * the same "feature". 1070 */ 1071 static inline u_int 1072 uipc_stream_sbspace(struct sockbuf *sb) 1073 { 1074 u_int space, mbspace; 1075 1076 if (__predict_true(sb->sb_hiwat >= sb->sb_ccc + sb->sb_ctl)) 1077 space = sb->sb_hiwat - sb->sb_ccc - sb->sb_ctl; 1078 else 1079 return (0); 1080 if (__predict_true(sb->sb_mbmax >= sb->sb_mbcnt)) 1081 mbspace = sb->sb_mbmax - sb->sb_mbcnt; 1082 else 1083 return (0); 1084 1085 return (min(space, mbspace)); 1086 } 1087 1088 /* 1089 * UNIX version of generic sbwait() for writes. We wait on peer's receive 1090 * buffer, using our timeout. 1091 */ 1092 static int 1093 uipc_stream_sbwait(struct socket *so, sbintime_t timeo) 1094 { 1095 struct sockbuf *sb = &so->so_rcv; 1096 1097 SOCK_RECVBUF_LOCK_ASSERT(so); 1098 sb->sb_flags |= SB_WAIT; 1099 return (msleep_sbt(&sb->sb_acc, SOCK_RECVBUF_MTX(so), PSOCK | PCATCH, 1100 "sbwait", timeo, 0, 0)); 1101 } 1102 1103 static int 1104 uipc_sosend_stream_or_seqpacket(struct socket *so, struct sockaddr *addr, 1105 struct uio *uio0, struct mbuf *m, struct mbuf *c, int flags, 1106 struct thread *td) 1107 { 1108 struct unpcb *unp2; 1109 struct socket *so2; 1110 struct sockbuf *sb; 1111 struct uio *uio; 1112 struct mchain mc, cmc; 1113 size_t resid, sent; 1114 bool nonblock, eor, aio; 1115 int error; 1116 1117 MPASS((uio0 != NULL && m == NULL) || (m != NULL && uio0 == NULL)); 1118 MPASS(m == NULL || c == NULL); 1119 1120 if (__predict_false(flags & MSG_OOB)) 1121 return (EOPNOTSUPP); 1122 1123 nonblock = (so->so_state & SS_NBIO) || 1124 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1125 eor = flags & MSG_EOR; 1126 1127 mc = MCHAIN_INITIALIZER(&mc); 1128 cmc = MCHAIN_INITIALIZER(&cmc); 1129 sent = 0; 1130 aio = false; 1131 1132 if (m == NULL) { 1133 if (c != NULL && (error = unp_internalize(c, &cmc, td))) 1134 goto out; 1135 /* 1136 * This function may read more data from the uio than it would 1137 * then place on socket. That would leave uio inconsistent 1138 * upon return. Normally uio is allocated on the stack of the 1139 * syscall thread and we don't care about leaving it consistent. 1140 * However, aio(9) will allocate a uio as part of job and will 1141 * use it to track progress. We detect aio(9) checking the 1142 * SB_AIO_RUNNING flag. It is safe to check it without lock 1143 * cause it is set and cleared in the same taskqueue thread. 1144 * 1145 * This check can also produce a false positive: there is 1146 * aio(9) job and also there is a syscall we are serving now. 1147 * No sane software does that, it would leave to a mess in 1148 * the socket buffer, as aio(9) doesn't grab the I/O sx(9). 1149 * But syzkaller can create this mess. For such false positive 1150 * our goal is just don't panic or leak memory. 1151 */ 1152 if (__predict_false(so->so_snd.sb_flags & SB_AIO_RUNNING)) { 1153 uio = cloneuio(uio0); 1154 aio = true; 1155 } else { 1156 uio = uio0; 1157 resid = uio->uio_resid; 1158 } 1159 /* 1160 * Optimization for a case when our send fits into the receive 1161 * buffer - do the copyin before taking any locks, sized to our 1162 * send buffer. Later copyins will also take into account 1163 * space in the peer's receive buffer. 1164 */ 1165 error = mc_uiotomc(&mc, uio, so->so_snd.sb_hiwat, 0, M_WAITOK, 1166 eor ? M_EOR : 0); 1167 if (__predict_false(error)) 1168 goto out2; 1169 } else 1170 uipc_reset_kernel_mbuf(m, &mc); 1171 1172 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 1173 if (error) 1174 goto out2; 1175 1176 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 1177 goto out3; 1178 1179 if (unp2->unp_flags & UNP_WANTCRED_MASK) { 1180 /* 1181 * Credentials are passed only once on SOCK_STREAM and 1182 * SOCK_SEQPACKET (LOCAL_CREDS => WANTCRED_ONESHOT), or 1183 * forever (LOCAL_CREDS_PERSISTENT => WANTCRED_ALWAYS). 1184 */ 1185 unp_addsockcred(td, &cmc, unp2->unp_flags); 1186 unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT; 1187 } 1188 1189 /* 1190 * Cycle through the data to send and available space in the peer's 1191 * receive buffer. Put a reference on the peer socket, so that it 1192 * doesn't get freed while we sbwait(). If peer goes away, we will 1193 * observe the SBS_CANTRCVMORE and our sorele() will finalize peer's 1194 * socket destruction. 1195 */ 1196 so2 = unp2->unp_socket; 1197 soref(so2); 1198 UNP_PCB_UNLOCK(unp2); 1199 sb = &so2->so_rcv; 1200 while (mc.mc_len + cmc.mc_len > 0) { 1201 struct mchain mcnext = MCHAIN_INITIALIZER(&mcnext); 1202 u_int space; 1203 1204 SOCK_RECVBUF_LOCK(so2); 1205 restart: 1206 UIPC_STREAM_SBCHECK(sb); 1207 if (__predict_false(cmc.mc_len > sb->sb_hiwat)) { 1208 SOCK_RECVBUF_UNLOCK(so2); 1209 error = EMSGSIZE; 1210 goto out4; 1211 } 1212 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1213 SOCK_RECVBUF_UNLOCK(so2); 1214 error = EPIPE; 1215 goto out4; 1216 } 1217 /* 1218 * Wait on the peer socket receive buffer until we have enough 1219 * space to put at least control. The data is a stream and can 1220 * be put partially, but control is really a datagram. 1221 */ 1222 space = uipc_stream_sbspace(sb); 1223 if (space < sb->sb_lowat || space < cmc.mc_len) { 1224 if (nonblock) { 1225 if (aio) 1226 sb->uxst_flags |= UXST_PEER_AIO; 1227 SOCK_RECVBUF_UNLOCK(so2); 1228 if (aio) { 1229 SOCK_SENDBUF_LOCK(so); 1230 so->so_snd.sb_ccc = 1231 so->so_snd.sb_hiwat - space; 1232 SOCK_SENDBUF_UNLOCK(so); 1233 } 1234 error = EWOULDBLOCK; 1235 goto out4; 1236 } 1237 if ((error = uipc_stream_sbwait(so2, 1238 so->so_snd.sb_timeo)) != 0) { 1239 SOCK_RECVBUF_UNLOCK(so2); 1240 goto out4; 1241 } else 1242 goto restart; 1243 } 1244 MPASS(space >= cmc.mc_len); 1245 space -= cmc.mc_len; 1246 if (space == 0) { 1247 /* There is space only to send control. */ 1248 MPASS(!STAILQ_EMPTY(&cmc.mc_q)); 1249 mcnext = mc; 1250 mc = MCHAIN_INITIALIZER(&mc); 1251 } else if (space < mc.mc_len) { 1252 /* Not enough space. */ 1253 if (__predict_false(mc_split(&mc, &mcnext, space, 1254 M_NOWAIT) == ENOMEM)) { 1255 /* 1256 * If allocation failed use M_WAITOK and merge 1257 * the chain back. Next time mc_split() will 1258 * easily split at the same place. Only if we 1259 * race with setsockopt(SO_RCVBUF) shrinking 1260 * sb_hiwat can this happen more than once. 1261 */ 1262 SOCK_RECVBUF_UNLOCK(so2); 1263 (void)mc_split(&mc, &mcnext, space, M_WAITOK); 1264 mc_concat(&mc, &mcnext); 1265 SOCK_RECVBUF_LOCK(so2); 1266 goto restart; 1267 } 1268 MPASS(mc.mc_len == space); 1269 } 1270 if (!STAILQ_EMPTY(&cmc.mc_q)) { 1271 STAILQ_CONCAT(&sb->uxst_mbq, &cmc.mc_q); 1272 sb->sb_ctl += cmc.mc_len; 1273 sb->sb_mbcnt += cmc.mc_mlen; 1274 cmc.mc_len = 0; 1275 } 1276 sent += mc.mc_len; 1277 if (sb->uxst_fnrdy == NULL) 1278 sb->sb_acc += mc.mc_len; 1279 sb->sb_ccc += mc.mc_len; 1280 sb->sb_mbcnt += mc.mc_mlen; 1281 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 1282 UIPC_STREAM_SBCHECK(sb); 1283 space = uipc_stream_sbspace(sb); 1284 sorwakeup_locked(so2); 1285 if (!STAILQ_EMPTY(&mcnext.mc_q)) { 1286 /* 1287 * Such assignment is unsafe in general, but it is 1288 * safe with !STAILQ_EMPTY(&mcnext.mc_q). In C++ we 1289 * could reload = for STAILQs :) 1290 */ 1291 mc = mcnext; 1292 } else if (uio != NULL && uio->uio_resid > 0) { 1293 /* 1294 * Copyin sum of peer's receive buffer space and our 1295 * sb_hiwat, which is our virtual send buffer size. 1296 * See comment above unpst_sendspace declaration. 1297 * We are reading sb_hiwat locklessly, cause a) we 1298 * don't care about an application that does send(2) 1299 * and setsockopt(2) racing internally, and for an 1300 * application that does this in sequence we will see 1301 * the correct value cause sbsetopt() uses buffer lock 1302 * and we also have already acquired it at least once. 1303 */ 1304 error = mc_uiotomc(&mc, uio, space + 1305 atomic_load_int(&so->so_snd.sb_hiwat), 0, M_WAITOK, 1306 eor ? M_EOR : 0); 1307 if (__predict_false(error)) 1308 goto out4; 1309 } else 1310 mc = MCHAIN_INITIALIZER(&mc); 1311 } 1312 1313 MPASS(STAILQ_EMPTY(&mc.mc_q)); 1314 1315 td->td_ru.ru_msgsnd++; 1316 out4: 1317 sorele(so2); 1318 out3: 1319 SOCK_IO_SEND_UNLOCK(so); 1320 out2: 1321 if (aio) { 1322 freeuio(uio); 1323 uioadvance(uio0, sent); 1324 } else if (uio != NULL) 1325 uio->uio_resid = resid - sent; 1326 if (!mc_empty(&cmc)) 1327 unp_scan(mc_first(&cmc), unp_freerights); 1328 out: 1329 mc_freem(&mc); 1330 mc_freem(&cmc); 1331 1332 return (error); 1333 } 1334 1335 /* 1336 * Wakeup a writer, used by recv(2) and shutdown(2). 1337 * 1338 * @param so Points to a connected stream socket with receive buffer locked 1339 * 1340 * In a blocking mode peer is sleeping on our receive buffer, and we need just 1341 * wakeup(9) on it. But to wake up various event engines, we need to reach 1342 * over to peer's selinfo. This can be safely done as the socket buffer 1343 * receive lock is protecting us from the peer going away. 1344 */ 1345 static void 1346 uipc_wakeup_writer(struct socket *so) 1347 { 1348 struct sockbuf *sb = &so->so_rcv; 1349 struct selinfo *sel; 1350 1351 SOCK_RECVBUF_LOCK_ASSERT(so); 1352 MPASS(sb->uxst_peer != NULL); 1353 1354 sel = &sb->uxst_peer->so_wrsel; 1355 1356 if (sb->uxst_flags & UXST_PEER_SEL) { 1357 selwakeuppri(sel, PSOCK); 1358 /* 1359 * XXXGL: sowakeup() does SEL_WAITING() without locks. 1360 */ 1361 if (!SEL_WAITING(sel)) 1362 sb->uxst_flags &= ~UXST_PEER_SEL; 1363 } 1364 if (sb->sb_flags & SB_WAIT) { 1365 sb->sb_flags &= ~SB_WAIT; 1366 wakeup(&sb->sb_acc); 1367 } 1368 KNOTE_LOCKED(&sel->si_note, 0); 1369 SOCK_RECVBUF_UNLOCK(so); 1370 } 1371 1372 static void 1373 uipc_cantrcvmore(struct socket *so) 1374 { 1375 1376 SOCK_RECVBUF_LOCK(so); 1377 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 1378 selwakeuppri(&so->so_rdsel, PSOCK); 1379 KNOTE_LOCKED(&so->so_rdsel.si_note, 0); 1380 if (so->so_rcv.uxst_peer != NULL) 1381 uipc_wakeup_writer(so); 1382 else 1383 SOCK_RECVBUF_UNLOCK(so); 1384 } 1385 1386 static int 1387 uipc_soreceive_stream_or_seqpacket(struct socket *so, struct sockaddr **psa, 1388 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1389 { 1390 struct sockbuf *sb = &so->so_rcv; 1391 struct mbuf *control, *m, *first, *part, *next; 1392 u_int ctl, space, datalen, mbcnt, partlen; 1393 int error, flags; 1394 bool nonblock, waitall, peek; 1395 1396 MPASS(mp0 == NULL); 1397 1398 if (psa != NULL) 1399 *psa = NULL; 1400 if (controlp != NULL) 1401 *controlp = NULL; 1402 1403 flags = flagsp != NULL ? *flagsp : 0; 1404 nonblock = (so->so_state & SS_NBIO) || 1405 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1406 peek = flags & MSG_PEEK; 1407 waitall = (flags & MSG_WAITALL) && !peek; 1408 1409 /* 1410 * This check may fail only on a socket that never went through 1411 * connect(2). We can check this locklessly, cause: a) for a new born 1412 * socket we don't care about applications that may race internally 1413 * between connect(2) and recv(2), and b) for a dying socket if we 1414 * miss update by unp_sosidisconnected(), we would still get the check 1415 * correct. For dying socket we would observe SBS_CANTRCVMORE later. 1416 */ 1417 if (__predict_false((atomic_load_short(&so->so_state) & 1418 (SS_ISCONNECTED|SS_ISDISCONNECTED)) == 0)) 1419 return (ENOTCONN); 1420 1421 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 1422 if (__predict_false(error)) 1423 return (error); 1424 1425 restart: 1426 SOCK_RECVBUF_LOCK(so); 1427 UIPC_STREAM_SBCHECK(sb); 1428 while (sb->sb_acc < sb->sb_lowat && 1429 (sb->sb_ctl == 0 || controlp == NULL)) { 1430 if (so->so_error) { 1431 error = so->so_error; 1432 if (!peek) 1433 so->so_error = 0; 1434 SOCK_RECVBUF_UNLOCK(so); 1435 SOCK_IO_RECV_UNLOCK(so); 1436 return (error); 1437 } 1438 if (sb->sb_state & SBS_CANTRCVMORE) { 1439 SOCK_RECVBUF_UNLOCK(so); 1440 SOCK_IO_RECV_UNLOCK(so); 1441 return (0); 1442 } 1443 if (nonblock) { 1444 SOCK_RECVBUF_UNLOCK(so); 1445 SOCK_IO_RECV_UNLOCK(so); 1446 return (EWOULDBLOCK); 1447 } 1448 error = sbwait(so, SO_RCV); 1449 if (error) { 1450 SOCK_RECVBUF_UNLOCK(so); 1451 SOCK_IO_RECV_UNLOCK(so); 1452 return (error); 1453 } 1454 } 1455 1456 MPASS(STAILQ_FIRST(&sb->uxst_mbq)); 1457 MPASS(sb->sb_acc > 0 || sb->sb_ctl > 0); 1458 1459 mbcnt = 0; 1460 ctl = 0; 1461 first = STAILQ_FIRST(&sb->uxst_mbq); 1462 if (first->m_type == MT_CONTROL) { 1463 control = first; 1464 STAILQ_FOREACH_FROM(first, &sb->uxst_mbq, m_stailq) { 1465 if (first->m_type != MT_CONTROL) 1466 break; 1467 ctl += first->m_len; 1468 mbcnt += MSIZE; 1469 if (first->m_flags & M_EXT) 1470 mbcnt += first->m_ext.ext_size; 1471 } 1472 } else 1473 control = NULL; 1474 1475 /* 1476 * Find split point for the next copyout. On exit from the loop, 1477 * 'next' points to the new head of the buffer STAILQ and 'datalen' 1478 * contains the amount of data we will copy out at the end. The 1479 * copyout is protected by the I/O lock only, as writers can only 1480 * append to the buffer. We need to record the socket buffer state 1481 * and do all length adjustments before dropping the socket buffer lock. 1482 */ 1483 for (space = uio->uio_resid, m = next = first, part = NULL, datalen = 0; 1484 space > 0 && m != sb->uxst_fnrdy && m->m_type == MT_DATA; 1485 m = STAILQ_NEXT(m, m_stailq)) { 1486 if (space >= m->m_len) { 1487 space -= m->m_len; 1488 datalen += m->m_len; 1489 mbcnt += MSIZE; 1490 if (m->m_flags & M_EXT) 1491 mbcnt += m->m_ext.ext_size; 1492 if (m->m_flags & M_EOR) { 1493 flags |= MSG_EOR; 1494 next = STAILQ_NEXT(m, m_stailq); 1495 break; 1496 } 1497 } else { 1498 datalen += space; 1499 partlen = space; 1500 if (!peek) { 1501 m->m_len -= partlen; 1502 m->m_data += partlen; 1503 } 1504 next = part = m; 1505 break; 1506 } 1507 next = STAILQ_NEXT(m, m_stailq); 1508 } 1509 1510 if (!peek) { 1511 if (next == NULL) 1512 STAILQ_INIT(&sb->uxst_mbq); 1513 else 1514 STAILQ_FIRST(&sb->uxst_mbq) = next; 1515 MPASS(sb->sb_acc >= datalen); 1516 sb->sb_acc -= datalen; 1517 sb->sb_ccc -= datalen; 1518 MPASS(sb->sb_ctl >= ctl); 1519 sb->sb_ctl -= ctl; 1520 MPASS(sb->sb_mbcnt >= mbcnt); 1521 sb->sb_mbcnt -= mbcnt; 1522 UIPC_STREAM_SBCHECK(sb); 1523 if (__predict_true(sb->uxst_peer != NULL)) { 1524 struct unpcb *unp2; 1525 bool aio; 1526 1527 if ((aio = sb->uxst_flags & UXST_PEER_AIO)) 1528 sb->uxst_flags &= ~UXST_PEER_AIO; 1529 1530 uipc_wakeup_writer(so); 1531 /* 1532 * XXXGL: need to go through uipc_lock_peer() after 1533 * the receive buffer lock dropped, it was protecting 1534 * us from unp_soisdisconnected(). The aio workarounds 1535 * should be refactored to the aio(4) side. 1536 */ 1537 if (aio && uipc_lock_peer(so, &unp2) == 0) { 1538 struct socket *so2 = unp2->unp_socket; 1539 1540 SOCK_SENDBUF_LOCK(so2); 1541 so2->so_snd.sb_ccc -= datalen; 1542 sowakeup_aio(so2, SO_SND); 1543 SOCK_SENDBUF_UNLOCK(so2); 1544 UNP_PCB_UNLOCK(unp2); 1545 } 1546 } else 1547 SOCK_RECVBUF_UNLOCK(so); 1548 } else 1549 SOCK_RECVBUF_UNLOCK(so); 1550 1551 while (control != NULL && control->m_type == MT_CONTROL) { 1552 if (!peek) { 1553 /* 1554 * unp_externalize() failure must abort entire read(2). 1555 * Such failure should also free the problematic 1556 * control, but link back the remaining data to the head 1557 * of the buffer, so that socket is not left in a state 1558 * where it can't progress forward with reading. 1559 * Probability of such a failure is really low, so it 1560 * is fine that we need to perform pretty complex 1561 * operation here to reconstruct the buffer. 1562 */ 1563 error = unp_externalize(control, controlp, flags); 1564 control = m_free(control); 1565 if (__predict_false(error && control != NULL)) { 1566 struct mchain cmc; 1567 1568 mc_init_m(&cmc, control); 1569 1570 SOCK_RECVBUF_LOCK(so); 1571 if (__predict_false( 1572 (sb->sb_state & SBS_CANTRCVMORE) || 1573 cmc.mc_len + sb->sb_ccc + sb->sb_ctl > 1574 sb->sb_hiwat)) { 1575 /* 1576 * While the lock was dropped and we 1577 * were failing in unp_externalize(), 1578 * the peer could has a) disconnected, 1579 * b) filled the buffer so that we 1580 * can't prepend data back. 1581 * These are two edge conditions that 1582 * we just can't handle, so lose the 1583 * data and return the error. 1584 */ 1585 SOCK_RECVBUF_UNLOCK(so); 1586 SOCK_IO_RECV_UNLOCK(so); 1587 unp_scan(mc_first(&cmc), 1588 unp_freerights); 1589 mc_freem(&cmc); 1590 return (error); 1591 } 1592 1593 UIPC_STREAM_SBCHECK(sb); 1594 /* XXXGL: STAILQ_PREPEND */ 1595 STAILQ_CONCAT(&cmc.mc_q, &sb->uxst_mbq); 1596 STAILQ_SWAP(&cmc.mc_q, &sb->uxst_mbq, mbuf); 1597 1598 sb->sb_ctl = sb->sb_acc = sb->sb_ccc = 1599 sb->sb_mbcnt = 0; 1600 STAILQ_FOREACH(m, &sb->uxst_mbq, m_stailq) { 1601 if (m->m_type == MT_DATA) { 1602 sb->sb_acc += m->m_len; 1603 sb->sb_ccc += m->m_len; 1604 } else { 1605 sb->sb_ctl += m->m_len; 1606 } 1607 sb->sb_mbcnt += MSIZE; 1608 if (m->m_flags & M_EXT) 1609 sb->sb_mbcnt += 1610 m->m_ext.ext_size; 1611 } 1612 UIPC_STREAM_SBCHECK(sb); 1613 SOCK_RECVBUF_UNLOCK(so); 1614 SOCK_IO_RECV_UNLOCK(so); 1615 return (error); 1616 } 1617 if (controlp != NULL) { 1618 while (*controlp != NULL) 1619 controlp = &(*controlp)->m_next; 1620 } 1621 } else { 1622 /* 1623 * XXXGL 1624 * 1625 * In MSG_PEEK case control is not externalized. This 1626 * means we are leaking some kernel pointers to the 1627 * userland. They are useless to a law-abiding 1628 * application, but may be useful to a malware. This 1629 * is what the historical implementation in the 1630 * soreceive_generic() did. To be improved? 1631 */ 1632 if (controlp != NULL) { 1633 *controlp = m_copym(control, 0, control->m_len, 1634 M_WAITOK); 1635 controlp = &(*controlp)->m_next; 1636 } 1637 control = STAILQ_NEXT(control, m_stailq); 1638 } 1639 } 1640 1641 for (m = first; datalen > 0; m = next) { 1642 void *data; 1643 u_int len; 1644 1645 next = STAILQ_NEXT(m, m_stailq); 1646 if (m == part) { 1647 data = peek ? 1648 mtod(m, char *) : mtod(m, char *) - partlen; 1649 len = partlen; 1650 } else { 1651 data = mtod(m, char *); 1652 len = m->m_len; 1653 } 1654 error = uiomove(data, len, uio); 1655 if (__predict_false(error)) { 1656 if (!peek) 1657 for (; m != part && datalen > 0; m = next) { 1658 next = STAILQ_NEXT(m, m_stailq); 1659 MPASS(datalen >= m->m_len); 1660 datalen -= m->m_len; 1661 m_free(m); 1662 } 1663 SOCK_IO_RECV_UNLOCK(so); 1664 return (error); 1665 } 1666 datalen -= len; 1667 if (!peek && m != part) 1668 m_free(m); 1669 } 1670 if (waitall && !(flags & MSG_EOR) && uio->uio_resid > 0) 1671 goto restart; 1672 SOCK_IO_RECV_UNLOCK(so); 1673 1674 if (flagsp != NULL) 1675 *flagsp |= flags; 1676 1677 uio->uio_td->td_ru.ru_msgrcv++; 1678 1679 return (0); 1680 } 1681 1682 static int 1683 uipc_sopoll_stream_or_seqpacket(struct socket *so, int events, 1684 struct thread *td) 1685 { 1686 struct unpcb *unp = sotounpcb(so); 1687 int revents; 1688 1689 UNP_PCB_LOCK(unp); 1690 if (SOLISTENING(so)) { 1691 /* The above check is safe, since conversion to listening uses 1692 * both protocol and socket lock. 1693 */ 1694 SOCK_LOCK(so); 1695 if (!(events & (POLLIN | POLLRDNORM))) 1696 revents = 0; 1697 else if (!TAILQ_EMPTY(&so->sol_comp)) 1698 revents = events & (POLLIN | POLLRDNORM); 1699 else if (so->so_error) 1700 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 1701 else { 1702 selrecord(td, &so->so_rdsel); 1703 revents = 0; 1704 } 1705 SOCK_UNLOCK(so); 1706 } else { 1707 if (so->so_state & SS_ISDISCONNECTED) 1708 revents = POLLHUP; 1709 else 1710 revents = 0; 1711 if (events & (POLLIN | POLLRDNORM | POLLRDHUP)) { 1712 SOCK_RECVBUF_LOCK(so); 1713 if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat || 1714 so->so_error || so->so_rerror) 1715 revents |= events & (POLLIN | POLLRDNORM); 1716 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1717 revents |= events & 1718 (POLLIN | POLLRDNORM | POLLRDHUP); 1719 if (!(revents & (POLLIN | POLLRDNORM | POLLRDHUP))) { 1720 selrecord(td, &so->so_rdsel); 1721 so->so_rcv.sb_flags |= SB_SEL; 1722 } 1723 SOCK_RECVBUF_UNLOCK(so); 1724 } 1725 if (events & (POLLOUT | POLLWRNORM)) { 1726 struct socket *so2 = so->so_rcv.uxst_peer; 1727 1728 if (so2 != NULL) { 1729 struct sockbuf *sb = &so2->so_rcv; 1730 1731 SOCK_RECVBUF_LOCK(so2); 1732 if (uipc_stream_sbspace(sb) >= sb->sb_lowat) 1733 revents |= events & 1734 (POLLOUT | POLLWRNORM); 1735 if (sb->sb_state & SBS_CANTRCVMORE) 1736 revents |= POLLHUP; 1737 if (!(revents & (POLLOUT | POLLWRNORM))) { 1738 so2->so_rcv.uxst_flags |= UXST_PEER_SEL; 1739 selrecord(td, &so->so_wrsel); 1740 } 1741 SOCK_RECVBUF_UNLOCK(so2); 1742 } else 1743 selrecord(td, &so->so_wrsel); 1744 } 1745 } 1746 UNP_PCB_UNLOCK(unp); 1747 return (revents); 1748 } 1749 1750 static void 1751 uipc_wrknl_lock(void *arg) 1752 { 1753 struct socket *so = arg; 1754 struct unpcb *unp = sotounpcb(so); 1755 1756 retry: 1757 if (SOLISTENING(so)) { 1758 SOLISTEN_LOCK(so); 1759 } else { 1760 UNP_PCB_LOCK(unp); 1761 if (__predict_false(SOLISTENING(so))) { 1762 UNP_PCB_UNLOCK(unp); 1763 goto retry; 1764 } 1765 if (so->so_rcv.uxst_peer != NULL) 1766 SOCK_RECVBUF_LOCK(so->so_rcv.uxst_peer); 1767 } 1768 } 1769 1770 static void 1771 uipc_wrknl_unlock(void *arg) 1772 { 1773 struct socket *so = arg; 1774 struct unpcb *unp = sotounpcb(so); 1775 1776 if (SOLISTENING(so)) 1777 SOLISTEN_UNLOCK(so); 1778 else { 1779 if (so->so_rcv.uxst_peer != NULL) 1780 SOCK_RECVBUF_UNLOCK(so->so_rcv.uxst_peer); 1781 UNP_PCB_UNLOCK(unp); 1782 } 1783 } 1784 1785 static void 1786 uipc_wrknl_assert_lock(void *arg, int what) 1787 { 1788 struct socket *so = arg; 1789 1790 if (SOLISTENING(so)) { 1791 if (what == LA_LOCKED) 1792 SOLISTEN_LOCK_ASSERT(so); 1793 else 1794 SOLISTEN_UNLOCK_ASSERT(so); 1795 } else { 1796 /* 1797 * The pr_soreceive method will put a note without owning the 1798 * unp lock, so we can't assert it here. But we can safely 1799 * dereference uxst_peer pointer, since receive buffer lock 1800 * is assumed to be held here. 1801 */ 1802 if (what == LA_LOCKED && so->so_rcv.uxst_peer != NULL) 1803 SOCK_RECVBUF_LOCK_ASSERT(so->so_rcv.uxst_peer); 1804 } 1805 } 1806 1807 static void 1808 uipc_filt_sowdetach(struct knote *kn) 1809 { 1810 struct socket *so = kn->kn_fp->f_data; 1811 1812 uipc_wrknl_lock(so); 1813 knlist_remove(&so->so_wrsel.si_note, kn, 1); 1814 uipc_wrknl_unlock(so); 1815 } 1816 1817 static int 1818 uipc_filt_sowrite(struct knote *kn, long hint) 1819 { 1820 struct socket *so = kn->kn_fp->f_data, *so2; 1821 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1822 1823 if (SOLISTENING(so)) 1824 return (0); 1825 1826 if (unp2 == NULL) { 1827 if (so->so_state & SS_ISDISCONNECTED) { 1828 kn->kn_flags |= EV_EOF; 1829 kn->kn_fflags = so->so_error; 1830 return (1); 1831 } else 1832 return (0); 1833 } 1834 1835 so2 = unp2->unp_socket; 1836 SOCK_RECVBUF_LOCK_ASSERT(so2); 1837 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1838 1839 if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { 1840 kn->kn_flags |= EV_EOF; 1841 return (1); 1842 } else if (kn->kn_sfflags & NOTE_LOWAT) 1843 return (kn->kn_data >= kn->kn_sdata); 1844 else 1845 return (kn->kn_data >= so2->so_rcv.sb_lowat); 1846 } 1847 1848 static int 1849 uipc_filt_soempty(struct knote *kn, long hint) 1850 { 1851 struct socket *so = kn->kn_fp->f_data, *so2; 1852 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1853 1854 if (SOLISTENING(so) || unp2 == NULL) 1855 return (1); 1856 1857 so2 = unp2->unp_socket; 1858 SOCK_RECVBUF_LOCK_ASSERT(so2); 1859 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1860 1861 return (kn->kn_data == 0 ? 1 : 0); 1862 } 1863 1864 static const struct filterops uipc_write_filtops = { 1865 .f_isfd = 1, 1866 .f_detach = uipc_filt_sowdetach, 1867 .f_event = uipc_filt_sowrite, 1868 .f_copy = knote_triv_copy, 1869 }; 1870 static const struct filterops uipc_empty_filtops = { 1871 .f_isfd = 1, 1872 .f_detach = uipc_filt_sowdetach, 1873 .f_event = uipc_filt_soempty, 1874 .f_copy = knote_triv_copy, 1875 }; 1876 1877 static int 1878 uipc_kqfilter_stream_or_seqpacket(struct socket *so, struct knote *kn) 1879 { 1880 struct unpcb *unp = sotounpcb(so); 1881 struct knlist *knl; 1882 1883 switch (kn->kn_filter) { 1884 case EVFILT_READ: 1885 return (sokqfilter_generic(so, kn)); 1886 case EVFILT_WRITE: 1887 kn->kn_fop = &uipc_write_filtops; 1888 break; 1889 case EVFILT_EMPTY: 1890 kn->kn_fop = &uipc_empty_filtops; 1891 break; 1892 default: 1893 return (EINVAL); 1894 } 1895 1896 knl = &so->so_wrsel.si_note; 1897 UNP_PCB_LOCK(unp); 1898 if (SOLISTENING(so)) { 1899 SOLISTEN_LOCK(so); 1900 knlist_add(knl, kn, 1); 1901 SOLISTEN_UNLOCK(so); 1902 } else { 1903 struct socket *so2 = so->so_rcv.uxst_peer; 1904 1905 if (so2 != NULL) 1906 SOCK_RECVBUF_LOCK(so2); 1907 knlist_add(knl, kn, 1); 1908 if (so2 != NULL) 1909 SOCK_RECVBUF_UNLOCK(so2); 1910 } 1911 UNP_PCB_UNLOCK(unp); 1912 return (0); 1913 } 1914 1915 /* PF_UNIX/SOCK_DGRAM version of sbspace() */ 1916 static inline bool 1917 uipc_dgram_sbspace(struct sockbuf *sb, u_int cc, u_int mbcnt) 1918 { 1919 u_int bleft, mleft; 1920 1921 /* 1922 * Negative space may happen if send(2) is followed by 1923 * setsockopt(SO_SNDBUF/SO_RCVBUF) that shrinks maximum. 1924 */ 1925 if (__predict_false(sb->sb_hiwat < sb->uxdg_cc || 1926 sb->sb_mbmax < sb->uxdg_mbcnt)) 1927 return (false); 1928 1929 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) 1930 return (false); 1931 1932 bleft = sb->sb_hiwat - sb->uxdg_cc; 1933 mleft = sb->sb_mbmax - sb->uxdg_mbcnt; 1934 1935 return (bleft >= cc && mleft >= mbcnt); 1936 } 1937 1938 /* 1939 * PF_UNIX/SOCK_DGRAM send 1940 * 1941 * Allocate a record consisting of 3 mbufs in the sequence of 1942 * from -> control -> data and append it to the socket buffer. 1943 * 1944 * The first mbuf carries sender's name and is a pkthdr that stores 1945 * overall length of datagram, its memory consumption and control length. 1946 */ 1947 #define ctllen PH_loc.thirtytwo[1] 1948 _Static_assert(offsetof(struct pkthdr, memlen) + sizeof(u_int) <= 1949 offsetof(struct pkthdr, ctllen), "unix/dgram can not store ctllen"); 1950 static int 1951 uipc_sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1952 struct mbuf *m, struct mbuf *c, int flags, struct thread *td) 1953 { 1954 struct unpcb *unp, *unp2; 1955 const struct sockaddr *from; 1956 struct socket *so2; 1957 struct sockbuf *sb; 1958 struct mchain cmc = MCHAIN_INITIALIZER(&cmc); 1959 struct mbuf *f; 1960 u_int cc, ctl, mbcnt; 1961 u_int dcc __diagused, dctl __diagused, dmbcnt __diagused; 1962 int error; 1963 1964 MPASS((uio != NULL && m == NULL) || (m != NULL && uio == NULL)); 1965 1966 error = 0; 1967 f = NULL; 1968 1969 if (__predict_false(flags & MSG_OOB)) { 1970 error = EOPNOTSUPP; 1971 goto out; 1972 } 1973 if (m == NULL) { 1974 if (__predict_false(uio->uio_resid > unpdg_maxdgram)) { 1975 error = EMSGSIZE; 1976 goto out; 1977 } 1978 m = m_uiotombuf(uio, M_WAITOK, 0, max_hdr, M_PKTHDR); 1979 if (__predict_false(m == NULL)) { 1980 error = EFAULT; 1981 goto out; 1982 } 1983 f = m_gethdr(M_WAITOK, MT_SONAME); 1984 cc = m->m_pkthdr.len; 1985 mbcnt = MSIZE + m->m_pkthdr.memlen; 1986 if (c != NULL && (error = unp_internalize(c, &cmc, td))) 1987 goto out; 1988 } else { 1989 struct mchain mc; 1990 1991 uipc_reset_kernel_mbuf(m, &mc); 1992 cc = mc.mc_len; 1993 mbcnt = mc.mc_mlen; 1994 if (__predict_false(m->m_pkthdr.len > unpdg_maxdgram)) { 1995 error = EMSGSIZE; 1996 goto out; 1997 } 1998 if ((f = m_gethdr(M_NOWAIT, MT_SONAME)) == NULL) { 1999 error = ENOBUFS; 2000 goto out; 2001 } 2002 } 2003 2004 unp = sotounpcb(so); 2005 MPASS(unp); 2006 2007 /* 2008 * XXXGL: would be cool to fully remove so_snd out of the equation 2009 * and avoid this lock, which is not only extraneous, but also being 2010 * released, thus still leaving possibility for a race. We can easily 2011 * handle SBS_CANTSENDMORE/SS_ISCONNECTED complement in unpcb, but it 2012 * is more difficult to invent something to handle so_error. 2013 */ 2014 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2015 if (error) 2016 goto out2; 2017 SOCK_SENDBUF_LOCK(so); 2018 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2019 SOCK_SENDBUF_UNLOCK(so); 2020 error = EPIPE; 2021 goto out3; 2022 } 2023 if (so->so_error != 0) { 2024 error = so->so_error; 2025 so->so_error = 0; 2026 SOCK_SENDBUF_UNLOCK(so); 2027 goto out3; 2028 } 2029 if (((so->so_state & SS_ISCONNECTED) == 0) && addr == NULL) { 2030 SOCK_SENDBUF_UNLOCK(so); 2031 error = EDESTADDRREQ; 2032 goto out3; 2033 } 2034 SOCK_SENDBUF_UNLOCK(so); 2035 2036 if (addr != NULL) { 2037 if ((error = unp_connectat(AT_FDCWD, so, addr, td, true))) 2038 goto out3; 2039 UNP_PCB_LOCK_ASSERT(unp); 2040 unp2 = unp->unp_conn; 2041 UNP_PCB_LOCK_ASSERT(unp2); 2042 } else { 2043 UNP_PCB_LOCK(unp); 2044 unp2 = unp_pcb_lock_peer(unp); 2045 if (unp2 == NULL) { 2046 UNP_PCB_UNLOCK(unp); 2047 error = ENOTCONN; 2048 goto out3; 2049 } 2050 } 2051 2052 if (unp2->unp_flags & UNP_WANTCRED_MASK) 2053 unp_addsockcred(td, &cmc, unp2->unp_flags); 2054 if (unp->unp_addr != NULL) 2055 from = (struct sockaddr *)unp->unp_addr; 2056 else 2057 from = &sun_noname; 2058 f->m_len = from->sa_len; 2059 MPASS(from->sa_len <= MLEN); 2060 bcopy(from, mtod(f, void *), from->sa_len); 2061 2062 /* 2063 * Concatenate mbufs: from -> control -> data. 2064 * Save overall cc and mbcnt in "from" mbuf. 2065 */ 2066 if (!STAILQ_EMPTY(&cmc.mc_q)) { 2067 f->m_next = mc_first(&cmc); 2068 mc_last(&cmc)->m_next = m; 2069 /* XXXGL: This is dirty as well as rollback after ENOBUFS. */ 2070 STAILQ_INIT(&cmc.mc_q); 2071 } else 2072 f->m_next = m; 2073 m = NULL; 2074 ctl = f->m_len + cmc.mc_len; 2075 mbcnt += cmc.mc_mlen; 2076 #ifdef INVARIANTS 2077 dcc = dctl = dmbcnt = 0; 2078 for (struct mbuf *mb = f; mb != NULL; mb = mb->m_next) { 2079 if (mb->m_type == MT_DATA) 2080 dcc += mb->m_len; 2081 else 2082 dctl += mb->m_len; 2083 dmbcnt += MSIZE; 2084 if (mb->m_flags & M_EXT) 2085 dmbcnt += mb->m_ext.ext_size; 2086 } 2087 MPASS(dcc == cc); 2088 MPASS(dctl == ctl); 2089 MPASS(dmbcnt == mbcnt); 2090 #endif 2091 f->m_pkthdr.len = cc + ctl; 2092 f->m_pkthdr.memlen = mbcnt; 2093 f->m_pkthdr.ctllen = ctl; 2094 2095 /* 2096 * Destination socket buffer selection. 2097 * 2098 * Unconnected sends, when !(so->so_state & SS_ISCONNECTED) and the 2099 * destination address is supplied, create a temporary connection for 2100 * the run time of the function (see call to unp_connectat() above and 2101 * to unp_disconnect() below). We distinguish them by condition of 2102 * (addr != NULL). We intentionally avoid adding 'bool connected' for 2103 * that condition, since, again, through the run time of this code we 2104 * are always connected. For such "unconnected" sends, the destination 2105 * buffer would be the receive buffer of destination socket so2. 2106 * 2107 * For connected sends, data lands on the send buffer of the sender's 2108 * socket "so". Then, if we just added the very first datagram 2109 * on this send buffer, we need to add the send buffer on to the 2110 * receiving socket's buffer list. We put ourselves on top of the 2111 * list. Such logic gives infrequent senders priority over frequent 2112 * senders. 2113 * 2114 * Note on byte count management. As long as event methods kevent(2), 2115 * select(2) are not protocol specific (yet), we need to maintain 2116 * meaningful values on the receive buffer. So, the receive buffer 2117 * would accumulate counters from all connected buffers potentially 2118 * having sb_ccc > sb_hiwat or sb_mbcnt > sb_mbmax. 2119 */ 2120 so2 = unp2->unp_socket; 2121 sb = (addr == NULL) ? &so->so_snd : &so2->so_rcv; 2122 SOCK_RECVBUF_LOCK(so2); 2123 if (uipc_dgram_sbspace(sb, cc + ctl, mbcnt)) { 2124 if (addr == NULL && STAILQ_EMPTY(&sb->uxdg_mb)) 2125 TAILQ_INSERT_HEAD(&so2->so_rcv.uxdg_conns, &so->so_snd, 2126 uxdg_clist); 2127 STAILQ_INSERT_TAIL(&sb->uxdg_mb, f, m_stailqpkt); 2128 sb->uxdg_cc += cc + ctl; 2129 sb->uxdg_ctl += ctl; 2130 sb->uxdg_mbcnt += mbcnt; 2131 so2->so_rcv.sb_acc += cc + ctl; 2132 so2->so_rcv.sb_ccc += cc + ctl; 2133 so2->so_rcv.sb_ctl += ctl; 2134 so2->so_rcv.sb_mbcnt += mbcnt; 2135 sorwakeup_locked(so2); 2136 f = NULL; 2137 } else { 2138 soroverflow_locked(so2); 2139 error = ENOBUFS; 2140 if (f->m_next->m_type == MT_CONTROL) { 2141 STAILQ_FIRST(&cmc.mc_q) = f->m_next; 2142 f->m_next = NULL; 2143 } 2144 } 2145 2146 if (addr != NULL) 2147 unp_disconnect(unp, unp2); 2148 else 2149 unp_pcb_unlock_pair(unp, unp2); 2150 2151 td->td_ru.ru_msgsnd++; 2152 2153 out3: 2154 SOCK_IO_SEND_UNLOCK(so); 2155 out2: 2156 if (!mc_empty(&cmc)) 2157 unp_scan(mc_first(&cmc), unp_freerights); 2158 out: 2159 if (f) 2160 m_freem(f); 2161 mc_freem(&cmc); 2162 if (m) 2163 m_freem(m); 2164 2165 return (error); 2166 } 2167 2168 /* 2169 * PF_UNIX/SOCK_DGRAM receive with MSG_PEEK. 2170 * The mbuf has already been unlinked from the uxdg_mb of socket buffer 2171 * and needs to be linked onto uxdg_peeked of receive socket buffer. 2172 */ 2173 static int 2174 uipc_peek_dgram(struct socket *so, struct mbuf *m, struct sockaddr **psa, 2175 struct uio *uio, struct mbuf **controlp, int *flagsp) 2176 { 2177 ssize_t len = 0; 2178 int error; 2179 2180 so->so_rcv.uxdg_peeked = m; 2181 so->so_rcv.uxdg_cc += m->m_pkthdr.len; 2182 so->so_rcv.uxdg_ctl += m->m_pkthdr.ctllen; 2183 so->so_rcv.uxdg_mbcnt += m->m_pkthdr.memlen; 2184 SOCK_RECVBUF_UNLOCK(so); 2185 2186 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2187 if (psa != NULL) 2188 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2189 2190 m = m->m_next; 2191 KASSERT(m, ("%s: no data or control after soname", __func__)); 2192 2193 /* 2194 * With MSG_PEEK the control isn't executed, just copied. 2195 */ 2196 while (m != NULL && m->m_type == MT_CONTROL) { 2197 if (controlp != NULL) { 2198 *controlp = m_copym(m, 0, m->m_len, M_WAITOK); 2199 controlp = &(*controlp)->m_next; 2200 } 2201 m = m->m_next; 2202 } 2203 KASSERT(m == NULL || m->m_type == MT_DATA, 2204 ("%s: not MT_DATA mbuf %p", __func__, m)); 2205 while (m != NULL && uio->uio_resid > 0) { 2206 len = uio->uio_resid; 2207 if (len > m->m_len) 2208 len = m->m_len; 2209 error = uiomove(mtod(m, char *), (int)len, uio); 2210 if (error) { 2211 SOCK_IO_RECV_UNLOCK(so); 2212 return (error); 2213 } 2214 if (len == m->m_len) 2215 m = m->m_next; 2216 } 2217 SOCK_IO_RECV_UNLOCK(so); 2218 2219 if (flagsp != NULL) { 2220 if (m != NULL) { 2221 if (*flagsp & MSG_TRUNC) { 2222 /* Report real length of the packet */ 2223 uio->uio_resid -= m_length(m, NULL) - len; 2224 } 2225 *flagsp |= MSG_TRUNC; 2226 } else 2227 *flagsp &= ~MSG_TRUNC; 2228 } 2229 2230 return (0); 2231 } 2232 2233 /* 2234 * PF_UNIX/SOCK_DGRAM receive 2235 */ 2236 static int 2237 uipc_soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2238 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2239 { 2240 struct sockbuf *sb = NULL; 2241 struct mbuf *m; 2242 int flags, error; 2243 ssize_t len = 0; 2244 bool nonblock; 2245 2246 MPASS(mp0 == NULL); 2247 2248 if (psa != NULL) 2249 *psa = NULL; 2250 if (controlp != NULL) 2251 *controlp = NULL; 2252 2253 flags = flagsp != NULL ? *flagsp : 0; 2254 nonblock = (so->so_state & SS_NBIO) || 2255 (flags & (MSG_DONTWAIT | MSG_NBIO)); 2256 2257 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 2258 if (__predict_false(error)) 2259 return (error); 2260 2261 /* 2262 * Loop blocking while waiting for a datagram. Prioritize connected 2263 * peers over unconnected sends. Set sb to selected socket buffer 2264 * containing an mbuf on exit from the wait loop. A datagram that 2265 * had already been peeked at has top priority. 2266 */ 2267 SOCK_RECVBUF_LOCK(so); 2268 while ((m = so->so_rcv.uxdg_peeked) == NULL && 2269 (sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) == NULL && 2270 (m = STAILQ_FIRST(&so->so_rcv.uxdg_mb)) == NULL) { 2271 if (so->so_error) { 2272 error = so->so_error; 2273 if (!(flags & MSG_PEEK)) 2274 so->so_error = 0; 2275 SOCK_RECVBUF_UNLOCK(so); 2276 SOCK_IO_RECV_UNLOCK(so); 2277 return (error); 2278 } 2279 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2280 uio->uio_resid == 0) { 2281 SOCK_RECVBUF_UNLOCK(so); 2282 SOCK_IO_RECV_UNLOCK(so); 2283 return (0); 2284 } 2285 if (nonblock) { 2286 SOCK_RECVBUF_UNLOCK(so); 2287 SOCK_IO_RECV_UNLOCK(so); 2288 return (EWOULDBLOCK); 2289 } 2290 error = sbwait(so, SO_RCV); 2291 if (error) { 2292 SOCK_RECVBUF_UNLOCK(so); 2293 SOCK_IO_RECV_UNLOCK(so); 2294 return (error); 2295 } 2296 } 2297 2298 if (sb == NULL) 2299 sb = &so->so_rcv; 2300 else if (m == NULL) 2301 m = STAILQ_FIRST(&sb->uxdg_mb); 2302 else 2303 MPASS(m == so->so_rcv.uxdg_peeked); 2304 2305 MPASS(sb->uxdg_cc > 0); 2306 M_ASSERTPKTHDR(m); 2307 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2308 2309 if (uio->uio_td) 2310 uio->uio_td->td_ru.ru_msgrcv++; 2311 2312 if (__predict_true(m != so->so_rcv.uxdg_peeked)) { 2313 STAILQ_REMOVE_HEAD(&sb->uxdg_mb, m_stailqpkt); 2314 if (STAILQ_EMPTY(&sb->uxdg_mb) && sb != &so->so_rcv) 2315 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 2316 } else 2317 so->so_rcv.uxdg_peeked = NULL; 2318 2319 sb->uxdg_cc -= m->m_pkthdr.len; 2320 sb->uxdg_ctl -= m->m_pkthdr.ctllen; 2321 sb->uxdg_mbcnt -= m->m_pkthdr.memlen; 2322 2323 if (__predict_false(flags & MSG_PEEK)) 2324 return (uipc_peek_dgram(so, m, psa, uio, controlp, flagsp)); 2325 2326 so->so_rcv.sb_acc -= m->m_pkthdr.len; 2327 so->so_rcv.sb_ccc -= m->m_pkthdr.len; 2328 so->so_rcv.sb_ctl -= m->m_pkthdr.ctllen; 2329 so->so_rcv.sb_mbcnt -= m->m_pkthdr.memlen; 2330 SOCK_RECVBUF_UNLOCK(so); 2331 2332 if (psa != NULL) 2333 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2334 m = m_free(m); 2335 KASSERT(m, ("%s: no data or control after soname", __func__)); 2336 2337 /* 2338 * Packet to copyout() is now in 'm' and it is disconnected from the 2339 * queue. 2340 * 2341 * Process one or more MT_CONTROL mbufs present before any data mbufs 2342 * in the first mbuf chain on the socket buffer. We call into the 2343 * unp_externalize() to perform externalization (or freeing if 2344 * controlp == NULL). In some cases there can be only MT_CONTROL mbufs 2345 * without MT_DATA mbufs. 2346 */ 2347 while (m != NULL && m->m_type == MT_CONTROL) { 2348 error = unp_externalize(m, controlp, flags); 2349 m = m_free(m); 2350 if (error != 0) { 2351 SOCK_IO_RECV_UNLOCK(so); 2352 unp_scan(m, unp_freerights); 2353 m_freem(m); 2354 return (error); 2355 } 2356 if (controlp != NULL) { 2357 while (*controlp != NULL) 2358 controlp = &(*controlp)->m_next; 2359 } 2360 } 2361 KASSERT(m == NULL || m->m_type == MT_DATA, 2362 ("%s: not MT_DATA mbuf %p", __func__, m)); 2363 while (m != NULL && uio->uio_resid > 0) { 2364 len = uio->uio_resid; 2365 if (len > m->m_len) 2366 len = m->m_len; 2367 error = uiomove(mtod(m, char *), (int)len, uio); 2368 if (error) { 2369 SOCK_IO_RECV_UNLOCK(so); 2370 m_freem(m); 2371 return (error); 2372 } 2373 if (len == m->m_len) 2374 m = m_free(m); 2375 else { 2376 m->m_data += len; 2377 m->m_len -= len; 2378 } 2379 } 2380 SOCK_IO_RECV_UNLOCK(so); 2381 2382 if (m != NULL) { 2383 if (flagsp != NULL) { 2384 if (flags & MSG_TRUNC) { 2385 /* Report real length of the packet */ 2386 uio->uio_resid -= m_length(m, NULL); 2387 } 2388 *flagsp |= MSG_TRUNC; 2389 } 2390 m_freem(m); 2391 } else if (flagsp != NULL) 2392 *flagsp &= ~MSG_TRUNC; 2393 2394 return (0); 2395 } 2396 2397 static int 2398 uipc_sendfile_wait(struct socket *so, off_t need, int *space) 2399 { 2400 struct unpcb *unp2; 2401 struct socket *so2; 2402 struct sockbuf *sb; 2403 bool nonblock, sockref; 2404 int error; 2405 2406 MPASS(so->so_type == SOCK_STREAM); 2407 MPASS(need > 0); 2408 MPASS(space != NULL); 2409 2410 nonblock = so->so_state & SS_NBIO; 2411 sockref = false; 2412 2413 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) 2414 return (ENOTCONN); 2415 2416 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2417 return (error); 2418 2419 so2 = unp2->unp_socket; 2420 sb = &so2->so_rcv; 2421 SOCK_RECVBUF_LOCK(so2); 2422 UNP_PCB_UNLOCK(unp2); 2423 while ((*space = uipc_stream_sbspace(sb)) < need && 2424 (*space < so->so_snd.sb_hiwat / 2)) { 2425 UIPC_STREAM_SBCHECK(sb); 2426 if (nonblock) { 2427 SOCK_RECVBUF_UNLOCK(so2); 2428 return (EAGAIN); 2429 } 2430 if (!sockref) 2431 soref(so2); 2432 error = uipc_stream_sbwait(so2, so->so_snd.sb_timeo); 2433 if (error == 0 && 2434 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) 2435 error = EPIPE; 2436 if (error) { 2437 SOCK_RECVBUF_UNLOCK(so2); 2438 sorele(so2); 2439 return (error); 2440 } 2441 } 2442 UIPC_STREAM_SBCHECK(sb); 2443 SOCK_RECVBUF_UNLOCK(so2); 2444 if (sockref) 2445 sorele(so2); 2446 2447 return (0); 2448 } 2449 2450 /* 2451 * Although this is a pr_send method, for unix(4) it is called only via 2452 * sendfile(2) path. This means we can be sure that mbufs are clear of 2453 * any extra flags and don't require any conditioning. 2454 */ 2455 static int 2456 uipc_sendfile(struct socket *so, int flags, struct mbuf *m, 2457 struct sockaddr *from, struct mbuf *control, struct thread *td) 2458 { 2459 struct mchain mc; 2460 struct unpcb *unp2; 2461 struct socket *so2; 2462 struct sockbuf *sb; 2463 bool notready, wakeup; 2464 int error; 2465 2466 MPASS(so->so_type == SOCK_STREAM); 2467 MPASS(from == NULL && control == NULL); 2468 KASSERT(!(m->m_flags & M_EXTPG), 2469 ("unix(4): TLS sendfile(2) not supported")); 2470 2471 notready = flags & PRUS_NOTREADY; 2472 2473 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) { 2474 error = ENOTCONN; 2475 goto out; 2476 } 2477 2478 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2479 goto out; 2480 2481 mc_init_m(&mc, m); 2482 2483 so2 = unp2->unp_socket; 2484 sb = &so2->so_rcv; 2485 SOCK_RECVBUF_LOCK(so2); 2486 UNP_PCB_UNLOCK(unp2); 2487 UIPC_STREAM_SBCHECK(sb); 2488 sb->sb_ccc += mc.mc_len; 2489 sb->sb_mbcnt += mc.mc_mlen; 2490 if (sb->uxst_fnrdy == NULL) { 2491 if (notready) { 2492 wakeup = false; 2493 STAILQ_FOREACH(m, &mc.mc_q, m_stailq) { 2494 if (m->m_flags & M_NOTREADY) { 2495 sb->uxst_fnrdy = m; 2496 break; 2497 } else { 2498 sb->sb_acc += m->m_len; 2499 wakeup = true; 2500 } 2501 } 2502 } else { 2503 wakeup = true; 2504 sb->sb_acc += mc.mc_len; 2505 } 2506 } else { 2507 wakeup = false; 2508 } 2509 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 2510 UIPC_STREAM_SBCHECK(sb); 2511 if (wakeup) 2512 sorwakeup_locked(so2); 2513 else 2514 SOCK_RECVBUF_UNLOCK(so2); 2515 2516 return (0); 2517 out: 2518 /* 2519 * In case of not ready data, uipc_ready() is responsible 2520 * for freeing memory. 2521 */ 2522 if (m != NULL && !notready) 2523 m_freem(m); 2524 2525 return (error); 2526 } 2527 2528 static int 2529 uipc_sbready(struct sockbuf *sb, struct mbuf *m, int count) 2530 { 2531 bool blocker; 2532 2533 /* assert locked */ 2534 2535 blocker = (sb->uxst_fnrdy == m); 2536 STAILQ_FOREACH_FROM(m, &sb->uxst_mbq, m_stailq) { 2537 if (count > 0) { 2538 MPASS(m->m_flags & M_NOTREADY); 2539 m->m_flags &= ~M_NOTREADY; 2540 if (blocker) 2541 sb->sb_acc += m->m_len; 2542 count--; 2543 } else if (m->m_flags & M_NOTREADY) 2544 break; 2545 else if (blocker) 2546 sb->sb_acc += m->m_len; 2547 } 2548 if (blocker) { 2549 sb->uxst_fnrdy = m; 2550 return (0); 2551 } else 2552 return (EINPROGRESS); 2553 } 2554 2555 static bool 2556 uipc_ready_scan(struct socket *so, struct mbuf *m, int count, int *errorp) 2557 { 2558 struct mbuf *mb; 2559 struct sockbuf *sb; 2560 2561 SOCK_LOCK(so); 2562 if (SOLISTENING(so)) { 2563 SOCK_UNLOCK(so); 2564 return (false); 2565 } 2566 mb = NULL; 2567 sb = &so->so_rcv; 2568 SOCK_RECVBUF_LOCK(so); 2569 if (sb->uxst_fnrdy != NULL) { 2570 STAILQ_FOREACH(mb, &sb->uxst_mbq, m_stailq) { 2571 if (mb == m) { 2572 *errorp = uipc_sbready(sb, m, count); 2573 break; 2574 } 2575 } 2576 } 2577 SOCK_RECVBUF_UNLOCK(so); 2578 SOCK_UNLOCK(so); 2579 return (mb != NULL); 2580 } 2581 2582 static int 2583 uipc_ready(struct socket *so, struct mbuf *m, int count) 2584 { 2585 struct unpcb *unp, *unp2; 2586 int error; 2587 2588 MPASS(so->so_type == SOCK_STREAM); 2589 2590 if (__predict_true(uipc_lock_peer(so, &unp2) == 0)) { 2591 struct socket *so2; 2592 struct sockbuf *sb; 2593 2594 so2 = unp2->unp_socket; 2595 sb = &so2->so_rcv; 2596 SOCK_RECVBUF_LOCK(so2); 2597 UNP_PCB_UNLOCK(unp2); 2598 UIPC_STREAM_SBCHECK(sb); 2599 error = uipc_sbready(sb, m, count); 2600 UIPC_STREAM_SBCHECK(sb); 2601 if (error == 0) 2602 sorwakeup_locked(so2); 2603 else 2604 SOCK_RECVBUF_UNLOCK(so2); 2605 } else { 2606 /* 2607 * The receiving socket has been disconnected, but may still 2608 * be valid. In this case, the not-ready mbufs are still 2609 * present in its socket buffer, so perform an exhaustive 2610 * search before giving up and freeing the mbufs. 2611 */ 2612 UNP_LINK_RLOCK(); 2613 LIST_FOREACH(unp, &unp_shead, unp_link) { 2614 if (uipc_ready_scan(unp->unp_socket, m, count, &error)) 2615 break; 2616 } 2617 UNP_LINK_RUNLOCK(); 2618 2619 if (unp == NULL) { 2620 for (int i = 0; i < count; i++) 2621 m = m_free(m); 2622 return (ECONNRESET); 2623 } 2624 } 2625 return (error); 2626 } 2627 2628 static int 2629 uipc_sense(struct socket *so, struct stat *sb) 2630 { 2631 struct unpcb *unp; 2632 2633 unp = sotounpcb(so); 2634 KASSERT(unp != NULL, ("uipc_sense: unp == NULL")); 2635 2636 sb->st_blksize = so->so_snd.sb_hiwat; 2637 sb->st_dev = NODEV; 2638 sb->st_ino = unp->unp_ino; 2639 return (0); 2640 } 2641 2642 static int 2643 uipc_shutdown(struct socket *so, enum shutdown_how how) 2644 { 2645 struct unpcb *unp = sotounpcb(so); 2646 int error; 2647 2648 SOCK_LOCK(so); 2649 if (SOLISTENING(so)) { 2650 if (how != SHUT_WR) { 2651 so->so_error = ECONNABORTED; 2652 solisten_wakeup(so); /* unlocks so */ 2653 } else 2654 SOCK_UNLOCK(so); 2655 return (ENOTCONN); 2656 } else if ((so->so_state & 2657 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2658 /* 2659 * POSIX mandates us to just return ENOTCONN when shutdown(2) is 2660 * invoked on a datagram sockets, however historically we would 2661 * actually tear socket down. This is known to be leveraged by 2662 * some applications to unblock process waiting in recv(2) by 2663 * other process that it shares that socket with. Try to meet 2664 * both backward-compatibility and POSIX requirements by forcing 2665 * ENOTCONN but still flushing buffers and performing wakeup(9). 2666 * 2667 * XXXGL: it remains unknown what applications expect this 2668 * behavior and is this isolated to unix/dgram or inet/dgram or 2669 * both. See: D10351, D3039. 2670 */ 2671 error = ENOTCONN; 2672 if (so->so_type != SOCK_DGRAM) { 2673 SOCK_UNLOCK(so); 2674 return (error); 2675 } 2676 } else 2677 error = 0; 2678 SOCK_UNLOCK(so); 2679 2680 switch (how) { 2681 case SHUT_RD: 2682 if (so->so_type == SOCK_DGRAM) 2683 socantrcvmore(so); 2684 else 2685 uipc_cantrcvmore(so); 2686 unp_dispose(so); 2687 break; 2688 case SHUT_RDWR: 2689 if (so->so_type == SOCK_DGRAM) 2690 socantrcvmore(so); 2691 else 2692 uipc_cantrcvmore(so); 2693 unp_dispose(so); 2694 /* FALLTHROUGH */ 2695 case SHUT_WR: 2696 if (so->so_type == SOCK_DGRAM) { 2697 socantsendmore(so); 2698 } else { 2699 UNP_PCB_LOCK(unp); 2700 if (unp->unp_conn != NULL) 2701 uipc_cantrcvmore(unp->unp_conn->unp_socket); 2702 UNP_PCB_UNLOCK(unp); 2703 } 2704 } 2705 wakeup(&so->so_timeo); 2706 2707 return (error); 2708 } 2709 2710 static int 2711 uipc_sockaddr(struct socket *so, struct sockaddr *ret) 2712 { 2713 struct unpcb *unp; 2714 const struct sockaddr *sa; 2715 2716 unp = sotounpcb(so); 2717 KASSERT(unp != NULL, ("uipc_sockaddr: unp == NULL")); 2718 2719 UNP_PCB_LOCK(unp); 2720 if (unp->unp_addr != NULL) 2721 sa = (struct sockaddr *) unp->unp_addr; 2722 else 2723 sa = &sun_noname; 2724 bcopy(sa, ret, sa->sa_len); 2725 UNP_PCB_UNLOCK(unp); 2726 return (0); 2727 } 2728 2729 static int 2730 uipc_ctloutput(struct socket *so, struct sockopt *sopt) 2731 { 2732 struct unpcb *unp; 2733 struct xucred xu; 2734 int error, optval; 2735 2736 if (sopt->sopt_level != SOL_LOCAL) 2737 return (EINVAL); 2738 2739 unp = sotounpcb(so); 2740 KASSERT(unp != NULL, ("uipc_ctloutput: unp == NULL")); 2741 error = 0; 2742 switch (sopt->sopt_dir) { 2743 case SOPT_GET: 2744 switch (sopt->sopt_name) { 2745 case LOCAL_PEERCRED: 2746 UNP_PCB_LOCK(unp); 2747 if (unp->unp_flags & UNP_HAVEPC) 2748 xu = unp->unp_peercred; 2749 else { 2750 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2751 error = ENOTCONN; 2752 else 2753 error = EINVAL; 2754 } 2755 UNP_PCB_UNLOCK(unp); 2756 if (error != 0) 2757 break; 2758 #ifdef COMPAT_FREEBSD32 2759 if (sopt->sopt_td && 2760 SV_PROC_FLAG(sopt->sopt_td->td_proc, SV_ILP32)) 2761 { 2762 struct xucred32 xu32 = {}; 2763 int i; 2764 2765 xu32.cr_version = xu.cr_version; 2766 xu32.cr_uid = xu.cr_uid; 2767 xu32.cr_ngroups = xu.cr_ngroups; 2768 for (i = 0; i < XU_NGROUPS; i++) 2769 xu32.cr_groups[i] = xu.cr_groups[i]; 2770 xu32.cr_pid = xu.cr_pid; 2771 error = sooptcopyout(sopt, &xu32, sizeof(xu32)); 2772 break; 2773 } 2774 #endif 2775 error = sooptcopyout(sopt, &xu, sizeof(xu)); 2776 break; 2777 2778 case LOCAL_CREDS: 2779 /* Unlocked read. */ 2780 optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0; 2781 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2782 break; 2783 2784 case LOCAL_CREDS_PERSISTENT: 2785 /* Unlocked read. */ 2786 optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0; 2787 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2788 break; 2789 2790 default: 2791 error = EOPNOTSUPP; 2792 break; 2793 } 2794 break; 2795 2796 case SOPT_SET: 2797 switch (sopt->sopt_name) { 2798 case LOCAL_CREDS: 2799 case LOCAL_CREDS_PERSISTENT: 2800 error = sooptcopyin(sopt, &optval, sizeof(optval), 2801 sizeof(optval)); 2802 if (error) 2803 break; 2804 2805 #define OPTSET(bit, exclusive) do { \ 2806 UNP_PCB_LOCK(unp); \ 2807 if (optval) { \ 2808 if ((unp->unp_flags & (exclusive)) != 0) { \ 2809 UNP_PCB_UNLOCK(unp); \ 2810 error = EINVAL; \ 2811 break; \ 2812 } \ 2813 unp->unp_flags |= (bit); \ 2814 } else \ 2815 unp->unp_flags &= ~(bit); \ 2816 UNP_PCB_UNLOCK(unp); \ 2817 } while (0) 2818 2819 switch (sopt->sopt_name) { 2820 case LOCAL_CREDS: 2821 OPTSET(UNP_WANTCRED_ONESHOT, UNP_WANTCRED_ALWAYS); 2822 break; 2823 2824 case LOCAL_CREDS_PERSISTENT: 2825 OPTSET(UNP_WANTCRED_ALWAYS, UNP_WANTCRED_ONESHOT); 2826 break; 2827 2828 default: 2829 break; 2830 } 2831 break; 2832 #undef OPTSET 2833 default: 2834 error = ENOPROTOOPT; 2835 break; 2836 } 2837 break; 2838 2839 default: 2840 error = EOPNOTSUPP; 2841 break; 2842 } 2843 return (error); 2844 } 2845 2846 static int 2847 unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 2848 { 2849 2850 return (unp_connectat(AT_FDCWD, so, nam, td, false)); 2851 } 2852 2853 static int 2854 unp_connectat(int fd, struct socket *so, struct sockaddr *nam, 2855 struct thread *td, bool return_locked) 2856 { 2857 struct mtx *vplock; 2858 struct sockaddr_un *soun; 2859 struct vnode *vp; 2860 struct socket *so2; 2861 struct unpcb *unp, *unp2, *unp3; 2862 struct nameidata nd; 2863 char buf[SOCK_MAXADDRLEN]; 2864 struct sockaddr *sa; 2865 cap_rights_t rights; 2866 int error, len; 2867 bool connreq; 2868 2869 CURVNET_ASSERT_SET(); 2870 2871 if (nam->sa_family != AF_UNIX) 2872 return (EAFNOSUPPORT); 2873 if (nam->sa_len > sizeof(struct sockaddr_un)) 2874 return (EINVAL); 2875 len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); 2876 if (len <= 0) 2877 return (EINVAL); 2878 soun = (struct sockaddr_un *)nam; 2879 bcopy(soun->sun_path, buf, len); 2880 buf[len] = 0; 2881 2882 error = 0; 2883 unp = sotounpcb(so); 2884 UNP_PCB_LOCK(unp); 2885 for (;;) { 2886 /* 2887 * Wait for connection state to stabilize. If a connection 2888 * already exists, give up. For datagram sockets, which permit 2889 * multiple consecutive connect(2) calls, upper layers are 2890 * responsible for disconnecting in advance of a subsequent 2891 * connect(2), but this is not synchronized with PCB connection 2892 * state. 2893 * 2894 * Also make sure that no threads are currently attempting to 2895 * lock the peer socket, to ensure that unp_conn cannot 2896 * transition between two valid sockets while locks are dropped. 2897 */ 2898 if (SOLISTENING(so)) 2899 error = EOPNOTSUPP; 2900 else if (unp->unp_conn != NULL) 2901 error = EISCONN; 2902 else if ((unp->unp_flags & UNP_CONNECTING) != 0) { 2903 error = EALREADY; 2904 } 2905 if (error != 0) { 2906 UNP_PCB_UNLOCK(unp); 2907 return (error); 2908 } 2909 if (unp->unp_pairbusy > 0) { 2910 unp->unp_flags |= UNP_WAITING; 2911 mtx_sleep(unp, UNP_PCB_LOCKPTR(unp), 0, "unpeer", 0); 2912 continue; 2913 } 2914 break; 2915 } 2916 unp->unp_flags |= UNP_CONNECTING; 2917 UNP_PCB_UNLOCK(unp); 2918 2919 connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0; 2920 if (connreq) 2921 sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK); 2922 else 2923 sa = NULL; 2924 NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF | 2925 (fd == AT_FDCWD ? 0 : EMPTYPATH), UIO_SYSSPACE, buf, fd, 2926 cap_rights_init_one(&rights, CAP_CONNECTAT)); 2927 error = namei(&nd); 2928 if (error) 2929 vp = NULL; 2930 else 2931 vp = nd.ni_vp; 2932 ASSERT_VOP_LOCKED(vp, "unp_connect"); 2933 if (error) 2934 goto bad; 2935 NDFREE_PNBUF(&nd); 2936 2937 if (vp->v_type != VSOCK) { 2938 error = ENOTSOCK; 2939 goto bad; 2940 } 2941 #ifdef MAC 2942 error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD); 2943 if (error) 2944 goto bad; 2945 #endif 2946 error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td); 2947 if (error) 2948 goto bad; 2949 2950 unp = sotounpcb(so); 2951 KASSERT(unp != NULL, ("unp_connect: unp == NULL")); 2952 2953 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 2954 mtx_lock(vplock); 2955 VOP_UNP_CONNECT(vp, &unp2); 2956 if (unp2 == NULL) { 2957 error = ECONNREFUSED; 2958 goto bad2; 2959 } 2960 so2 = unp2->unp_socket; 2961 if (so->so_type != so2->so_type) { 2962 error = EPROTOTYPE; 2963 goto bad2; 2964 } 2965 if (connreq) { 2966 if (SOLISTENING(so2)) 2967 so2 = solisten_clone(so2); 2968 else 2969 so2 = NULL; 2970 if (so2 == NULL) { 2971 error = ECONNREFUSED; 2972 goto bad2; 2973 } 2974 if ((error = uipc_attach(so2, 0, NULL)) != 0) { 2975 sodealloc(so2); 2976 goto bad2; 2977 } 2978 unp3 = sotounpcb(so2); 2979 unp_pcb_lock_pair(unp2, unp3); 2980 if (unp2->unp_addr != NULL) { 2981 bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); 2982 unp3->unp_addr = (struct sockaddr_un *) sa; 2983 sa = NULL; 2984 } 2985 2986 unp_copy_peercred(td, unp3, unp, unp2); 2987 2988 UNP_PCB_UNLOCK(unp2); 2989 unp2 = unp3; 2990 2991 /* 2992 * It is safe to block on the PCB lock here since unp2 is 2993 * nascent and cannot be connected to any other sockets. 2994 */ 2995 UNP_PCB_LOCK(unp); 2996 #ifdef MAC 2997 mac_socketpeer_set_from_socket(so, so2); 2998 mac_socketpeer_set_from_socket(so2, so); 2999 #endif 3000 } else { 3001 unp_pcb_lock_pair(unp, unp2); 3002 } 3003 KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && 3004 sotounpcb(so2) == unp2, 3005 ("%s: unp2 %p so2 %p", __func__, unp2, so2)); 3006 unp_connect2(so, so2, connreq); 3007 if (connreq) 3008 (void)solisten_enqueue(so2, SS_ISCONNECTED); 3009 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 3010 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 3011 unp->unp_flags &= ~UNP_CONNECTING; 3012 if (!return_locked) 3013 unp_pcb_unlock_pair(unp, unp2); 3014 bad2: 3015 mtx_unlock(vplock); 3016 bad: 3017 if (vp != NULL) { 3018 /* 3019 * If we are returning locked (called via uipc_sosend_dgram()), 3020 * we need to be sure that vput() won't sleep. This is 3021 * guaranteed by VOP_UNP_CONNECT() call above and unp2 lock. 3022 * SOCK_STREAM/SEQPACKET can't request return_locked (yet). 3023 */ 3024 MPASS(!(return_locked && connreq)); 3025 vput(vp); 3026 } 3027 free(sa, M_SONAME); 3028 if (__predict_false(error)) { 3029 UNP_PCB_LOCK(unp); 3030 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 3031 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 3032 unp->unp_flags &= ~UNP_CONNECTING; 3033 UNP_PCB_UNLOCK(unp); 3034 } 3035 return (error); 3036 } 3037 3038 /* 3039 * Set socket peer credentials at connection time. 3040 * 3041 * The client's PCB credentials are copied from its process structure. The 3042 * server's PCB credentials are copied from the socket on which it called 3043 * listen(2). uipc_listen cached that process's credentials at the time. 3044 */ 3045 void 3046 unp_copy_peercred(struct thread *td, struct unpcb *client_unp, 3047 struct unpcb *server_unp, struct unpcb *listen_unp) 3048 { 3049 cru2xt(td, &client_unp->unp_peercred); 3050 client_unp->unp_flags |= UNP_HAVEPC; 3051 3052 memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred, 3053 sizeof(server_unp->unp_peercred)); 3054 server_unp->unp_flags |= UNP_HAVEPC; 3055 client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK); 3056 } 3057 3058 /* 3059 * unix/stream & unix/seqpacket version of soisconnected(). 3060 * 3061 * The crucial thing we are doing here is setting up the uxst_peer linkage, 3062 * holding unp and receive buffer locks of the both sockets. The disconnect 3063 * procedure does the same. This gives as a safe way to access the peer in the 3064 * send(2) and recv(2) during the socket lifetime. 3065 * 3066 * The less important thing is event notification of the fact that a socket is 3067 * now connected. It is unusual for a software to put a socket into event 3068 * mechanism before connect(2), but is supposed to be supported. Note that 3069 * there can not be any sleeping I/O on the socket, yet, only presence in the 3070 * select/poll/kevent. 3071 * 3072 * This function can be called via two call paths: 3073 * 1) socketpair(2) - in this case socket has not been yet reported to userland 3074 * and just can't have any event notifications mechanisms set up. The 3075 * 'wakeup' boolean is always false. 3076 * 2) connect(2) of existing socket to a recent clone of a listener: 3077 * 2.1) Socket that connect(2)s will have 'wakeup' true. An application 3078 * could have already put it into event mechanism, is it shall be 3079 * reported as readable and as writable. 3080 * 2.2) Socket that was just cloned with solisten_clone(). Same as 1). 3081 */ 3082 static void 3083 unp_soisconnected(struct socket *so, bool wakeup) 3084 { 3085 struct socket *so2 = sotounpcb(so)->unp_conn->unp_socket; 3086 struct sockbuf *sb; 3087 3088 SOCK_LOCK_ASSERT(so); 3089 UNP_PCB_LOCK_ASSERT(sotounpcb(so)); 3090 UNP_PCB_LOCK_ASSERT(sotounpcb(so2)); 3091 SOCK_RECVBUF_LOCK_ASSERT(so); 3092 SOCK_RECVBUF_LOCK_ASSERT(so2); 3093 3094 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3095 MPASS((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 3096 SS_ISDISCONNECTING)) == 0); 3097 MPASS(so->so_qstate == SQ_NONE); 3098 3099 so->so_state &= ~SS_ISDISCONNECTED; 3100 so->so_state |= SS_ISCONNECTED; 3101 3102 sb = &so2->so_rcv; 3103 sb->uxst_peer = so; 3104 3105 if (wakeup) { 3106 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); 3107 sb = &so->so_rcv; 3108 selwakeuppri(sb->sb_sel, PSOCK); 3109 SOCK_SENDBUF_LOCK_ASSERT(so); 3110 sb = &so->so_snd; 3111 selwakeuppri(sb->sb_sel, PSOCK); 3112 SOCK_SENDBUF_UNLOCK(so); 3113 } 3114 } 3115 3116 static void 3117 unp_connect2(struct socket *so, struct socket *so2, bool wakeup) 3118 { 3119 struct unpcb *unp; 3120 struct unpcb *unp2; 3121 3122 MPASS(so2->so_type == so->so_type); 3123 unp = sotounpcb(so); 3124 KASSERT(unp != NULL, ("unp_connect2: unp == NULL")); 3125 unp2 = sotounpcb(so2); 3126 KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL")); 3127 3128 UNP_PCB_LOCK_ASSERT(unp); 3129 UNP_PCB_LOCK_ASSERT(unp2); 3130 KASSERT(unp->unp_conn == NULL, 3131 ("%s: socket %p is already connected", __func__, unp)); 3132 3133 unp->unp_conn = unp2; 3134 unp_pcb_hold(unp2); 3135 unp_pcb_hold(unp); 3136 switch (so->so_type) { 3137 case SOCK_DGRAM: 3138 UNP_REF_LIST_LOCK(); 3139 LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); 3140 UNP_REF_LIST_UNLOCK(); 3141 soisconnected(so); 3142 break; 3143 3144 case SOCK_STREAM: 3145 case SOCK_SEQPACKET: 3146 KASSERT(unp2->unp_conn == NULL, 3147 ("%s: socket %p is already connected", __func__, unp2)); 3148 unp2->unp_conn = unp; 3149 SOCK_LOCK(so); 3150 SOCK_LOCK(so2); 3151 if (wakeup) /* Avoid LOR with receive buffer lock. */ 3152 SOCK_SENDBUF_LOCK(so); 3153 SOCK_RECVBUF_LOCK(so); 3154 SOCK_RECVBUF_LOCK(so2); 3155 unp_soisconnected(so, wakeup); /* Will unlock send buffer. */ 3156 unp_soisconnected(so2, false); 3157 SOCK_RECVBUF_UNLOCK(so); 3158 SOCK_RECVBUF_UNLOCK(so2); 3159 SOCK_UNLOCK(so); 3160 SOCK_UNLOCK(so2); 3161 break; 3162 3163 default: 3164 panic("unp_connect2"); 3165 } 3166 } 3167 3168 static void 3169 unp_soisdisconnected(struct socket *so) 3170 { 3171 SOCK_LOCK_ASSERT(so); 3172 SOCK_RECVBUF_LOCK_ASSERT(so); 3173 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3174 MPASS(!SOLISTENING(so)); 3175 MPASS((so->so_state & (SS_ISCONNECTING | SS_ISDISCONNECTING | 3176 SS_ISDISCONNECTED)) == 0); 3177 MPASS(so->so_state & SS_ISCONNECTED); 3178 3179 so->so_state |= SS_ISDISCONNECTED; 3180 so->so_state &= ~SS_ISCONNECTED; 3181 so->so_rcv.uxst_peer = NULL; 3182 selwakeuppri(&so->so_wrsel, PSOCK); 3183 KNOTE_LOCKED(&so->so_snd.sb_sel->si_note, 0); 3184 socantrcvmore_locked(so); 3185 } 3186 3187 static void 3188 unp_disconnect(struct unpcb *unp, struct unpcb *unp2) 3189 { 3190 struct socket *so, *so2; 3191 struct mbuf *m = NULL; 3192 #ifdef INVARIANTS 3193 struct unpcb *unptmp; 3194 #endif 3195 3196 UNP_PCB_LOCK_ASSERT(unp); 3197 UNP_PCB_LOCK_ASSERT(unp2); 3198 KASSERT(unp->unp_conn == unp2, 3199 ("%s: unpcb %p is not connected to %p", __func__, unp, unp2)); 3200 3201 unp->unp_conn = NULL; 3202 so = unp->unp_socket; 3203 so2 = unp2->unp_socket; 3204 switch (unp->unp_socket->so_type) { 3205 case SOCK_DGRAM: 3206 /* 3207 * Remove our send socket buffer from the peer's receive buffer. 3208 * Move the data to the receive buffer only if it is empty. 3209 * This is a protection against a scenario where a peer 3210 * connects, floods and disconnects, effectively blocking 3211 * sendto() from unconnected sockets. 3212 */ 3213 SOCK_RECVBUF_LOCK(so2); 3214 if (!STAILQ_EMPTY(&so->so_snd.uxdg_mb)) { 3215 TAILQ_REMOVE(&so2->so_rcv.uxdg_conns, &so->so_snd, 3216 uxdg_clist); 3217 if (__predict_true((so2->so_rcv.sb_state & 3218 SBS_CANTRCVMORE) == 0) && 3219 STAILQ_EMPTY(&so2->so_rcv.uxdg_mb)) { 3220 STAILQ_CONCAT(&so2->so_rcv.uxdg_mb, 3221 &so->so_snd.uxdg_mb); 3222 so2->so_rcv.uxdg_cc += so->so_snd.uxdg_cc; 3223 so2->so_rcv.uxdg_ctl += so->so_snd.uxdg_ctl; 3224 so2->so_rcv.uxdg_mbcnt += so->so_snd.uxdg_mbcnt; 3225 } else { 3226 m = STAILQ_FIRST(&so->so_snd.uxdg_mb); 3227 STAILQ_INIT(&so->so_snd.uxdg_mb); 3228 so2->so_rcv.sb_acc -= so->so_snd.uxdg_cc; 3229 so2->so_rcv.sb_ccc -= so->so_snd.uxdg_cc; 3230 so2->so_rcv.sb_ctl -= so->so_snd.uxdg_ctl; 3231 so2->so_rcv.sb_mbcnt -= so->so_snd.uxdg_mbcnt; 3232 } 3233 /* Note: so may reconnect. */ 3234 so->so_snd.uxdg_cc = 0; 3235 so->so_snd.uxdg_ctl = 0; 3236 so->so_snd.uxdg_mbcnt = 0; 3237 } 3238 SOCK_RECVBUF_UNLOCK(so2); 3239 UNP_REF_LIST_LOCK(); 3240 #ifdef INVARIANTS 3241 LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) { 3242 if (unptmp == unp) 3243 break; 3244 } 3245 KASSERT(unptmp != NULL, 3246 ("%s: %p not found in reflist of %p", __func__, unp, unp2)); 3247 #endif 3248 LIST_REMOVE(unp, unp_reflink); 3249 UNP_REF_LIST_UNLOCK(); 3250 SOCK_LOCK(so); 3251 so->so_state &= ~SS_ISCONNECTED; 3252 SOCK_UNLOCK(so); 3253 break; 3254 3255 case SOCK_STREAM: 3256 case SOCK_SEQPACKET: 3257 SOCK_LOCK(so); 3258 SOCK_LOCK(so2); 3259 SOCK_RECVBUF_LOCK(so); 3260 SOCK_RECVBUF_LOCK(so2); 3261 unp_soisdisconnected(so); 3262 MPASS(unp2->unp_conn == unp); 3263 unp2->unp_conn = NULL; 3264 unp_soisdisconnected(so2); 3265 SOCK_UNLOCK(so); 3266 SOCK_UNLOCK(so2); 3267 break; 3268 } 3269 3270 if (unp == unp2) { 3271 unp_pcb_rele_notlast(unp); 3272 if (!unp_pcb_rele(unp)) 3273 UNP_PCB_UNLOCK(unp); 3274 } else { 3275 if (!unp_pcb_rele(unp)) 3276 UNP_PCB_UNLOCK(unp); 3277 if (!unp_pcb_rele(unp2)) 3278 UNP_PCB_UNLOCK(unp2); 3279 } 3280 3281 if (m != NULL) { 3282 unp_scan(m, unp_freerights); 3283 m_freemp(m); 3284 } 3285 } 3286 3287 /* 3288 * unp_pcblist() walks the global list of struct unpcb's to generate a 3289 * pointer list, bumping the refcount on each unpcb. It then copies them out 3290 * sequentially, validating the generation number on each to see if it has 3291 * been detached. All of this is necessary because copyout() may sleep on 3292 * disk I/O. 3293 */ 3294 static int 3295 unp_pcblist(SYSCTL_HANDLER_ARGS) 3296 { 3297 struct unpcb *unp, **unp_list; 3298 unp_gen_t gencnt; 3299 struct xunpgen *xug; 3300 struct unp_head *head; 3301 struct xunpcb *xu; 3302 u_int i; 3303 int error, n; 3304 3305 switch ((intptr_t)arg1) { 3306 case SOCK_STREAM: 3307 head = &unp_shead; 3308 break; 3309 3310 case SOCK_DGRAM: 3311 head = &unp_dhead; 3312 break; 3313 3314 case SOCK_SEQPACKET: 3315 head = &unp_sphead; 3316 break; 3317 3318 default: 3319 panic("unp_pcblist: arg1 %d", (int)(intptr_t)arg1); 3320 } 3321 3322 /* 3323 * The process of preparing the PCB list is too time-consuming and 3324 * resource-intensive to repeat twice on every request. 3325 */ 3326 if (req->oldptr == NULL) { 3327 n = unp_count; 3328 req->oldidx = 2 * (sizeof *xug) 3329 + (n + n/8) * sizeof(struct xunpcb); 3330 return (0); 3331 } 3332 3333 if (req->newptr != NULL) 3334 return (EPERM); 3335 3336 /* 3337 * OK, now we're committed to doing something. 3338 */ 3339 xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK | M_ZERO); 3340 UNP_LINK_RLOCK(); 3341 gencnt = unp_gencnt; 3342 n = unp_count; 3343 UNP_LINK_RUNLOCK(); 3344 3345 xug->xug_len = sizeof *xug; 3346 xug->xug_count = n; 3347 xug->xug_gen = gencnt; 3348 xug->xug_sogen = so_gencnt; 3349 error = SYSCTL_OUT(req, xug, sizeof *xug); 3350 if (error) { 3351 free(xug, M_TEMP); 3352 return (error); 3353 } 3354 3355 unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK); 3356 3357 UNP_LINK_RLOCK(); 3358 for (unp = LIST_FIRST(head), i = 0; unp && i < n; 3359 unp = LIST_NEXT(unp, unp_link)) { 3360 UNP_PCB_LOCK(unp); 3361 if (unp->unp_gencnt <= gencnt) { 3362 if (cr_cansee(req->td->td_ucred, 3363 unp->unp_socket->so_cred)) { 3364 UNP_PCB_UNLOCK(unp); 3365 continue; 3366 } 3367 unp_list[i++] = unp; 3368 unp_pcb_hold(unp); 3369 } 3370 UNP_PCB_UNLOCK(unp); 3371 } 3372 UNP_LINK_RUNLOCK(); 3373 n = i; /* In case we lost some during malloc. */ 3374 3375 error = 0; 3376 xu = malloc(sizeof(*xu), M_TEMP, M_WAITOK | M_ZERO); 3377 for (i = 0; i < n; i++) { 3378 unp = unp_list[i]; 3379 UNP_PCB_LOCK(unp); 3380 if (unp_pcb_rele(unp)) 3381 continue; 3382 3383 if (unp->unp_gencnt <= gencnt) { 3384 xu->xu_len = sizeof *xu; 3385 xu->xu_unpp = (uintptr_t)unp; 3386 /* 3387 * XXX - need more locking here to protect against 3388 * connect/disconnect races for SMP. 3389 */ 3390 if (unp->unp_addr != NULL) 3391 bcopy(unp->unp_addr, &xu->xu_addr, 3392 unp->unp_addr->sun_len); 3393 else 3394 bzero(&xu->xu_addr, sizeof(xu->xu_addr)); 3395 if (unp->unp_conn != NULL && 3396 unp->unp_conn->unp_addr != NULL) 3397 bcopy(unp->unp_conn->unp_addr, 3398 &xu->xu_caddr, 3399 unp->unp_conn->unp_addr->sun_len); 3400 else 3401 bzero(&xu->xu_caddr, sizeof(xu->xu_caddr)); 3402 xu->unp_vnode = (uintptr_t)unp->unp_vnode; 3403 xu->unp_conn = (uintptr_t)unp->unp_conn; 3404 xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs); 3405 xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink); 3406 xu->unp_gencnt = unp->unp_gencnt; 3407 sotoxsocket(unp->unp_socket, &xu->xu_socket); 3408 UNP_PCB_UNLOCK(unp); 3409 error = SYSCTL_OUT(req, xu, sizeof *xu); 3410 } else { 3411 UNP_PCB_UNLOCK(unp); 3412 } 3413 } 3414 free(xu, M_TEMP); 3415 if (!error) { 3416 /* 3417 * Give the user an updated idea of our state. If the 3418 * generation differs from what we told her before, she knows 3419 * that something happened while we were processing this 3420 * request, and it might be necessary to retry. 3421 */ 3422 xug->xug_gen = unp_gencnt; 3423 xug->xug_sogen = so_gencnt; 3424 xug->xug_count = unp_count; 3425 error = SYSCTL_OUT(req, xug, sizeof *xug); 3426 } 3427 free(unp_list, M_TEMP); 3428 free(xug, M_TEMP); 3429 return (error); 3430 } 3431 3432 SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, 3433 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3434 (void *)(intptr_t)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb", 3435 "List of active local datagram sockets"); 3436 SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, 3437 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3438 (void *)(intptr_t)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb", 3439 "List of active local stream sockets"); 3440 SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist, 3441 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3442 (void *)(intptr_t)SOCK_SEQPACKET, 0, unp_pcblist, "S,xunpcb", 3443 "List of active local seqpacket sockets"); 3444 3445 static void 3446 unp_drop(struct unpcb *unp) 3447 { 3448 struct socket *so; 3449 struct unpcb *unp2; 3450 3451 /* 3452 * Regardless of whether the socket's peer dropped the connection 3453 * with this socket by aborting or disconnecting, POSIX requires 3454 * that ECONNRESET is returned on next connected send(2) in case of 3455 * a SOCK_DGRAM socket and EPIPE for SOCK_STREAM. 3456 */ 3457 UNP_PCB_LOCK(unp); 3458 if ((so = unp->unp_socket) != NULL) 3459 so->so_error = 3460 so->so_proto->pr_type == SOCK_DGRAM ? ECONNRESET : EPIPE; 3461 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) { 3462 /* Last reference dropped in unp_disconnect(). */ 3463 unp_pcb_rele_notlast(unp); 3464 unp_disconnect(unp, unp2); 3465 } else if (!unp_pcb_rele(unp)) { 3466 UNP_PCB_UNLOCK(unp); 3467 } 3468 } 3469 3470 static void 3471 unp_freerights(struct filedescent **fdep, int fdcount) 3472 { 3473 struct file *fp; 3474 int i; 3475 3476 KASSERT(fdcount > 0, ("%s: fdcount %d", __func__, fdcount)); 3477 3478 for (i = 0; i < fdcount; i++) { 3479 fp = fdep[i]->fde_file; 3480 filecaps_free(&fdep[i]->fde_caps); 3481 unp_discard(fp); 3482 } 3483 free(fdep[0], M_FILECAPS); 3484 } 3485 3486 static bool 3487 restrict_rights(struct file *fp, struct thread *td) 3488 { 3489 struct prison *prison1, *prison2; 3490 3491 prison1 = fp->f_cred->cr_prison; 3492 prison2 = td->td_ucred->cr_prison; 3493 return (prison1 != prison2 && prison1->pr_root != prison2->pr_root && 3494 prison2 != &prison0); 3495 } 3496 3497 static int 3498 unp_externalize(struct mbuf *control, struct mbuf **controlp, int flags) 3499 { 3500 struct thread *td = curthread; /* XXX */ 3501 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 3502 int *fdp; 3503 struct filedesc *fdesc = td->td_proc->p_fd; 3504 struct filedescent **fdep; 3505 void *data; 3506 socklen_t clen = control->m_len, datalen; 3507 int error, fdflags, newfds; 3508 u_int newlen; 3509 3510 UNP_LINK_UNLOCK_ASSERT(); 3511 3512 fdflags = ((flags & MSG_CMSG_CLOEXEC) ? O_CLOEXEC : 0) | 3513 ((flags & MSG_CMSG_CLOFORK) ? O_CLOFORK : 0); 3514 3515 error = 0; 3516 if (controlp != NULL) /* controlp == NULL => free control messages */ 3517 *controlp = NULL; 3518 while (cm != NULL) { 3519 MPASS(clen >= sizeof(*cm) && clen >= cm->cmsg_len); 3520 3521 data = CMSG_DATA(cm); 3522 datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; 3523 if (cm->cmsg_level == SOL_SOCKET 3524 && cm->cmsg_type == SCM_RIGHTS) { 3525 newfds = datalen / sizeof(*fdep); 3526 if (newfds == 0) 3527 goto next; 3528 fdep = data; 3529 3530 /* If we're not outputting the descriptors free them. */ 3531 if (error || controlp == NULL) { 3532 unp_freerights(fdep, newfds); 3533 goto next; 3534 } 3535 FILEDESC_XLOCK(fdesc); 3536 3537 /* 3538 * Now change each pointer to an fd in the global 3539 * table to an integer that is the index to the local 3540 * fd table entry that we set up to point to the 3541 * global one we are transferring. 3542 */ 3543 newlen = newfds * sizeof(int); 3544 *controlp = sbcreatecontrol(NULL, newlen, 3545 SCM_RIGHTS, SOL_SOCKET, M_WAITOK); 3546 3547 fdp = (int *) 3548 CMSG_DATA(mtod(*controlp, struct cmsghdr *)); 3549 if ((error = fdallocn(td, 0, fdp, newfds))) { 3550 FILEDESC_XUNLOCK(fdesc); 3551 unp_freerights(fdep, newfds); 3552 m_freem(*controlp); 3553 *controlp = NULL; 3554 goto next; 3555 } 3556 for (int i = 0; i < newfds; i++, fdp++) { 3557 struct file *fp; 3558 3559 fp = fdep[i]->fde_file; 3560 _finstall(fdesc, fp, *fdp, fdflags | 3561 (restrict_rights(fp, td) ? 3562 O_RESOLVE_BENEATH : 0), &fdep[i]->fde_caps); 3563 unp_externalize_fp(fp); 3564 } 3565 3566 /* 3567 * The new type indicates that the mbuf data refers to 3568 * kernel resources that may need to be released before 3569 * the mbuf is freed. 3570 */ 3571 m_chtype(*controlp, MT_EXTCONTROL); 3572 FILEDESC_XUNLOCK(fdesc); 3573 free(fdep[0], M_FILECAPS); 3574 } else { 3575 /* We can just copy anything else across. */ 3576 if (error || controlp == NULL) 3577 goto next; 3578 *controlp = sbcreatecontrol(NULL, datalen, 3579 cm->cmsg_type, cm->cmsg_level, M_WAITOK); 3580 bcopy(data, 3581 CMSG_DATA(mtod(*controlp, struct cmsghdr *)), 3582 datalen); 3583 } 3584 controlp = &(*controlp)->m_next; 3585 3586 next: 3587 if (CMSG_SPACE(datalen) < clen) { 3588 clen -= CMSG_SPACE(datalen); 3589 cm = (struct cmsghdr *) 3590 ((caddr_t)cm + CMSG_SPACE(datalen)); 3591 } else { 3592 clen = 0; 3593 cm = NULL; 3594 } 3595 } 3596 3597 return (error); 3598 } 3599 3600 static void 3601 unp_zone_change(void *tag) 3602 { 3603 3604 uma_zone_set_max(unp_zone, maxsockets); 3605 } 3606 3607 #ifdef INVARIANTS 3608 static void 3609 unp_zdtor(void *mem, int size __unused, void *arg __unused) 3610 { 3611 struct unpcb *unp; 3612 3613 unp = mem; 3614 3615 KASSERT(LIST_EMPTY(&unp->unp_refs), 3616 ("%s: unpcb %p has lingering refs", __func__, unp)); 3617 KASSERT(unp->unp_socket == NULL, 3618 ("%s: unpcb %p has socket backpointer", __func__, unp)); 3619 KASSERT(unp->unp_vnode == NULL, 3620 ("%s: unpcb %p has vnode references", __func__, unp)); 3621 KASSERT(unp->unp_conn == NULL, 3622 ("%s: unpcb %p is still connected", __func__, unp)); 3623 KASSERT(unp->unp_addr == NULL, 3624 ("%s: unpcb %p has leaked addr", __func__, unp)); 3625 } 3626 #endif 3627 3628 static void 3629 unp_init(void *arg __unused) 3630 { 3631 uma_dtor dtor; 3632 3633 #ifdef INVARIANTS 3634 dtor = unp_zdtor; 3635 #else 3636 dtor = NULL; 3637 #endif 3638 unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, dtor, 3639 NULL, NULL, UMA_ALIGN_CACHE, 0); 3640 uma_zone_set_max(unp_zone, maxsockets); 3641 uma_zone_set_warning(unp_zone, "kern.ipc.maxsockets limit reached"); 3642 EVENTHANDLER_REGISTER(maxsockets_change, unp_zone_change, 3643 NULL, EVENTHANDLER_PRI_ANY); 3644 LIST_INIT(&unp_dhead); 3645 LIST_INIT(&unp_shead); 3646 LIST_INIT(&unp_sphead); 3647 SLIST_INIT(&unp_defers); 3648 TIMEOUT_TASK_INIT(taskqueue_thread, &unp_gc_task, 0, unp_gc, NULL); 3649 TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL); 3650 UNP_LINK_LOCK_INIT(); 3651 UNP_DEFERRED_LOCK_INIT(); 3652 unp_vp_mtxpool = mtx_pool_create("unp vp mtxpool", 32, MTX_DEF); 3653 } 3654 SYSINIT(unp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_SECOND, unp_init, NULL); 3655 3656 static void 3657 unp_internalize_cleanup_rights(struct mbuf *control) 3658 { 3659 struct cmsghdr *cp; 3660 struct mbuf *m; 3661 void *data; 3662 socklen_t datalen; 3663 3664 for (m = control; m != NULL; m = m->m_next) { 3665 cp = mtod(m, struct cmsghdr *); 3666 if (cp->cmsg_level != SOL_SOCKET || 3667 cp->cmsg_type != SCM_RIGHTS) 3668 continue; 3669 data = CMSG_DATA(cp); 3670 datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data; 3671 unp_freerights(data, datalen / sizeof(struct filedesc *)); 3672 } 3673 } 3674 3675 static int 3676 unp_internalize(struct mbuf *control, struct mchain *mc, struct thread *td) 3677 { 3678 struct proc *p; 3679 struct filedesc *fdesc; 3680 struct bintime *bt; 3681 struct cmsghdr *cm; 3682 struct cmsgcred *cmcred; 3683 struct mbuf *m; 3684 struct filedescent *fde, **fdep, *fdev; 3685 struct file *fp; 3686 struct timeval *tv; 3687 struct timespec *ts; 3688 void *data; 3689 socklen_t clen, datalen; 3690 int i, j, error, *fdp, oldfds; 3691 u_int newlen; 3692 3693 MPASS(control->m_next == NULL); /* COMPAT_OLDSOCK may violate */ 3694 UNP_LINK_UNLOCK_ASSERT(); 3695 3696 p = td->td_proc; 3697 fdesc = p->p_fd; 3698 error = 0; 3699 *mc = MCHAIN_INITIALIZER(mc); 3700 for (clen = control->m_len, cm = mtod(control, struct cmsghdr *), 3701 data = CMSG_DATA(cm); 3702 3703 clen >= sizeof(*cm) && cm->cmsg_level == SOL_SOCKET && 3704 clen >= cm->cmsg_len && cm->cmsg_len >= sizeof(*cm) && 3705 (char *)cm + cm->cmsg_len >= (char *)data; 3706 3707 clen -= min(CMSG_SPACE(datalen), clen), 3708 cm = (struct cmsghdr *) ((char *)cm + CMSG_SPACE(datalen)), 3709 data = CMSG_DATA(cm)) { 3710 datalen = (char *)cm + cm->cmsg_len - (char *)data; 3711 switch (cm->cmsg_type) { 3712 case SCM_CREDS: 3713 m = sbcreatecontrol(NULL, sizeof(*cmcred), SCM_CREDS, 3714 SOL_SOCKET, M_WAITOK); 3715 cmcred = (struct cmsgcred *) 3716 CMSG_DATA(mtod(m, struct cmsghdr *)); 3717 cmcred->cmcred_pid = p->p_pid; 3718 cmcred->cmcred_uid = td->td_ucred->cr_ruid; 3719 cmcred->cmcred_gid = td->td_ucred->cr_rgid; 3720 cmcred->cmcred_euid = td->td_ucred->cr_uid; 3721 _Static_assert(CMGROUP_MAX >= 1, 3722 "Room needed for the effective GID."); 3723 cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups + 1, 3724 CMGROUP_MAX); 3725 cmcred->cmcred_groups[0] = td->td_ucred->cr_gid; 3726 for (i = 1; i < cmcred->cmcred_ngroups; i++) 3727 cmcred->cmcred_groups[i] = 3728 td->td_ucred->cr_groups[i - 1]; 3729 break; 3730 3731 case SCM_RIGHTS: 3732 oldfds = datalen / sizeof (int); 3733 if (oldfds == 0) 3734 continue; 3735 /* On some machines sizeof pointer is bigger than 3736 * sizeof int, so we need to check if data fits into 3737 * single mbuf. We could allocate several mbufs, and 3738 * unp_externalize() should even properly handle that. 3739 * But it is not worth to complicate the code for an 3740 * insane scenario of passing over 200 file descriptors 3741 * at once. 3742 */ 3743 newlen = oldfds * sizeof(fdep[0]); 3744 if (CMSG_SPACE(newlen) > MCLBYTES) { 3745 error = EMSGSIZE; 3746 goto out; 3747 } 3748 /* 3749 * Check that all the FDs passed in refer to legal 3750 * files. If not, reject the entire operation. 3751 */ 3752 fdp = data; 3753 FILEDESC_SLOCK(fdesc); 3754 for (i = 0; i < oldfds; i++, fdp++) { 3755 fp = fget_noref(fdesc, *fdp); 3756 if (fp == NULL) { 3757 FILEDESC_SUNLOCK(fdesc); 3758 error = EBADF; 3759 goto out; 3760 } 3761 if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) { 3762 FILEDESC_SUNLOCK(fdesc); 3763 error = EOPNOTSUPP; 3764 goto out; 3765 } 3766 } 3767 3768 /* 3769 * Now replace the integer FDs with pointers to the 3770 * file structure and capability rights. 3771 */ 3772 m = sbcreatecontrol(NULL, newlen, SCM_RIGHTS, 3773 SOL_SOCKET, M_WAITOK); 3774 fdp = data; 3775 for (i = 0; i < oldfds; i++, fdp++) { 3776 if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) { 3777 fdp = data; 3778 for (j = 0; j < i; j++, fdp++) { 3779 fdrop(fdesc->fd_ofiles[*fdp]. 3780 fde_file, td); 3781 } 3782 FILEDESC_SUNLOCK(fdesc); 3783 error = EBADF; 3784 goto out; 3785 } 3786 } 3787 fdp = data; 3788 fdep = (struct filedescent **) 3789 CMSG_DATA(mtod(m, struct cmsghdr *)); 3790 fdev = malloc(sizeof(*fdev) * oldfds, M_FILECAPS, 3791 M_WAITOK); 3792 for (i = 0; i < oldfds; i++, fdev++, fdp++) { 3793 fde = &fdesc->fd_ofiles[*fdp]; 3794 fdep[i] = fdev; 3795 fdep[i]->fde_file = fde->fde_file; 3796 filecaps_copy(&fde->fde_caps, 3797 &fdep[i]->fde_caps, true); 3798 unp_internalize_fp(fdep[i]->fde_file); 3799 } 3800 FILEDESC_SUNLOCK(fdesc); 3801 break; 3802 3803 case SCM_TIMESTAMP: 3804 m = sbcreatecontrol(NULL, sizeof(*tv), SCM_TIMESTAMP, 3805 SOL_SOCKET, M_WAITOK); 3806 tv = (struct timeval *) 3807 CMSG_DATA(mtod(m, struct cmsghdr *)); 3808 microtime(tv); 3809 break; 3810 3811 case SCM_BINTIME: 3812 m = sbcreatecontrol(NULL, sizeof(*bt), SCM_BINTIME, 3813 SOL_SOCKET, M_WAITOK); 3814 bt = (struct bintime *) 3815 CMSG_DATA(mtod(m, struct cmsghdr *)); 3816 bintime(bt); 3817 break; 3818 3819 case SCM_REALTIME: 3820 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_REALTIME, 3821 SOL_SOCKET, M_WAITOK); 3822 ts = (struct timespec *) 3823 CMSG_DATA(mtod(m, struct cmsghdr *)); 3824 nanotime(ts); 3825 break; 3826 3827 case SCM_MONOTONIC: 3828 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_MONOTONIC, 3829 SOL_SOCKET, M_WAITOK); 3830 ts = (struct timespec *) 3831 CMSG_DATA(mtod(m, struct cmsghdr *)); 3832 nanouptime(ts); 3833 break; 3834 3835 default: 3836 error = EINVAL; 3837 goto out; 3838 } 3839 3840 mc_append(mc, m); 3841 } 3842 if (clen > 0) 3843 error = EINVAL; 3844 3845 out: 3846 if (error != 0) 3847 unp_internalize_cleanup_rights(mc_first(mc)); 3848 m_freem(control); 3849 return (error); 3850 } 3851 3852 static void 3853 unp_addsockcred(struct thread *td, struct mchain *mc, int mode) 3854 { 3855 struct mbuf *m, *n, *n_prev; 3856 const struct cmsghdr *cm; 3857 int ngroups, i, cmsgtype; 3858 size_t ctrlsz; 3859 3860 ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX); 3861 if (mode & UNP_WANTCRED_ALWAYS) { 3862 ctrlsz = SOCKCRED2SIZE(ngroups); 3863 cmsgtype = SCM_CREDS2; 3864 } else { 3865 ctrlsz = SOCKCREDSIZE(ngroups); 3866 cmsgtype = SCM_CREDS; 3867 } 3868 3869 /* XXXGL: uipc_sosend_*() need to be improved so that we can M_WAITOK */ 3870 m = sbcreatecontrol(NULL, ctrlsz, cmsgtype, SOL_SOCKET, M_NOWAIT); 3871 if (m == NULL) 3872 return; 3873 MPASS((m->m_flags & M_EXT) == 0 && m->m_next == NULL); 3874 3875 if (mode & UNP_WANTCRED_ALWAYS) { 3876 struct sockcred2 *sc; 3877 3878 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3879 sc->sc_version = 0; 3880 sc->sc_pid = td->td_proc->p_pid; 3881 sc->sc_uid = td->td_ucred->cr_ruid; 3882 sc->sc_euid = td->td_ucred->cr_uid; 3883 sc->sc_gid = td->td_ucred->cr_rgid; 3884 sc->sc_egid = td->td_ucred->cr_gid; 3885 sc->sc_ngroups = ngroups; 3886 for (i = 0; i < sc->sc_ngroups; i++) 3887 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3888 } else { 3889 struct sockcred *sc; 3890 3891 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3892 sc->sc_uid = td->td_ucred->cr_ruid; 3893 sc->sc_euid = td->td_ucred->cr_uid; 3894 sc->sc_gid = td->td_ucred->cr_rgid; 3895 sc->sc_egid = td->td_ucred->cr_gid; 3896 sc->sc_ngroups = ngroups; 3897 for (i = 0; i < sc->sc_ngroups; i++) 3898 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3899 } 3900 3901 /* 3902 * Unlink SCM_CREDS control messages (struct cmsgcred), since just 3903 * created SCM_CREDS control message (struct sockcred) has another 3904 * format. 3905 */ 3906 if (!STAILQ_EMPTY(&mc->mc_q) && cmsgtype == SCM_CREDS) 3907 STAILQ_FOREACH_SAFE(n, &mc->mc_q, m_stailq, n_prev) { 3908 cm = mtod(n, struct cmsghdr *); 3909 if (cm->cmsg_level == SOL_SOCKET && 3910 cm->cmsg_type == SCM_CREDS) { 3911 mc_remove(mc, n); 3912 m_free(n); 3913 } 3914 } 3915 3916 /* Prepend it to the head. */ 3917 mc_prepend(mc, m); 3918 } 3919 3920 static struct unpcb * 3921 fptounp(struct file *fp) 3922 { 3923 struct socket *so; 3924 3925 if (fp->f_type != DTYPE_SOCKET) 3926 return (NULL); 3927 if ((so = fp->f_data) == NULL) 3928 return (NULL); 3929 if (so->so_proto->pr_domain != &localdomain) 3930 return (NULL); 3931 return sotounpcb(so); 3932 } 3933 3934 static void 3935 unp_discard(struct file *fp) 3936 { 3937 struct unp_defer *dr; 3938 3939 if (unp_externalize_fp(fp)) { 3940 dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK); 3941 dr->ud_fp = fp; 3942 UNP_DEFERRED_LOCK(); 3943 SLIST_INSERT_HEAD(&unp_defers, dr, ud_link); 3944 UNP_DEFERRED_UNLOCK(); 3945 atomic_add_int(&unp_defers_count, 1); 3946 taskqueue_enqueue(taskqueue_thread, &unp_defer_task); 3947 } else 3948 closef_nothread(fp); 3949 } 3950 3951 static void 3952 unp_process_defers(void *arg __unused, int pending) 3953 { 3954 struct unp_defer *dr; 3955 SLIST_HEAD(, unp_defer) drl; 3956 int count; 3957 3958 SLIST_INIT(&drl); 3959 for (;;) { 3960 UNP_DEFERRED_LOCK(); 3961 if (SLIST_FIRST(&unp_defers) == NULL) { 3962 UNP_DEFERRED_UNLOCK(); 3963 break; 3964 } 3965 SLIST_SWAP(&unp_defers, &drl, unp_defer); 3966 UNP_DEFERRED_UNLOCK(); 3967 count = 0; 3968 while ((dr = SLIST_FIRST(&drl)) != NULL) { 3969 SLIST_REMOVE_HEAD(&drl, ud_link); 3970 closef_nothread(dr->ud_fp); 3971 free(dr, M_TEMP); 3972 count++; 3973 } 3974 atomic_add_int(&unp_defers_count, -count); 3975 } 3976 } 3977 3978 static void 3979 unp_internalize_fp(struct file *fp) 3980 { 3981 struct unpcb *unp; 3982 3983 UNP_LINK_WLOCK(); 3984 if ((unp = fptounp(fp)) != NULL) { 3985 unp->unp_file = fp; 3986 unp->unp_msgcount++; 3987 } 3988 unp_rights++; 3989 UNP_LINK_WUNLOCK(); 3990 } 3991 3992 static int 3993 unp_externalize_fp(struct file *fp) 3994 { 3995 struct unpcb *unp; 3996 int ret; 3997 3998 UNP_LINK_WLOCK(); 3999 if ((unp = fptounp(fp)) != NULL) { 4000 unp->unp_msgcount--; 4001 ret = 1; 4002 } else 4003 ret = 0; 4004 unp_rights--; 4005 UNP_LINK_WUNLOCK(); 4006 return (ret); 4007 } 4008 4009 /* 4010 * unp_defer indicates whether additional work has been defered for a future 4011 * pass through unp_gc(). It is thread local and does not require explicit 4012 * synchronization. 4013 */ 4014 static int unp_marked; 4015 4016 static void 4017 unp_remove_dead_ref(struct filedescent **fdep, int fdcount) 4018 { 4019 struct unpcb *unp; 4020 struct file *fp; 4021 int i; 4022 4023 /* 4024 * This function can only be called from the gc task. 4025 */ 4026 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4027 ("%s: not on gc callout", __func__)); 4028 UNP_LINK_LOCK_ASSERT(); 4029 4030 for (i = 0; i < fdcount; i++) { 4031 fp = fdep[i]->fde_file; 4032 if ((unp = fptounp(fp)) == NULL) 4033 continue; 4034 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4035 continue; 4036 unp->unp_gcrefs--; 4037 } 4038 } 4039 4040 static void 4041 unp_restore_undead_ref(struct filedescent **fdep, int fdcount) 4042 { 4043 struct unpcb *unp; 4044 struct file *fp; 4045 int i; 4046 4047 /* 4048 * This function can only be called from the gc task. 4049 */ 4050 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4051 ("%s: not on gc callout", __func__)); 4052 UNP_LINK_LOCK_ASSERT(); 4053 4054 for (i = 0; i < fdcount; i++) { 4055 fp = fdep[i]->fde_file; 4056 if ((unp = fptounp(fp)) == NULL) 4057 continue; 4058 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4059 continue; 4060 unp->unp_gcrefs++; 4061 unp_marked++; 4062 } 4063 } 4064 4065 static void 4066 unp_scan_socket(struct socket *so, void (*op)(struct filedescent **, int)) 4067 { 4068 struct sockbuf *sb; 4069 4070 SOCK_LOCK_ASSERT(so); 4071 4072 if (sotounpcb(so)->unp_gcflag & UNPGC_IGNORE_RIGHTS) 4073 return; 4074 4075 SOCK_RECVBUF_LOCK(so); 4076 switch (so->so_type) { 4077 case SOCK_DGRAM: 4078 unp_scan(STAILQ_FIRST(&so->so_rcv.uxdg_mb), op); 4079 unp_scan(so->so_rcv.uxdg_peeked, op); 4080 TAILQ_FOREACH(sb, &so->so_rcv.uxdg_conns, uxdg_clist) 4081 unp_scan(STAILQ_FIRST(&sb->uxdg_mb), op); 4082 break; 4083 case SOCK_STREAM: 4084 case SOCK_SEQPACKET: 4085 unp_scan(STAILQ_FIRST(&so->so_rcv.uxst_mbq), op); 4086 break; 4087 } 4088 SOCK_RECVBUF_UNLOCK(so); 4089 } 4090 4091 static void 4092 unp_gc_scan(struct unpcb *unp, void (*op)(struct filedescent **, int)) 4093 { 4094 struct socket *so, *soa; 4095 4096 so = unp->unp_socket; 4097 SOCK_LOCK(so); 4098 if (SOLISTENING(so)) { 4099 /* 4100 * Mark all sockets in our accept queue. 4101 */ 4102 TAILQ_FOREACH(soa, &so->sol_comp, so_list) 4103 unp_scan_socket(soa, op); 4104 } else { 4105 /* 4106 * Mark all sockets we reference with RIGHTS. 4107 */ 4108 unp_scan_socket(so, op); 4109 } 4110 SOCK_UNLOCK(so); 4111 } 4112 4113 static int unp_recycled; 4114 SYSCTL_INT(_net_local, OID_AUTO, recycled, CTLFLAG_RD, &unp_recycled, 0, 4115 "Number of unreachable sockets claimed by the garbage collector."); 4116 4117 static int unp_taskcount; 4118 SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0, 4119 "Number of times the garbage collector has run."); 4120 4121 SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0, 4122 "Number of active local sockets."); 4123 4124 static void 4125 unp_gc(__unused void *arg, int pending) 4126 { 4127 struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead, 4128 NULL }; 4129 struct unp_head **head; 4130 struct unp_head unp_deadhead; /* List of potentially-dead sockets. */ 4131 struct file *f, **unref; 4132 struct unpcb *unp, *unptmp; 4133 int i, total, unp_unreachable; 4134 4135 LIST_INIT(&unp_deadhead); 4136 unp_taskcount++; 4137 UNP_LINK_RLOCK(); 4138 /* 4139 * First determine which sockets may be in cycles. 4140 */ 4141 unp_unreachable = 0; 4142 4143 for (head = heads; *head != NULL; head++) 4144 LIST_FOREACH(unp, *head, unp_link) { 4145 KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0, 4146 ("%s: unp %p has unexpected gc flags 0x%x", 4147 __func__, unp, (unsigned int)unp->unp_gcflag)); 4148 4149 f = unp->unp_file; 4150 4151 /* 4152 * Check for an unreachable socket potentially in a 4153 * cycle. It must be in a queue as indicated by 4154 * msgcount, and this must equal the file reference 4155 * count. Note that when msgcount is 0 the file is 4156 * NULL. 4157 */ 4158 if (f != NULL && unp->unp_msgcount != 0 && 4159 refcount_load(&f->f_count) == unp->unp_msgcount) { 4160 LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead); 4161 unp->unp_gcflag |= UNPGC_DEAD; 4162 unp->unp_gcrefs = unp->unp_msgcount; 4163 unp_unreachable++; 4164 } 4165 } 4166 4167 /* 4168 * Scan all sockets previously marked as potentially being in a cycle 4169 * and remove the references each socket holds on any UNPGC_DEAD 4170 * sockets in its queue. After this step, all remaining references on 4171 * sockets marked UNPGC_DEAD should not be part of any cycle. 4172 */ 4173 LIST_FOREACH(unp, &unp_deadhead, unp_dead) 4174 unp_gc_scan(unp, unp_remove_dead_ref); 4175 4176 /* 4177 * If a socket still has a non-negative refcount, it cannot be in a 4178 * cycle. In this case increment refcount of all children iteratively. 4179 * Stop the scan once we do a complete loop without discovering 4180 * a new reachable socket. 4181 */ 4182 do { 4183 unp_marked = 0; 4184 LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp) 4185 if (unp->unp_gcrefs > 0) { 4186 unp->unp_gcflag &= ~UNPGC_DEAD; 4187 LIST_REMOVE(unp, unp_dead); 4188 KASSERT(unp_unreachable > 0, 4189 ("%s: unp_unreachable underflow.", 4190 __func__)); 4191 unp_unreachable--; 4192 unp_gc_scan(unp, unp_restore_undead_ref); 4193 } 4194 } while (unp_marked); 4195 4196 UNP_LINK_RUNLOCK(); 4197 4198 if (unp_unreachable == 0) 4199 return; 4200 4201 /* 4202 * Allocate space for a local array of dead unpcbs. 4203 * TODO: can this path be simplified by instead using the local 4204 * dead list at unp_deadhead, after taking out references 4205 * on the file object and/or unpcb and dropping the link lock? 4206 */ 4207 unref = malloc(unp_unreachable * sizeof(struct file *), 4208 M_TEMP, M_WAITOK); 4209 4210 /* 4211 * Iterate looking for sockets which have been specifically marked 4212 * as unreachable and store them locally. 4213 */ 4214 UNP_LINK_RLOCK(); 4215 total = 0; 4216 LIST_FOREACH(unp, &unp_deadhead, unp_dead) { 4217 KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0, 4218 ("%s: unp %p not marked UNPGC_DEAD", __func__, unp)); 4219 unp->unp_gcflag &= ~UNPGC_DEAD; 4220 f = unp->unp_file; 4221 if (unp->unp_msgcount == 0 || f == NULL || 4222 refcount_load(&f->f_count) != unp->unp_msgcount || 4223 !fhold(f)) 4224 continue; 4225 unref[total++] = f; 4226 KASSERT(total <= unp_unreachable, 4227 ("%s: incorrect unreachable count.", __func__)); 4228 } 4229 UNP_LINK_RUNLOCK(); 4230 4231 /* 4232 * Now flush all sockets, free'ing rights. This will free the 4233 * struct files associated with these sockets but leave each socket 4234 * with one remaining ref. 4235 */ 4236 for (i = 0; i < total; i++) { 4237 struct socket *so; 4238 4239 so = unref[i]->f_data; 4240 if (!SOLISTENING(so)) { 4241 CURVNET_SET(so->so_vnet); 4242 socantrcvmore(so); 4243 unp_dispose(so); 4244 CURVNET_RESTORE(); 4245 } 4246 } 4247 4248 /* 4249 * And finally release the sockets so they can be reclaimed. 4250 */ 4251 for (i = 0; i < total; i++) 4252 fdrop(unref[i], NULL); 4253 unp_recycled += total; 4254 free(unref, M_TEMP); 4255 } 4256 4257 /* 4258 * Synchronize against unp_gc, which can trip over data as we are freeing it. 4259 */ 4260 static void 4261 unp_dispose(struct socket *so) 4262 { 4263 struct sockbuf *sb; 4264 struct unpcb *unp; 4265 struct mbuf *m; 4266 int error __diagused; 4267 4268 MPASS(!SOLISTENING(so)); 4269 4270 unp = sotounpcb(so); 4271 UNP_LINK_WLOCK(); 4272 unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS; 4273 UNP_LINK_WUNLOCK(); 4274 4275 /* 4276 * Grab our special mbufs before calling sbrelease(). 4277 */ 4278 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 4279 MPASS(!error); 4280 SOCK_RECVBUF_LOCK(so); 4281 switch (so->so_type) { 4282 case SOCK_DGRAM: 4283 while ((sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) != NULL) { 4284 STAILQ_CONCAT(&so->so_rcv.uxdg_mb, &sb->uxdg_mb); 4285 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 4286 /* Note: socket of sb may reconnect. */ 4287 sb->uxdg_cc = sb->uxdg_ctl = sb->uxdg_mbcnt = 0; 4288 } 4289 sb = &so->so_rcv; 4290 if (sb->uxdg_peeked != NULL) { 4291 STAILQ_INSERT_HEAD(&sb->uxdg_mb, sb->uxdg_peeked, 4292 m_stailqpkt); 4293 sb->uxdg_peeked = NULL; 4294 } 4295 m = STAILQ_FIRST(&sb->uxdg_mb); 4296 STAILQ_INIT(&sb->uxdg_mb); 4297 break; 4298 case SOCK_STREAM: 4299 case SOCK_SEQPACKET: 4300 sb = &so->so_rcv; 4301 m = STAILQ_FIRST(&sb->uxst_mbq); 4302 STAILQ_INIT(&sb->uxst_mbq); 4303 sb->sb_acc = sb->sb_ccc = sb->sb_ctl = sb->sb_mbcnt = 0; 4304 /* 4305 * Trim M_NOTREADY buffers from the free list. They are 4306 * referenced by the I/O thread. 4307 */ 4308 if (sb->uxst_fnrdy != NULL) { 4309 struct mbuf *n, *prev; 4310 4311 while (m != NULL && m->m_flags & M_NOTREADY) 4312 m = m->m_next; 4313 for (prev = n = m; n != NULL; n = n->m_next) { 4314 if (n->m_flags & M_NOTREADY) 4315 prev->m_next = n->m_next; 4316 else 4317 prev = n; 4318 } 4319 sb->uxst_fnrdy = NULL; 4320 } 4321 break; 4322 } 4323 /* 4324 * Mark sb with SBS_CANTRCVMORE. This is needed to prevent 4325 * uipc_sosend_*() or unp_disconnect() adding more data to the socket. 4326 * We came here either through shutdown(2) or from the final sofree(). 4327 * The sofree() case is simple as it guarantees that no more sends will 4328 * happen, however we can race with unp_disconnect() from our peer. 4329 * The shutdown(2) case is more exotic. It would call into 4330 * unp_dispose() only if socket is SS_ISCONNECTED. This is possible if 4331 * we did connect(2) on this socket and we also had it bound with 4332 * bind(2) and receive connections from other sockets. Because 4333 * uipc_shutdown() violates POSIX (see comment there) this applies to 4334 * SOCK_DGRAM as well. For SOCK_DGRAM this SBS_CANTRCVMORE will have 4335 * affect not only on the peer we connect(2)ed to, but also on all of 4336 * the peers who had connect(2)ed to us. Their sends would end up 4337 * with ENOBUFS. 4338 */ 4339 sb->sb_state |= SBS_CANTRCVMORE; 4340 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 4341 RLIM_INFINITY); 4342 SOCK_RECVBUF_UNLOCK(so); 4343 SOCK_IO_RECV_UNLOCK(so); 4344 4345 if (m != NULL) { 4346 unp_scan(m, unp_freerights); 4347 m_freemp(m); 4348 } 4349 } 4350 4351 static void 4352 unp_scan(struct mbuf *m0, void (*op)(struct filedescent **, int)) 4353 { 4354 struct mbuf *m; 4355 struct cmsghdr *cm; 4356 void *data; 4357 socklen_t clen, datalen; 4358 4359 while (m0 != NULL) { 4360 for (m = m0; m; m = m->m_next) { 4361 if (m->m_type != MT_CONTROL) 4362 continue; 4363 4364 cm = mtod(m, struct cmsghdr *); 4365 clen = m->m_len; 4366 4367 while (cm != NULL) { 4368 if (sizeof(*cm) > clen || cm->cmsg_len > clen) 4369 break; 4370 4371 data = CMSG_DATA(cm); 4372 datalen = (caddr_t)cm + cm->cmsg_len 4373 - (caddr_t)data; 4374 4375 if (cm->cmsg_level == SOL_SOCKET && 4376 cm->cmsg_type == SCM_RIGHTS) { 4377 (*op)(data, datalen / 4378 sizeof(struct filedescent *)); 4379 } 4380 4381 if (CMSG_SPACE(datalen) < clen) { 4382 clen -= CMSG_SPACE(datalen); 4383 cm = (struct cmsghdr *) 4384 ((caddr_t)cm + CMSG_SPACE(datalen)); 4385 } else { 4386 clen = 0; 4387 cm = NULL; 4388 } 4389 } 4390 } 4391 m0 = m0->m_nextpkt; 4392 } 4393 } 4394 4395 /* 4396 * Definitions of protocols supported in the LOCAL domain. 4397 */ 4398 static struct protosw streamproto = { 4399 .pr_type = SOCK_STREAM, 4400 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4401 .pr_ctloutput = &uipc_ctloutput, 4402 .pr_abort = uipc_abort, 4403 .pr_accept = uipc_peeraddr, 4404 .pr_attach = uipc_attach, 4405 .pr_bind = uipc_bind, 4406 .pr_bindat = uipc_bindat, 4407 .pr_connect = uipc_connect, 4408 .pr_connectat = uipc_connectat, 4409 .pr_connect2 = uipc_connect2, 4410 .pr_detach = uipc_detach, 4411 .pr_disconnect = uipc_disconnect, 4412 .pr_fdclose = uipc_fdclose, 4413 .pr_listen = uipc_listen, 4414 .pr_peeraddr = uipc_peeraddr, 4415 .pr_send = uipc_sendfile, 4416 .pr_sendfile_wait = uipc_sendfile_wait, 4417 .pr_ready = uipc_ready, 4418 .pr_sense = uipc_sense, 4419 .pr_shutdown = uipc_shutdown, 4420 .pr_sockaddr = uipc_sockaddr, 4421 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4422 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4423 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4424 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4425 .pr_close = uipc_close, 4426 .pr_chmod = uipc_chmod, 4427 }; 4428 4429 static struct protosw dgramproto = { 4430 .pr_type = SOCK_DGRAM, 4431 .pr_flags = PR_ATOMIC | PR_ADDR | PR_CAPATTACH | PR_SOCKBUF, 4432 .pr_ctloutput = &uipc_ctloutput, 4433 .pr_abort = uipc_abort, 4434 .pr_accept = uipc_peeraddr, 4435 .pr_attach = uipc_attach, 4436 .pr_bind = uipc_bind, 4437 .pr_bindat = uipc_bindat, 4438 .pr_connect = uipc_connect, 4439 .pr_connectat = uipc_connectat, 4440 .pr_connect2 = uipc_connect2, 4441 .pr_detach = uipc_detach, 4442 .pr_disconnect = uipc_disconnect, 4443 .pr_fdclose = uipc_fdclose, 4444 .pr_peeraddr = uipc_peeraddr, 4445 .pr_sosend = uipc_sosend_dgram, 4446 .pr_sense = uipc_sense, 4447 .pr_shutdown = uipc_shutdown, 4448 .pr_sockaddr = uipc_sockaddr, 4449 .pr_soreceive = uipc_soreceive_dgram, 4450 .pr_close = uipc_close, 4451 .pr_chmod = uipc_chmod, 4452 }; 4453 4454 static struct protosw seqpacketproto = { 4455 .pr_type = SOCK_SEQPACKET, 4456 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4457 .pr_ctloutput = &uipc_ctloutput, 4458 .pr_abort = uipc_abort, 4459 .pr_accept = uipc_peeraddr, 4460 .pr_attach = uipc_attach, 4461 .pr_bind = uipc_bind, 4462 .pr_bindat = uipc_bindat, 4463 .pr_connect = uipc_connect, 4464 .pr_connectat = uipc_connectat, 4465 .pr_connect2 = uipc_connect2, 4466 .pr_detach = uipc_detach, 4467 .pr_disconnect = uipc_disconnect, 4468 .pr_fdclose = uipc_fdclose, 4469 .pr_listen = uipc_listen, 4470 .pr_peeraddr = uipc_peeraddr, 4471 .pr_sense = uipc_sense, 4472 .pr_shutdown = uipc_shutdown, 4473 .pr_sockaddr = uipc_sockaddr, 4474 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4475 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4476 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4477 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4478 .pr_close = uipc_close, 4479 .pr_chmod = uipc_chmod, 4480 }; 4481 4482 static struct domain localdomain = { 4483 .dom_family = AF_LOCAL, 4484 .dom_name = "local", 4485 .dom_nprotosw = 3, 4486 .dom_protosw = { 4487 &streamproto, 4488 &dgramproto, 4489 &seqpacketproto, 4490 } 4491 }; 4492 DOMAIN_SET(local); 4493 4494 /* 4495 * A helper function called by VFS before socket-type vnode reclamation. 4496 * For an active vnode it clears unp_vnode pointer and decrements unp_vnode 4497 * use count. 4498 */ 4499 void 4500 vfs_unp_reclaim(struct vnode *vp) 4501 { 4502 struct unpcb *unp; 4503 int active; 4504 struct mtx *vplock; 4505 4506 ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim"); 4507 KASSERT(vp->v_type == VSOCK, 4508 ("vfs_unp_reclaim: vp->v_type != VSOCK")); 4509 4510 active = 0; 4511 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 4512 mtx_lock(vplock); 4513 VOP_UNP_CONNECT(vp, &unp); 4514 if (unp == NULL) 4515 goto done; 4516 UNP_PCB_LOCK(unp); 4517 if (unp->unp_vnode == vp) { 4518 VOP_UNP_DETACH(vp); 4519 unp->unp_vnode = NULL; 4520 active = 1; 4521 } 4522 UNP_PCB_UNLOCK(unp); 4523 done: 4524 mtx_unlock(vplock); 4525 if (active) 4526 vunref(vp); 4527 } 4528 4529 #ifdef DDB 4530 static void 4531 db_print_indent(int indent) 4532 { 4533 int i; 4534 4535 for (i = 0; i < indent; i++) 4536 db_printf(" "); 4537 } 4538 4539 static void 4540 db_print_unpflags(int unp_flags) 4541 { 4542 int comma; 4543 4544 comma = 0; 4545 if (unp_flags & UNP_HAVEPC) { 4546 db_printf("%sUNP_HAVEPC", comma ? ", " : ""); 4547 comma = 1; 4548 } 4549 if (unp_flags & UNP_WANTCRED_ALWAYS) { 4550 db_printf("%sUNP_WANTCRED_ALWAYS", comma ? ", " : ""); 4551 comma = 1; 4552 } 4553 if (unp_flags & UNP_WANTCRED_ONESHOT) { 4554 db_printf("%sUNP_WANTCRED_ONESHOT", comma ? ", " : ""); 4555 comma = 1; 4556 } 4557 if (unp_flags & UNP_CONNECTING) { 4558 db_printf("%sUNP_CONNECTING", comma ? ", " : ""); 4559 comma = 1; 4560 } 4561 if (unp_flags & UNP_BINDING) { 4562 db_printf("%sUNP_BINDING", comma ? ", " : ""); 4563 comma = 1; 4564 } 4565 } 4566 4567 static void 4568 db_print_xucred(int indent, struct xucred *xu) 4569 { 4570 int comma, i; 4571 4572 db_print_indent(indent); 4573 db_printf("cr_version: %u cr_uid: %u cr_pid: %d cr_ngroups: %d\n", 4574 xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups); 4575 db_print_indent(indent); 4576 db_printf("cr_groups: "); 4577 comma = 0; 4578 for (i = 0; i < xu->cr_ngroups; i++) { 4579 db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]); 4580 comma = 1; 4581 } 4582 db_printf("\n"); 4583 } 4584 4585 static void 4586 db_print_unprefs(int indent, struct unp_head *uh) 4587 { 4588 struct unpcb *unp; 4589 int counter; 4590 4591 counter = 0; 4592 LIST_FOREACH(unp, uh, unp_reflink) { 4593 if (counter % 4 == 0) 4594 db_print_indent(indent); 4595 db_printf("%p ", unp); 4596 if (counter % 4 == 3) 4597 db_printf("\n"); 4598 counter++; 4599 } 4600 if (counter != 0 && counter % 4 != 0) 4601 db_printf("\n"); 4602 } 4603 4604 DB_SHOW_COMMAND(unpcb, db_show_unpcb) 4605 { 4606 struct unpcb *unp; 4607 4608 if (!have_addr) { 4609 db_printf("usage: show unpcb <addr>\n"); 4610 return; 4611 } 4612 unp = (struct unpcb *)addr; 4613 4614 db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket, 4615 unp->unp_vnode); 4616 4617 db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino, 4618 unp->unp_conn); 4619 4620 db_printf("unp_refs:\n"); 4621 db_print_unprefs(2, &unp->unp_refs); 4622 4623 /* XXXRW: Would be nice to print the full address, if any. */ 4624 db_printf("unp_addr: %p\n", unp->unp_addr); 4625 4626 db_printf("unp_gencnt: %llu\n", 4627 (unsigned long long)unp->unp_gencnt); 4628 4629 db_printf("unp_flags: %x (", unp->unp_flags); 4630 db_print_unpflags(unp->unp_flags); 4631 db_printf(")\n"); 4632 4633 db_printf("unp_peercred:\n"); 4634 db_print_xucred(2, &unp->unp_peercred); 4635 4636 db_printf("unp_refcount: %u\n", unp->unp_refcount); 4637 } 4638 #endif 4639