1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1989, 1991, 1993 5 * The Regents of the University of California. All Rights Reserved. 6 * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved. 7 * Copyright (c) 2018 Matthew Macy 8 * Copyright (c) 2022-2025 Gleb Smirnoff <glebius@FreeBSD.org> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * UNIX Domain (Local) Sockets 37 * 38 * This is an implementation of UNIX (local) domain sockets. Each socket has 39 * an associated struct unpcb (UNIX protocol control block). Stream sockets 40 * may be connected to 0 or 1 other socket. Datagram sockets may be 41 * connected to 0, 1, or many other sockets. Sockets may be created and 42 * connected in pairs (socketpair(2)), or bound/connected to using the file 43 * system name space. For most purposes, only the receive socket buffer is 44 * used, as sending on one socket delivers directly to the receive socket 45 * buffer of a second socket. 46 * 47 * The implementation is substantially complicated by the fact that 48 * "ancillary data", such as file descriptors or credentials, may be passed 49 * across UNIX domain sockets. The potential for passing UNIX domain sockets 50 * over other UNIX domain sockets requires the implementation of a simple 51 * garbage collector to find and tear down cycles of disconnected sockets. 52 * 53 * TODO: 54 * RDM 55 * rethink name space problems 56 * need a proper out-of-band 57 */ 58 59 #include "opt_ddb.h" 60 61 #include <sys/param.h> 62 #include <sys/capsicum.h> 63 #include <sys/domain.h> 64 #include <sys/eventhandler.h> 65 #include <sys/fcntl.h> 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/jail.h> 69 #include <sys/kernel.h> 70 #include <sys/lock.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/mount.h> 74 #include <sys/mutex.h> 75 #include <sys/namei.h> 76 #include <sys/poll.h> 77 #include <sys/proc.h> 78 #include <sys/protosw.h> 79 #include <sys/queue.h> 80 #include <sys/resourcevar.h> 81 #include <sys/rwlock.h> 82 #include <sys/socket.h> 83 #include <sys/socketvar.h> 84 #include <sys/signalvar.h> 85 #include <sys/stat.h> 86 #include <sys/sx.h> 87 #include <sys/sysctl.h> 88 #include <sys/systm.h> 89 #include <sys/taskqueue.h> 90 #include <sys/un.h> 91 #include <sys/unpcb.h> 92 #include <sys/vnode.h> 93 94 #include <net/vnet.h> 95 96 #ifdef DDB 97 #include <ddb/ddb.h> 98 #endif 99 100 #include <security/mac/mac_framework.h> 101 102 #include <vm/uma.h> 103 104 MALLOC_DECLARE(M_FILECAPS); 105 106 static struct domain localdomain; 107 108 static uma_zone_t unp_zone; 109 static unp_gen_t unp_gencnt; /* (l) */ 110 static u_int unp_count; /* (l) Count of local sockets. */ 111 static ino_t unp_ino; /* Prototype for fake inode numbers. */ 112 static int unp_rights; /* (g) File descriptors in flight. */ 113 static struct unp_head unp_shead; /* (l) List of stream sockets. */ 114 static struct unp_head unp_dhead; /* (l) List of datagram sockets. */ 115 static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */ 116 static struct mtx_pool *unp_vp_mtxpool; 117 118 struct unp_defer { 119 SLIST_ENTRY(unp_defer) ud_link; 120 struct file *ud_fp; 121 }; 122 static SLIST_HEAD(, unp_defer) unp_defers; 123 static int unp_defers_count; 124 125 static const struct sockaddr sun_noname = { 126 .sa_len = sizeof(sun_noname), 127 .sa_family = AF_LOCAL, 128 }; 129 130 /* 131 * Garbage collection of cyclic file descriptor/socket references occurs 132 * asynchronously in a taskqueue context in order to avoid recursion and 133 * reentrance in the UNIX domain socket, file descriptor, and socket layer 134 * code. See unp_gc() for a full description. 135 */ 136 static struct timeout_task unp_gc_task; 137 138 /* 139 * The close of unix domain sockets attached as SCM_RIGHTS is 140 * postponed to the taskqueue, to avoid arbitrary recursion depth. 141 * The attached sockets might have another sockets attached. 142 */ 143 static struct task unp_defer_task; 144 145 /* 146 * SOCK_STREAM and SOCK_SEQPACKET unix(4) sockets fully bypass the send buffer, 147 * however the notion of send buffer still makes sense with them. Its size is 148 * the amount of space that a send(2) syscall may copyin(9) before checking 149 * with the receive buffer of a peer. Although not linked anywhere yet, 150 * pointed to by a stack variable, effectively it is a buffer that needs to be 151 * sized. 152 * 153 * SOCK_DGRAM sockets really use the sendspace as the maximum datagram size, 154 * and don't really want to reserve the sendspace. Their recvspace should be 155 * large enough for at least one max-size datagram plus address. 156 */ 157 static u_long unpst_sendspace = 64*1024; 158 static u_long unpst_recvspace = 64*1024; 159 static u_long unpdg_maxdgram = 8*1024; /* support 8KB syslog msgs */ 160 static u_long unpdg_recvspace = 16*1024; 161 static u_long unpsp_sendspace = 64*1024; 162 static u_long unpsp_recvspace = 64*1024; 163 164 static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 165 "Local domain"); 166 static SYSCTL_NODE(_net_local, SOCK_STREAM, stream, 167 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 168 "SOCK_STREAM"); 169 static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, 170 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 171 "SOCK_DGRAM"); 172 static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket, 173 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 174 "SOCK_SEQPACKET"); 175 176 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW, 177 &unpst_sendspace, 0, "Default stream send space."); 178 SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW, 179 &unpst_recvspace, 0, "Default stream receive space."); 180 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW, 181 &unpdg_maxdgram, 0, "Maximum datagram size."); 182 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW, 183 &unpdg_recvspace, 0, "Default datagram receive space."); 184 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW, 185 &unpsp_sendspace, 0, "Default seqpacket send space."); 186 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW, 187 &unpsp_recvspace, 0, "Default seqpacket receive space."); 188 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, 189 "File descriptors in flight."); 190 SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, 191 &unp_defers_count, 0, 192 "File descriptors deferred to taskqueue for close."); 193 194 /* 195 * Locking and synchronization: 196 * 197 * Several types of locks exist in the local domain socket implementation: 198 * - a global linkage lock 199 * - a global connection list lock 200 * - the mtxpool lock 201 * - per-unpcb mutexes 202 * 203 * The linkage lock protects the global socket lists, the generation number 204 * counter and garbage collector state. 205 * 206 * The connection list lock protects the list of referring sockets in a datagram 207 * socket PCB. This lock is also overloaded to protect a global list of 208 * sockets whose buffers contain socket references in the form of SCM_RIGHTS 209 * messages. To avoid recursion, such references are released by a dedicated 210 * thread. 211 * 212 * The mtxpool lock protects the vnode from being modified while referenced. 213 * Lock ordering rules require that it be acquired before any PCB locks. 214 * 215 * The unpcb lock (unp_mtx) protects the most commonly referenced fields in the 216 * unpcb. This includes the unp_conn field, which either links two connected 217 * PCBs together (for connected socket types) or points at the destination 218 * socket (for connectionless socket types). The operations of creating or 219 * destroying a connection therefore involve locking multiple PCBs. To avoid 220 * lock order reversals, in some cases this involves dropping a PCB lock and 221 * using a reference counter to maintain liveness. 222 * 223 * UNIX domain sockets each have an unpcb hung off of their so_pcb pointer, 224 * allocated in pr_attach() and freed in pr_detach(). The validity of that 225 * pointer is an invariant, so no lock is required to dereference the so_pcb 226 * pointer if a valid socket reference is held by the caller. In practice, 227 * this is always true during operations performed on a socket. Each unpcb 228 * has a back-pointer to its socket, unp_socket, which will be stable under 229 * the same circumstances. 230 * 231 * This pointer may only be safely dereferenced as long as a valid reference 232 * to the unpcb is held. Typically, this reference will be from the socket, 233 * or from another unpcb when the referring unpcb's lock is held (in order 234 * that the reference not be invalidated during use). For example, to follow 235 * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee 236 * that detach is not run clearing unp_socket. 237 * 238 * Blocking with UNIX domain sockets is a tricky issue: unlike most network 239 * protocols, bind() is a non-atomic operation, and connect() requires 240 * potential sleeping in the protocol, due to potentially waiting on local or 241 * distributed file systems. We try to separate "lookup" operations, which 242 * may sleep, and the IPC operations themselves, which typically can occur 243 * with relative atomicity as locks can be held over the entire operation. 244 * 245 * Another tricky issue is simultaneous multi-threaded or multi-process 246 * access to a single UNIX domain socket. These are handled by the flags 247 * UNP_CONNECTING and UNP_BINDING, which prevent concurrent connecting or 248 * binding, both of which involve dropping UNIX domain socket locks in order 249 * to perform namei() and other file system operations. 250 */ 251 static struct rwlock unp_link_rwlock; 252 static struct mtx unp_defers_lock; 253 254 #define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \ 255 "unp_link_rwlock") 256 257 #define UNP_LINK_LOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 258 RA_LOCKED) 259 #define UNP_LINK_UNLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 260 RA_UNLOCKED) 261 262 #define UNP_LINK_RLOCK() rw_rlock(&unp_link_rwlock) 263 #define UNP_LINK_RUNLOCK() rw_runlock(&unp_link_rwlock) 264 #define UNP_LINK_WLOCK() rw_wlock(&unp_link_rwlock) 265 #define UNP_LINK_WUNLOCK() rw_wunlock(&unp_link_rwlock) 266 #define UNP_LINK_WLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 267 RA_WLOCKED) 268 #define UNP_LINK_WOWNED() rw_wowned(&unp_link_rwlock) 269 270 #define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \ 271 "unp_defer", NULL, MTX_DEF) 272 #define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock) 273 #define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock) 274 275 #define UNP_REF_LIST_LOCK() UNP_DEFERRED_LOCK(); 276 #define UNP_REF_LIST_UNLOCK() UNP_DEFERRED_UNLOCK(); 277 278 #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \ 279 "unp", "unp", \ 280 MTX_DUPOK|MTX_DEF) 281 #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx) 282 #define UNP_PCB_LOCKPTR(unp) (&(unp)->unp_mtx) 283 #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx) 284 #define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx) 285 #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx) 286 #define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx) 287 #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED) 288 #define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED) 289 290 static int uipc_connect2(struct socket *, struct socket *); 291 static int uipc_ctloutput(struct socket *, struct sockopt *); 292 static int unp_connect(struct socket *, struct sockaddr *, 293 struct thread *); 294 static int unp_connectat(int, struct socket *, struct sockaddr *, 295 struct thread *, bool); 296 static void unp_connect2(struct socket *, struct socket *, bool); 297 static void unp_disconnect(struct unpcb *unp, struct unpcb *unp2); 298 static void unp_dispose(struct socket *so); 299 static void unp_drop(struct unpcb *); 300 static void unp_gc(__unused void *, int); 301 static void unp_scan(struct mbuf *, void (*)(struct filedescent **, int)); 302 static void unp_discard(struct file *); 303 static void unp_freerights(struct filedescent **, int); 304 static int unp_internalize(struct mbuf *, struct mchain *, 305 struct thread *); 306 static void unp_internalize_fp(struct file *); 307 static int unp_externalize(struct mbuf *, struct mbuf **, int); 308 static int unp_externalize_fp(struct file *); 309 static void unp_addsockcred(struct thread *, struct mchain *, int); 310 static void unp_process_defers(void * __unused, int); 311 312 static void uipc_wrknl_lock(void *); 313 static void uipc_wrknl_unlock(void *); 314 static void uipc_wrknl_assert_lock(void *, int); 315 316 static void 317 unp_pcb_hold(struct unpcb *unp) 318 { 319 u_int old __unused; 320 321 old = refcount_acquire(&unp->unp_refcount); 322 KASSERT(old > 0, ("%s: unpcb %p has no references", __func__, unp)); 323 } 324 325 static __result_use_check bool 326 unp_pcb_rele(struct unpcb *unp) 327 { 328 bool ret; 329 330 UNP_PCB_LOCK_ASSERT(unp); 331 332 if ((ret = refcount_release(&unp->unp_refcount))) { 333 UNP_PCB_UNLOCK(unp); 334 UNP_PCB_LOCK_DESTROY(unp); 335 uma_zfree(unp_zone, unp); 336 } 337 return (ret); 338 } 339 340 static void 341 unp_pcb_rele_notlast(struct unpcb *unp) 342 { 343 bool ret __unused; 344 345 ret = refcount_release(&unp->unp_refcount); 346 KASSERT(!ret, ("%s: unpcb %p has no references", __func__, unp)); 347 } 348 349 static void 350 unp_pcb_lock_pair(struct unpcb *unp, struct unpcb *unp2) 351 { 352 UNP_PCB_UNLOCK_ASSERT(unp); 353 UNP_PCB_UNLOCK_ASSERT(unp2); 354 355 if (unp == unp2) { 356 UNP_PCB_LOCK(unp); 357 } else if ((uintptr_t)unp2 > (uintptr_t)unp) { 358 UNP_PCB_LOCK(unp); 359 UNP_PCB_LOCK(unp2); 360 } else { 361 UNP_PCB_LOCK(unp2); 362 UNP_PCB_LOCK(unp); 363 } 364 } 365 366 static void 367 unp_pcb_unlock_pair(struct unpcb *unp, struct unpcb *unp2) 368 { 369 UNP_PCB_UNLOCK(unp); 370 if (unp != unp2) 371 UNP_PCB_UNLOCK(unp2); 372 } 373 374 /* 375 * Try to lock the connected peer of an already locked socket. In some cases 376 * this requires that we unlock the current socket. The pairbusy counter is 377 * used to block concurrent connection attempts while the lock is dropped. The 378 * caller must be careful to revalidate PCB state. 379 */ 380 static struct unpcb * 381 unp_pcb_lock_peer(struct unpcb *unp) 382 { 383 struct unpcb *unp2; 384 385 UNP_PCB_LOCK_ASSERT(unp); 386 unp2 = unp->unp_conn; 387 if (unp2 == NULL) 388 return (NULL); 389 if (__predict_false(unp == unp2)) 390 return (unp); 391 392 UNP_PCB_UNLOCK_ASSERT(unp2); 393 394 if (__predict_true(UNP_PCB_TRYLOCK(unp2))) 395 return (unp2); 396 if ((uintptr_t)unp2 > (uintptr_t)unp) { 397 UNP_PCB_LOCK(unp2); 398 return (unp2); 399 } 400 unp->unp_pairbusy++; 401 unp_pcb_hold(unp2); 402 UNP_PCB_UNLOCK(unp); 403 404 UNP_PCB_LOCK(unp2); 405 UNP_PCB_LOCK(unp); 406 KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL, 407 ("%s: socket %p was reconnected", __func__, unp)); 408 if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) { 409 unp->unp_flags &= ~UNP_WAITING; 410 wakeup(unp); 411 } 412 if (unp_pcb_rele(unp2)) { 413 /* unp2 is unlocked. */ 414 return (NULL); 415 } 416 if (unp->unp_conn == NULL) { 417 UNP_PCB_UNLOCK(unp2); 418 return (NULL); 419 } 420 return (unp2); 421 } 422 423 /* 424 * Try to lock peer of our socket for purposes of sending data to it. 425 */ 426 static int 427 uipc_lock_peer(struct socket *so, struct unpcb **unp2) 428 { 429 struct unpcb *unp; 430 int error; 431 432 unp = sotounpcb(so); 433 UNP_PCB_LOCK(unp); 434 *unp2 = unp_pcb_lock_peer(unp); 435 if (__predict_false(so->so_error != 0)) { 436 error = so->so_error; 437 so->so_error = 0; 438 UNP_PCB_UNLOCK(unp); 439 if (*unp2 != NULL) 440 UNP_PCB_UNLOCK(*unp2); 441 return (error); 442 } 443 if (__predict_false(*unp2 == NULL)) { 444 /* 445 * Different error code for a previously connected socket and 446 * a never connected one. The SS_ISDISCONNECTED is set in the 447 * unp_soisdisconnected() and is synchronized by the pcb lock. 448 */ 449 error = so->so_state & SS_ISDISCONNECTED ? EPIPE : ENOTCONN; 450 UNP_PCB_UNLOCK(unp); 451 return (error); 452 } 453 UNP_PCB_UNLOCK(unp); 454 455 return (0); 456 } 457 458 static void 459 uipc_abort(struct socket *so) 460 { 461 struct unpcb *unp, *unp2; 462 463 unp = sotounpcb(so); 464 KASSERT(unp != NULL, ("uipc_abort: unp == NULL")); 465 UNP_PCB_UNLOCK_ASSERT(unp); 466 467 UNP_PCB_LOCK(unp); 468 unp2 = unp->unp_conn; 469 if (unp2 != NULL) { 470 unp_pcb_hold(unp2); 471 UNP_PCB_UNLOCK(unp); 472 unp_drop(unp2); 473 } else 474 UNP_PCB_UNLOCK(unp); 475 } 476 477 static int 478 uipc_attach(struct socket *so, int proto, struct thread *td) 479 { 480 u_long sendspace, recvspace; 481 struct unpcb *unp; 482 int error, rcvmtxopts; 483 bool locked; 484 485 KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); 486 switch (so->so_type) { 487 case SOCK_DGRAM: 488 STAILQ_INIT(&so->so_rcv.uxdg_mb); 489 STAILQ_INIT(&so->so_snd.uxdg_mb); 490 TAILQ_INIT(&so->so_rcv.uxdg_conns); 491 /* 492 * Since send buffer is either bypassed or is a part 493 * of one-to-many receive buffer, we assign both space 494 * limits to unpdg_recvspace. 495 */ 496 sendspace = recvspace = unpdg_recvspace; 497 rcvmtxopts = 0; 498 break; 499 500 case SOCK_STREAM: 501 sendspace = unpst_sendspace; 502 recvspace = unpst_recvspace; 503 goto common; 504 505 case SOCK_SEQPACKET: 506 sendspace = unpsp_sendspace; 507 recvspace = unpsp_recvspace; 508 common: 509 rcvmtxopts = MTX_DUPOK; 510 knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, 511 uipc_wrknl_unlock, uipc_wrknl_assert_lock); 512 STAILQ_INIT(&so->so_rcv.uxst_mbq); 513 break; 514 default: 515 panic("uipc_attach"); 516 } 517 mtx_init(&so->so_rcv_mtx, "unix so_rcv", NULL, MTX_DEF | rcvmtxopts); 518 mtx_init(&so->so_snd_mtx, "unix so_snd", NULL, MTX_DEF); 519 error = soreserve(so, sendspace, recvspace); 520 if (error) 521 return (error); 522 unp = uma_zalloc(unp_zone, M_NOWAIT | M_ZERO); 523 if (unp == NULL) 524 return (ENOBUFS); 525 LIST_INIT(&unp->unp_refs); 526 UNP_PCB_LOCK_INIT(unp); 527 unp->unp_socket = so; 528 so->so_pcb = unp; 529 refcount_init(&unp->unp_refcount, 1); 530 unp->unp_mode = ACCESSPERMS; 531 532 if ((locked = UNP_LINK_WOWNED()) == false) 533 UNP_LINK_WLOCK(); 534 535 unp->unp_gencnt = ++unp_gencnt; 536 unp->unp_ino = ++unp_ino; 537 unp_count++; 538 switch (so->so_type) { 539 case SOCK_STREAM: 540 LIST_INSERT_HEAD(&unp_shead, unp, unp_link); 541 break; 542 543 case SOCK_DGRAM: 544 LIST_INSERT_HEAD(&unp_dhead, unp, unp_link); 545 break; 546 547 case SOCK_SEQPACKET: 548 LIST_INSERT_HEAD(&unp_sphead, unp, unp_link); 549 break; 550 551 default: 552 panic("uipc_attach"); 553 } 554 555 if (locked == false) 556 UNP_LINK_WUNLOCK(); 557 558 return (0); 559 } 560 561 static int 562 uipc_bindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 563 { 564 struct sockaddr_un *soun = (struct sockaddr_un *)nam; 565 struct vattr vattr; 566 int error, namelen; 567 struct nameidata nd; 568 struct unpcb *unp; 569 struct vnode *vp; 570 struct mount *mp; 571 cap_rights_t rights; 572 char *buf; 573 mode_t mode; 574 575 if (nam->sa_family != AF_UNIX) 576 return (EAFNOSUPPORT); 577 578 unp = sotounpcb(so); 579 KASSERT(unp != NULL, ("uipc_bind: unp == NULL")); 580 581 if (soun->sun_len > sizeof(struct sockaddr_un)) 582 return (EINVAL); 583 namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); 584 if (namelen <= 0) 585 return (EINVAL); 586 587 /* 588 * We don't allow simultaneous bind() calls on a single UNIX domain 589 * socket, so flag in-progress operations, and return an error if an 590 * operation is already in progress. 591 * 592 * Historically, we have not allowed a socket to be rebound, so this 593 * also returns an error. Not allowing re-binding simplifies the 594 * implementation and avoids a great many possible failure modes. 595 */ 596 UNP_PCB_LOCK(unp); 597 if (unp->unp_vnode != NULL) { 598 UNP_PCB_UNLOCK(unp); 599 return (EINVAL); 600 } 601 if (unp->unp_flags & UNP_BINDING) { 602 UNP_PCB_UNLOCK(unp); 603 return (EALREADY); 604 } 605 unp->unp_flags |= UNP_BINDING; 606 mode = unp->unp_mode & ~td->td_proc->p_pd->pd_cmask; 607 UNP_PCB_UNLOCK(unp); 608 609 buf = malloc(namelen + 1, M_TEMP, M_WAITOK); 610 bcopy(soun->sun_path, buf, namelen); 611 buf[namelen] = 0; 612 613 restart: 614 NDINIT_ATRIGHTS(&nd, CREATE, NOFOLLOW | LOCKPARENT | NOCACHE, 615 UIO_SYSSPACE, buf, fd, cap_rights_init_one(&rights, CAP_BINDAT)); 616 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */ 617 error = namei(&nd); 618 if (error) 619 goto error; 620 vp = nd.ni_vp; 621 if (vp != NULL || vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) { 622 NDFREE_PNBUF(&nd); 623 if (nd.ni_dvp == vp) 624 vrele(nd.ni_dvp); 625 else 626 vput(nd.ni_dvp); 627 if (vp != NULL) { 628 vrele(vp); 629 error = EADDRINUSE; 630 goto error; 631 } 632 error = vn_start_write(NULL, &mp, V_XSLEEP | V_PCATCH); 633 if (error) 634 goto error; 635 goto restart; 636 } 637 VATTR_NULL(&vattr); 638 vattr.va_type = VSOCK; 639 vattr.va_mode = mode; 640 #ifdef MAC 641 error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd, 642 &vattr); 643 #endif 644 if (error == 0) { 645 /* 646 * The prior lookup may have left LK_SHARED in cn_lkflags, 647 * and VOP_CREATE technically only requires the new vnode to 648 * be locked shared. Most filesystems will return the new vnode 649 * locked exclusive regardless, but we should explicitly 650 * specify that here since we require it and assert to that 651 * effect below. 652 */ 653 nd.ni_cnd.cn_lkflags = (nd.ni_cnd.cn_lkflags & ~LK_SHARED) | 654 LK_EXCLUSIVE; 655 error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr); 656 } 657 NDFREE_PNBUF(&nd); 658 if (error) { 659 VOP_VPUT_PAIR(nd.ni_dvp, NULL, true); 660 vn_finished_write(mp); 661 if (error == ERELOOKUP) 662 goto restart; 663 goto error; 664 } 665 vp = nd.ni_vp; 666 ASSERT_VOP_ELOCKED(vp, "uipc_bind"); 667 soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK); 668 669 UNP_PCB_LOCK(unp); 670 VOP_UNP_BIND(vp, unp); 671 unp->unp_vnode = vp; 672 unp->unp_addr = soun; 673 unp->unp_flags &= ~UNP_BINDING; 674 UNP_PCB_UNLOCK(unp); 675 vref(vp); 676 VOP_VPUT_PAIR(nd.ni_dvp, &vp, true); 677 vn_finished_write(mp); 678 free(buf, M_TEMP); 679 return (0); 680 681 error: 682 UNP_PCB_LOCK(unp); 683 unp->unp_flags &= ~UNP_BINDING; 684 UNP_PCB_UNLOCK(unp); 685 free(buf, M_TEMP); 686 return (error); 687 } 688 689 static int 690 uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td) 691 { 692 693 return (uipc_bindat(AT_FDCWD, so, nam, td)); 694 } 695 696 static int 697 uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 698 { 699 int error; 700 701 KASSERT(td == curthread, ("uipc_connect: td != curthread")); 702 error = unp_connect(so, nam, td); 703 return (error); 704 } 705 706 static int 707 uipc_connectat(int fd, struct socket *so, struct sockaddr *nam, 708 struct thread *td) 709 { 710 int error; 711 712 KASSERT(td == curthread, ("uipc_connectat: td != curthread")); 713 error = unp_connectat(fd, so, nam, td, false); 714 return (error); 715 } 716 717 static void 718 uipc_close(struct socket *so) 719 { 720 struct unpcb *unp, *unp2; 721 struct vnode *vp = NULL; 722 struct mtx *vplock; 723 724 unp = sotounpcb(so); 725 KASSERT(unp != NULL, ("uipc_close: unp == NULL")); 726 727 vplock = NULL; 728 if ((vp = unp->unp_vnode) != NULL) { 729 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 730 mtx_lock(vplock); 731 } 732 UNP_PCB_LOCK(unp); 733 if (vp && unp->unp_vnode == NULL) { 734 mtx_unlock(vplock); 735 vp = NULL; 736 } 737 if (vp != NULL) { 738 VOP_UNP_DETACH(vp); 739 unp->unp_vnode = NULL; 740 } 741 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 742 unp_disconnect(unp, unp2); 743 else 744 UNP_PCB_UNLOCK(unp); 745 if (vp) { 746 mtx_unlock(vplock); 747 vrele(vp); 748 } 749 } 750 751 static int 752 uipc_chmod(struct socket *so, mode_t mode, struct ucred *cred __unused, 753 struct thread *td __unused) 754 { 755 struct unpcb *unp; 756 int error; 757 758 if ((mode & ~ACCESSPERMS) != 0) 759 return (EINVAL); 760 761 error = 0; 762 unp = sotounpcb(so); 763 UNP_PCB_LOCK(unp); 764 if (unp->unp_vnode != NULL || (unp->unp_flags & UNP_BINDING) != 0) 765 error = EINVAL; 766 else 767 unp->unp_mode = mode; 768 UNP_PCB_UNLOCK(unp); 769 return (error); 770 } 771 772 static int 773 uipc_connect2(struct socket *so1, struct socket *so2) 774 { 775 struct unpcb *unp, *unp2; 776 777 if (so1->so_type != so2->so_type) 778 return (EPROTOTYPE); 779 780 unp = so1->so_pcb; 781 KASSERT(unp != NULL, ("uipc_connect2: unp == NULL")); 782 unp2 = so2->so_pcb; 783 KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL")); 784 unp_pcb_lock_pair(unp, unp2); 785 unp_connect2(so1, so2, false); 786 unp_pcb_unlock_pair(unp, unp2); 787 788 return (0); 789 } 790 791 static void 792 maybe_schedule_gc(void) 793 { 794 if (atomic_load_int(&unp_rights) != 0) 795 taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); 796 } 797 798 static void 799 uipc_detach(struct socket *so) 800 { 801 struct unpcb *unp, *unp2; 802 struct mtx *vplock; 803 struct vnode *vp; 804 805 unp = sotounpcb(so); 806 KASSERT(unp != NULL, ("uipc_detach: unp == NULL")); 807 808 vp = NULL; 809 vplock = NULL; 810 811 if (!SOLISTENING(so)) 812 unp_dispose(so); 813 814 UNP_LINK_WLOCK(); 815 LIST_REMOVE(unp, unp_link); 816 if (unp->unp_gcflag & UNPGC_DEAD) 817 LIST_REMOVE(unp, unp_dead); 818 unp->unp_gencnt = ++unp_gencnt; 819 --unp_count; 820 UNP_LINK_WUNLOCK(); 821 822 UNP_PCB_UNLOCK_ASSERT(unp); 823 restart: 824 if ((vp = unp->unp_vnode) != NULL) { 825 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 826 mtx_lock(vplock); 827 } 828 UNP_PCB_LOCK(unp); 829 if (unp->unp_vnode != vp && unp->unp_vnode != NULL) { 830 if (vplock) 831 mtx_unlock(vplock); 832 UNP_PCB_UNLOCK(unp); 833 goto restart; 834 } 835 if ((vp = unp->unp_vnode) != NULL) { 836 VOP_UNP_DETACH(vp); 837 unp->unp_vnode = NULL; 838 } 839 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 840 unp_disconnect(unp, unp2); 841 else 842 UNP_PCB_UNLOCK(unp); 843 844 UNP_REF_LIST_LOCK(); 845 while (!LIST_EMPTY(&unp->unp_refs)) { 846 struct unpcb *ref = LIST_FIRST(&unp->unp_refs); 847 848 unp_pcb_hold(ref); 849 UNP_REF_LIST_UNLOCK(); 850 851 MPASS(ref != unp); 852 UNP_PCB_UNLOCK_ASSERT(ref); 853 unp_drop(ref); 854 UNP_REF_LIST_LOCK(); 855 } 856 UNP_REF_LIST_UNLOCK(); 857 858 UNP_PCB_LOCK(unp); 859 unp->unp_socket->so_pcb = NULL; 860 unp->unp_socket = NULL; 861 free(unp->unp_addr, M_SONAME); 862 unp->unp_addr = NULL; 863 if (!unp_pcb_rele(unp)) 864 UNP_PCB_UNLOCK(unp); 865 if (vp) { 866 mtx_unlock(vplock); 867 vrele(vp); 868 } 869 maybe_schedule_gc(); 870 871 switch (so->so_type) { 872 case SOCK_STREAM: 873 case SOCK_SEQPACKET: 874 MPASS(SOLISTENING(so) || (STAILQ_EMPTY(&so->so_rcv.uxst_mbq) && 875 so->so_rcv.uxst_peer == NULL)); 876 break; 877 case SOCK_DGRAM: 878 /* 879 * Everything should have been unlinked/freed by unp_dispose() 880 * and/or unp_disconnect(). 881 */ 882 MPASS(so->so_rcv.uxdg_peeked == NULL); 883 MPASS(STAILQ_EMPTY(&so->so_rcv.uxdg_mb)); 884 MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); 885 MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); 886 } 887 888 mtx_destroy(&so->so_snd_mtx); 889 mtx_destroy(&so->so_rcv_mtx); 890 } 891 892 static int 893 uipc_disconnect(struct socket *so) 894 { 895 struct unpcb *unp, *unp2; 896 897 unp = sotounpcb(so); 898 KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL")); 899 900 UNP_PCB_LOCK(unp); 901 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 902 unp_disconnect(unp, unp2); 903 else 904 UNP_PCB_UNLOCK(unp); 905 return (0); 906 } 907 908 static void 909 uipc_fdclose(struct socket *so __unused) 910 { 911 /* 912 * Ensure that userspace can't create orphaned file descriptors without 913 * triggering garbage collection. Triggering GC from uipc_detach() is 914 * not sufficient, since that's only closed once a socket reference 915 * count drops to zero. 916 */ 917 maybe_schedule_gc(); 918 } 919 920 static int 921 uipc_listen(struct socket *so, int backlog, struct thread *td) 922 { 923 struct unpcb *unp; 924 int error; 925 926 MPASS(so->so_type != SOCK_DGRAM); 927 928 /* 929 * Synchronize with concurrent connection attempts. 930 */ 931 error = 0; 932 unp = sotounpcb(so); 933 UNP_PCB_LOCK(unp); 934 if (unp->unp_conn != NULL || (unp->unp_flags & UNP_CONNECTING) != 0) 935 error = EINVAL; 936 else if (unp->unp_vnode == NULL) 937 error = EDESTADDRREQ; 938 if (error != 0) { 939 UNP_PCB_UNLOCK(unp); 940 return (error); 941 } 942 943 SOCK_LOCK(so); 944 error = solisten_proto_check(so); 945 if (error == 0) { 946 cru2xt(td, &unp->unp_peercred); 947 if (!SOLISTENING(so)) { 948 (void)chgsbsize(so->so_cred->cr_uidinfo, 949 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 950 (void)chgsbsize(so->so_cred->cr_uidinfo, 951 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 952 } 953 solisten_proto(so, backlog); 954 } 955 SOCK_UNLOCK(so); 956 UNP_PCB_UNLOCK(unp); 957 return (error); 958 } 959 960 static int 961 uipc_peeraddr(struct socket *so, struct sockaddr *ret) 962 { 963 struct unpcb *unp, *unp2; 964 const struct sockaddr *sa; 965 966 unp = sotounpcb(so); 967 KASSERT(unp != NULL, ("uipc_peeraddr: unp == NULL")); 968 969 UNP_PCB_LOCK(unp); 970 unp2 = unp_pcb_lock_peer(unp); 971 if (unp2 != NULL) { 972 if (unp2->unp_addr != NULL) 973 sa = (struct sockaddr *)unp2->unp_addr; 974 else 975 sa = &sun_noname; 976 bcopy(sa, ret, sa->sa_len); 977 unp_pcb_unlock_pair(unp, unp2); 978 } else { 979 UNP_PCB_UNLOCK(unp); 980 sa = &sun_noname; 981 bcopy(sa, ret, sa->sa_len); 982 } 983 return (0); 984 } 985 986 /* 987 * pr_sosend() called with mbuf instead of uio is a kernel thread. NFS, 988 * netgraph(4) and other subsystems can call into socket code. The 989 * function will condition the mbuf so that it can be safely put onto socket 990 * buffer and calculate its char count and mbuf count. 991 * 992 * Note: we don't support receiving control data from a kernel thread. Our 993 * pr_sosend methods have MPASS() to check that. This may change. 994 */ 995 static void 996 uipc_reset_kernel_mbuf(struct mbuf *m, struct mchain *mc) 997 { 998 999 M_ASSERTPKTHDR(m); 1000 1001 m_clrprotoflags(m); 1002 m_tag_delete_chain(m, NULL); 1003 m->m_pkthdr.rcvif = NULL; 1004 m->m_pkthdr.flowid = 0; 1005 m->m_pkthdr.csum_flags = 0; 1006 m->m_pkthdr.fibnum = 0; 1007 m->m_pkthdr.rsstype = 0; 1008 1009 mc_init_m(mc, m); 1010 MPASS(m->m_pkthdr.len == mc->mc_len); 1011 } 1012 1013 #ifdef SOCKBUF_DEBUG 1014 static inline void 1015 uipc_stream_sbcheck(struct sockbuf *sb) 1016 { 1017 struct mbuf *d; 1018 u_int dacc, dccc, dctl, dmbcnt; 1019 bool notready = false; 1020 1021 dacc = dccc = dctl = dmbcnt = 0; 1022 STAILQ_FOREACH(d, &sb->uxst_mbq, m_stailq) { 1023 if (d == sb->uxst_fnrdy) { 1024 MPASS(d->m_flags & M_NOTREADY); 1025 notready = true; 1026 } 1027 if (d->m_type == MT_CONTROL) 1028 dctl += d->m_len; 1029 else if (d->m_type == MT_DATA) { 1030 dccc += d->m_len; 1031 if (!notready) 1032 dacc += d->m_len; 1033 } else 1034 MPASS(0); 1035 dmbcnt += MSIZE; 1036 if (d->m_flags & M_EXT) 1037 dmbcnt += d->m_ext.ext_size; 1038 if (d->m_stailq.stqe_next == NULL) 1039 MPASS(sb->uxst_mbq.stqh_last == &d->m_stailq.stqe_next); 1040 } 1041 MPASS(sb->uxst_fnrdy == NULL || notready); 1042 MPASS(dacc == sb->sb_acc); 1043 MPASS(dccc == sb->sb_ccc); 1044 MPASS(dctl == sb->sb_ctl); 1045 MPASS(dmbcnt == sb->sb_mbcnt); 1046 (void)STAILQ_EMPTY(&sb->uxst_mbq); 1047 } 1048 #define UIPC_STREAM_SBCHECK(sb) uipc_stream_sbcheck(sb) 1049 #else 1050 #define UIPC_STREAM_SBCHECK(sb) do {} while (0) 1051 #endif 1052 1053 /* 1054 * uipc_stream_sbspace() returns how much a writer can send, limited by char 1055 * count or mbuf memory use, whatever ends first. 1056 * 1057 * An obvious and legitimate reason for a socket having more data than allowed, 1058 * is lowering the limit with setsockopt(SO_RCVBUF) on already full buffer. 1059 * Also, sb_mbcnt may overcommit sb_mbmax in case if previous write observed 1060 * 'space < mbspace', but mchain allocated to hold 'space' bytes of data ended 1061 * up with 'mc_mlen > mbspace'. A typical scenario would be a full buffer with 1062 * writer trying to push in a large write, and a slow reader, that reads just 1063 * a few bytes at a time. In that case writer will keep creating new mbufs 1064 * with mc_split(). These mbufs will carry little chars, but will all point at 1065 * the same cluster, thus each adding cluster size to sb_mbcnt. This means we 1066 * will count same cluster many times potentially underutilizing socket buffer. 1067 * We aren't optimizing towards ineffective readers. Classic socket buffer had 1068 * the same "feature". 1069 */ 1070 static inline u_int 1071 uipc_stream_sbspace(struct sockbuf *sb) 1072 { 1073 u_int space, mbspace; 1074 1075 if (__predict_true(sb->sb_hiwat >= sb->sb_ccc + sb->sb_ctl)) 1076 space = sb->sb_hiwat - sb->sb_ccc - sb->sb_ctl; 1077 else 1078 return (0); 1079 if (__predict_true(sb->sb_mbmax >= sb->sb_mbcnt)) 1080 mbspace = sb->sb_mbmax - sb->sb_mbcnt; 1081 else 1082 return (0); 1083 1084 return (min(space, mbspace)); 1085 } 1086 1087 /* 1088 * UNIX version of generic sbwait() for writes. We wait on peer's receive 1089 * buffer, using our timeout. 1090 */ 1091 static int 1092 uipc_stream_sbwait(struct socket *so, sbintime_t timeo) 1093 { 1094 struct sockbuf *sb = &so->so_rcv; 1095 1096 SOCK_RECVBUF_LOCK_ASSERT(so); 1097 sb->sb_flags |= SB_WAIT; 1098 return (msleep_sbt(&sb->sb_acc, SOCK_RECVBUF_MTX(so), PSOCK | PCATCH, 1099 "sbwait", timeo, 0, 0)); 1100 } 1101 1102 static int 1103 uipc_sosend_stream_or_seqpacket(struct socket *so, struct sockaddr *addr, 1104 struct uio *uio0, struct mbuf *m, struct mbuf *c, int flags, 1105 struct thread *td) 1106 { 1107 struct unpcb *unp2; 1108 struct socket *so2; 1109 struct sockbuf *sb; 1110 struct uio *uio; 1111 struct mchain mc, cmc; 1112 size_t resid, sent; 1113 bool nonblock, eor, aio; 1114 int error; 1115 1116 MPASS((uio0 != NULL && m == NULL) || (m != NULL && uio0 == NULL)); 1117 MPASS(m == NULL || c == NULL); 1118 1119 if (__predict_false(flags & MSG_OOB)) 1120 return (EOPNOTSUPP); 1121 1122 nonblock = (so->so_state & SS_NBIO) || 1123 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1124 eor = flags & MSG_EOR; 1125 1126 mc = MCHAIN_INITIALIZER(&mc); 1127 cmc = MCHAIN_INITIALIZER(&cmc); 1128 sent = 0; 1129 aio = false; 1130 1131 if (m == NULL) { 1132 if (c != NULL && (error = unp_internalize(c, &cmc, td))) 1133 goto out; 1134 /* 1135 * This function may read more data from the uio than it would 1136 * then place on socket. That would leave uio inconsistent 1137 * upon return. Normally uio is allocated on the stack of the 1138 * syscall thread and we don't care about leaving it consistent. 1139 * However, aio(9) will allocate a uio as part of job and will 1140 * use it to track progress. We detect aio(9) checking the 1141 * SB_AIO_RUNNING flag. It is safe to check it without lock 1142 * cause it is set and cleared in the same taskqueue thread. 1143 * 1144 * This check can also produce a false positive: there is 1145 * aio(9) job and also there is a syscall we are serving now. 1146 * No sane software does that, it would leave to a mess in 1147 * the socket buffer, as aio(9) doesn't grab the I/O sx(9). 1148 * But syzkaller can create this mess. For such false positive 1149 * our goal is just don't panic or leak memory. 1150 */ 1151 if (__predict_false(so->so_snd.sb_flags & SB_AIO_RUNNING)) { 1152 uio = cloneuio(uio0); 1153 aio = true; 1154 } else { 1155 uio = uio0; 1156 resid = uio->uio_resid; 1157 } 1158 /* 1159 * Optimization for a case when our send fits into the receive 1160 * buffer - do the copyin before taking any locks, sized to our 1161 * send buffer. Later copyins will also take into account 1162 * space in the peer's receive buffer. 1163 */ 1164 error = mc_uiotomc(&mc, uio, so->so_snd.sb_hiwat, 0, M_WAITOK, 1165 eor ? M_EOR : 0); 1166 if (__predict_false(error)) 1167 goto out2; 1168 } else 1169 uipc_reset_kernel_mbuf(m, &mc); 1170 1171 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 1172 if (error) 1173 goto out2; 1174 1175 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 1176 goto out3; 1177 1178 if (unp2->unp_flags & UNP_WANTCRED_MASK) { 1179 /* 1180 * Credentials are passed only once on SOCK_STREAM and 1181 * SOCK_SEQPACKET (LOCAL_CREDS => WANTCRED_ONESHOT), or 1182 * forever (LOCAL_CREDS_PERSISTENT => WANTCRED_ALWAYS). 1183 */ 1184 unp_addsockcred(td, &cmc, unp2->unp_flags); 1185 unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT; 1186 } 1187 1188 /* 1189 * Cycle through the data to send and available space in the peer's 1190 * receive buffer. Put a reference on the peer socket, so that it 1191 * doesn't get freed while we sbwait(). If peer goes away, we will 1192 * observe the SBS_CANTRCVMORE and our sorele() will finalize peer's 1193 * socket destruction. 1194 */ 1195 so2 = unp2->unp_socket; 1196 soref(so2); 1197 UNP_PCB_UNLOCK(unp2); 1198 sb = &so2->so_rcv; 1199 while (mc.mc_len + cmc.mc_len > 0) { 1200 struct mchain mcnext = MCHAIN_INITIALIZER(&mcnext); 1201 u_int space; 1202 1203 SOCK_RECVBUF_LOCK(so2); 1204 restart: 1205 UIPC_STREAM_SBCHECK(sb); 1206 if (__predict_false(cmc.mc_len > sb->sb_hiwat)) { 1207 SOCK_RECVBUF_UNLOCK(so2); 1208 error = EMSGSIZE; 1209 goto out4; 1210 } 1211 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1212 SOCK_RECVBUF_UNLOCK(so2); 1213 error = EPIPE; 1214 goto out4; 1215 } 1216 /* 1217 * Wait on the peer socket receive buffer until we have enough 1218 * space to put at least control. The data is a stream and can 1219 * be put partially, but control is really a datagram. 1220 */ 1221 space = uipc_stream_sbspace(sb); 1222 if (space < sb->sb_lowat || space < cmc.mc_len) { 1223 if (nonblock) { 1224 if (aio) 1225 sb->uxst_flags |= UXST_PEER_AIO; 1226 SOCK_RECVBUF_UNLOCK(so2); 1227 if (aio) { 1228 SOCK_SENDBUF_LOCK(so); 1229 so->so_snd.sb_ccc = 1230 so->so_snd.sb_hiwat - space; 1231 SOCK_SENDBUF_UNLOCK(so); 1232 } 1233 error = EWOULDBLOCK; 1234 goto out4; 1235 } 1236 if ((error = uipc_stream_sbwait(so2, 1237 so->so_snd.sb_timeo)) != 0) { 1238 SOCK_RECVBUF_UNLOCK(so2); 1239 goto out4; 1240 } else 1241 goto restart; 1242 } 1243 MPASS(space >= cmc.mc_len); 1244 space -= cmc.mc_len; 1245 if (space == 0) { 1246 /* There is space only to send control. */ 1247 MPASS(!STAILQ_EMPTY(&cmc.mc_q)); 1248 mcnext = mc; 1249 mc = MCHAIN_INITIALIZER(&mc); 1250 } else if (space < mc.mc_len) { 1251 /* Not enough space. */ 1252 if (__predict_false(mc_split(&mc, &mcnext, space, 1253 M_NOWAIT) == ENOMEM)) { 1254 /* 1255 * If allocation failed use M_WAITOK and merge 1256 * the chain back. Next time mc_split() will 1257 * easily split at the same place. Only if we 1258 * race with setsockopt(SO_RCVBUF) shrinking 1259 * sb_hiwat can this happen more than once. 1260 */ 1261 SOCK_RECVBUF_UNLOCK(so2); 1262 (void)mc_split(&mc, &mcnext, space, M_WAITOK); 1263 mc_concat(&mc, &mcnext); 1264 SOCK_RECVBUF_LOCK(so2); 1265 goto restart; 1266 } 1267 MPASS(mc.mc_len == space); 1268 } 1269 if (!STAILQ_EMPTY(&cmc.mc_q)) { 1270 STAILQ_CONCAT(&sb->uxst_mbq, &cmc.mc_q); 1271 sb->sb_ctl += cmc.mc_len; 1272 sb->sb_mbcnt += cmc.mc_mlen; 1273 cmc.mc_len = 0; 1274 } 1275 sent += mc.mc_len; 1276 if (sb->uxst_fnrdy == NULL) 1277 sb->sb_acc += mc.mc_len; 1278 sb->sb_ccc += mc.mc_len; 1279 sb->sb_mbcnt += mc.mc_mlen; 1280 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 1281 UIPC_STREAM_SBCHECK(sb); 1282 space = uipc_stream_sbspace(sb); 1283 sorwakeup_locked(so2); 1284 if (!STAILQ_EMPTY(&mcnext.mc_q)) { 1285 /* 1286 * Such assignment is unsafe in general, but it is 1287 * safe with !STAILQ_EMPTY(&mcnext.mc_q). In C++ we 1288 * could reload = for STAILQs :) 1289 */ 1290 mc = mcnext; 1291 } else if (uio != NULL && uio->uio_resid > 0) { 1292 /* 1293 * Copyin sum of peer's receive buffer space and our 1294 * sb_hiwat, which is our virtual send buffer size. 1295 * See comment above unpst_sendspace declaration. 1296 * We are reading sb_hiwat locklessly, cause a) we 1297 * don't care about an application that does send(2) 1298 * and setsockopt(2) racing internally, and for an 1299 * application that does this in sequence we will see 1300 * the correct value cause sbsetopt() uses buffer lock 1301 * and we also have already acquired it at least once. 1302 */ 1303 error = mc_uiotomc(&mc, uio, space + 1304 atomic_load_int(&so->so_snd.sb_hiwat), 0, M_WAITOK, 1305 eor ? M_EOR : 0); 1306 if (__predict_false(error)) 1307 goto out4; 1308 } else 1309 mc = MCHAIN_INITIALIZER(&mc); 1310 } 1311 1312 MPASS(STAILQ_EMPTY(&mc.mc_q)); 1313 1314 td->td_ru.ru_msgsnd++; 1315 out4: 1316 sorele(so2); 1317 out3: 1318 SOCK_IO_SEND_UNLOCK(so); 1319 out2: 1320 if (aio) { 1321 freeuio(uio); 1322 uioadvance(uio0, sent); 1323 } else if (uio != NULL) 1324 uio->uio_resid = resid - sent; 1325 if (!mc_empty(&cmc)) 1326 unp_scan(mc_first(&cmc), unp_freerights); 1327 out: 1328 mc_freem(&mc); 1329 mc_freem(&cmc); 1330 1331 return (error); 1332 } 1333 1334 /* 1335 * Wakeup a writer, used by recv(2) and shutdown(2). 1336 * 1337 * @param so Points to a connected stream socket with receive buffer locked 1338 * 1339 * In a blocking mode peer is sleeping on our receive buffer, and we need just 1340 * wakeup(9) on it. But to wake up various event engines, we need to reach 1341 * over to peer's selinfo. This can be safely done as the socket buffer 1342 * receive lock is protecting us from the peer going away. 1343 */ 1344 static void 1345 uipc_wakeup_writer(struct socket *so) 1346 { 1347 struct sockbuf *sb = &so->so_rcv; 1348 struct selinfo *sel; 1349 1350 SOCK_RECVBUF_LOCK_ASSERT(so); 1351 MPASS(sb->uxst_peer != NULL); 1352 1353 sel = &sb->uxst_peer->so_wrsel; 1354 1355 if (sb->uxst_flags & UXST_PEER_SEL) { 1356 selwakeuppri(sel, PSOCK); 1357 /* 1358 * XXXGL: sowakeup() does SEL_WAITING() without locks. 1359 */ 1360 if (!SEL_WAITING(sel)) 1361 sb->uxst_flags &= ~UXST_PEER_SEL; 1362 } 1363 if (sb->sb_flags & SB_WAIT) { 1364 sb->sb_flags &= ~SB_WAIT; 1365 wakeup(&sb->sb_acc); 1366 } 1367 KNOTE_LOCKED(&sel->si_note, 0); 1368 SOCK_RECVBUF_UNLOCK(so); 1369 } 1370 1371 static void 1372 uipc_cantrcvmore(struct socket *so) 1373 { 1374 1375 SOCK_RECVBUF_LOCK(so); 1376 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 1377 selwakeuppri(&so->so_rdsel, PSOCK); 1378 KNOTE_LOCKED(&so->so_rdsel.si_note, 0); 1379 if (so->so_rcv.uxst_peer != NULL) 1380 uipc_wakeup_writer(so); 1381 else 1382 SOCK_RECVBUF_UNLOCK(so); 1383 } 1384 1385 static int 1386 uipc_soreceive_stream_or_seqpacket(struct socket *so, struct sockaddr **psa, 1387 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1388 { 1389 struct sockbuf *sb = &so->so_rcv; 1390 struct mbuf *control, *m, *first, *part, *next; 1391 u_int ctl, space, datalen, mbcnt, partlen; 1392 int error, flags; 1393 bool nonblock, waitall, peek; 1394 1395 MPASS(mp0 == NULL); 1396 1397 if (psa != NULL) 1398 *psa = NULL; 1399 if (controlp != NULL) 1400 *controlp = NULL; 1401 1402 flags = flagsp != NULL ? *flagsp : 0; 1403 nonblock = (so->so_state & SS_NBIO) || 1404 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1405 peek = flags & MSG_PEEK; 1406 waitall = (flags & MSG_WAITALL) && !peek; 1407 1408 /* 1409 * This check may fail only on a socket that never went through 1410 * connect(2). We can check this locklessly, cause: a) for a new born 1411 * socket we don't care about applications that may race internally 1412 * between connect(2) and recv(2), and b) for a dying socket if we 1413 * miss update by unp_sosidisconnected(), we would still get the check 1414 * correct. For dying socket we would observe SBS_CANTRCVMORE later. 1415 */ 1416 if (__predict_false((atomic_load_short(&so->so_state) & 1417 (SS_ISCONNECTED|SS_ISDISCONNECTED)) == 0)) 1418 return (ENOTCONN); 1419 1420 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 1421 if (__predict_false(error)) 1422 return (error); 1423 1424 restart: 1425 SOCK_RECVBUF_LOCK(so); 1426 UIPC_STREAM_SBCHECK(sb); 1427 while (sb->sb_acc < sb->sb_lowat && 1428 (sb->sb_ctl == 0 || controlp == NULL)) { 1429 if (so->so_error) { 1430 error = so->so_error; 1431 if (!peek) 1432 so->so_error = 0; 1433 SOCK_RECVBUF_UNLOCK(so); 1434 SOCK_IO_RECV_UNLOCK(so); 1435 return (error); 1436 } 1437 if (sb->sb_state & SBS_CANTRCVMORE) { 1438 SOCK_RECVBUF_UNLOCK(so); 1439 SOCK_IO_RECV_UNLOCK(so); 1440 return (0); 1441 } 1442 if (nonblock) { 1443 SOCK_RECVBUF_UNLOCK(so); 1444 SOCK_IO_RECV_UNLOCK(so); 1445 return (EWOULDBLOCK); 1446 } 1447 error = sbwait(so, SO_RCV); 1448 if (error) { 1449 SOCK_RECVBUF_UNLOCK(so); 1450 SOCK_IO_RECV_UNLOCK(so); 1451 return (error); 1452 } 1453 } 1454 1455 MPASS(STAILQ_FIRST(&sb->uxst_mbq)); 1456 MPASS(sb->sb_acc > 0 || sb->sb_ctl > 0); 1457 1458 mbcnt = 0; 1459 ctl = 0; 1460 first = STAILQ_FIRST(&sb->uxst_mbq); 1461 if (first->m_type == MT_CONTROL) { 1462 control = first; 1463 STAILQ_FOREACH_FROM(first, &sb->uxst_mbq, m_stailq) { 1464 if (first->m_type != MT_CONTROL) 1465 break; 1466 ctl += first->m_len; 1467 mbcnt += MSIZE; 1468 if (first->m_flags & M_EXT) 1469 mbcnt += first->m_ext.ext_size; 1470 } 1471 } else 1472 control = NULL; 1473 1474 /* 1475 * Find split point for the next copyout. On exit from the loop, 1476 * 'next' points to the new head of the buffer STAILQ and 'datalen' 1477 * contains the amount of data we will copy out at the end. The 1478 * copyout is protected by the I/O lock only, as writers can only 1479 * append to the buffer. We need to record the socket buffer state 1480 * and do all length adjustments before dropping the socket buffer lock. 1481 */ 1482 for (space = uio->uio_resid, m = next = first, part = NULL, datalen = 0; 1483 space > 0 && m != sb->uxst_fnrdy && m->m_type == MT_DATA; 1484 m = STAILQ_NEXT(m, m_stailq)) { 1485 if (space >= m->m_len) { 1486 space -= m->m_len; 1487 datalen += m->m_len; 1488 mbcnt += MSIZE; 1489 if (m->m_flags & M_EXT) 1490 mbcnt += m->m_ext.ext_size; 1491 if (m->m_flags & M_EOR) { 1492 flags |= MSG_EOR; 1493 next = STAILQ_NEXT(m, m_stailq); 1494 break; 1495 } 1496 } else { 1497 datalen += space; 1498 partlen = space; 1499 if (!peek) { 1500 m->m_len -= partlen; 1501 m->m_data += partlen; 1502 } 1503 next = part = m; 1504 break; 1505 } 1506 next = STAILQ_NEXT(m, m_stailq); 1507 } 1508 1509 if (!peek) { 1510 if (next == NULL) 1511 STAILQ_INIT(&sb->uxst_mbq); 1512 else 1513 STAILQ_FIRST(&sb->uxst_mbq) = next; 1514 MPASS(sb->sb_acc >= datalen); 1515 sb->sb_acc -= datalen; 1516 sb->sb_ccc -= datalen; 1517 MPASS(sb->sb_ctl >= ctl); 1518 sb->sb_ctl -= ctl; 1519 MPASS(sb->sb_mbcnt >= mbcnt); 1520 sb->sb_mbcnt -= mbcnt; 1521 UIPC_STREAM_SBCHECK(sb); 1522 if (__predict_true(sb->uxst_peer != NULL)) { 1523 struct unpcb *unp2; 1524 bool aio; 1525 1526 if ((aio = sb->uxst_flags & UXST_PEER_AIO)) 1527 sb->uxst_flags &= ~UXST_PEER_AIO; 1528 1529 uipc_wakeup_writer(so); 1530 /* 1531 * XXXGL: need to go through uipc_lock_peer() after 1532 * the receive buffer lock dropped, it was protecting 1533 * us from unp_soisdisconnected(). The aio workarounds 1534 * should be refactored to the aio(4) side. 1535 */ 1536 if (aio && uipc_lock_peer(so, &unp2) == 0) { 1537 struct socket *so2 = unp2->unp_socket; 1538 1539 SOCK_SENDBUF_LOCK(so2); 1540 so2->so_snd.sb_ccc -= datalen; 1541 sowakeup_aio(so2, SO_SND); 1542 SOCK_SENDBUF_UNLOCK(so2); 1543 UNP_PCB_UNLOCK(unp2); 1544 } 1545 } else 1546 SOCK_RECVBUF_UNLOCK(so); 1547 } else 1548 SOCK_RECVBUF_UNLOCK(so); 1549 1550 while (control != NULL && control->m_type == MT_CONTROL) { 1551 if (!peek) { 1552 /* 1553 * unp_externalize() failure must abort entire read(2). 1554 * Such failure should also free the problematic 1555 * control, but link back the remaining data to the head 1556 * of the buffer, so that socket is not left in a state 1557 * where it can't progress forward with reading. 1558 * Probability of such a failure is really low, so it 1559 * is fine that we need to perform pretty complex 1560 * operation here to reconstruct the buffer. 1561 */ 1562 error = unp_externalize(control, controlp, flags); 1563 control = m_free(control); 1564 if (__predict_false(error && control != NULL)) { 1565 struct mchain cmc; 1566 1567 mc_init_m(&cmc, control); 1568 1569 SOCK_RECVBUF_LOCK(so); 1570 if (__predict_false( 1571 (sb->sb_state & SBS_CANTRCVMORE) || 1572 cmc.mc_len + sb->sb_ccc + sb->sb_ctl > 1573 sb->sb_hiwat)) { 1574 /* 1575 * While the lock was dropped and we 1576 * were failing in unp_externalize(), 1577 * the peer could has a) disconnected, 1578 * b) filled the buffer so that we 1579 * can't prepend data back. 1580 * These are two edge conditions that 1581 * we just can't handle, so lose the 1582 * data and return the error. 1583 */ 1584 SOCK_RECVBUF_UNLOCK(so); 1585 SOCK_IO_RECV_UNLOCK(so); 1586 unp_scan(mc_first(&cmc), 1587 unp_freerights); 1588 mc_freem(&cmc); 1589 return (error); 1590 } 1591 1592 UIPC_STREAM_SBCHECK(sb); 1593 /* XXXGL: STAILQ_PREPEND */ 1594 STAILQ_CONCAT(&cmc.mc_q, &sb->uxst_mbq); 1595 STAILQ_SWAP(&cmc.mc_q, &sb->uxst_mbq, mbuf); 1596 1597 sb->sb_ctl = sb->sb_acc = sb->sb_ccc = 1598 sb->sb_mbcnt = 0; 1599 STAILQ_FOREACH(m, &sb->uxst_mbq, m_stailq) { 1600 if (m->m_type == MT_DATA) { 1601 sb->sb_acc += m->m_len; 1602 sb->sb_ccc += m->m_len; 1603 } else { 1604 sb->sb_ctl += m->m_len; 1605 } 1606 sb->sb_mbcnt += MSIZE; 1607 if (m->m_flags & M_EXT) 1608 sb->sb_mbcnt += 1609 m->m_ext.ext_size; 1610 } 1611 UIPC_STREAM_SBCHECK(sb); 1612 SOCK_RECVBUF_UNLOCK(so); 1613 SOCK_IO_RECV_UNLOCK(so); 1614 return (error); 1615 } 1616 if (controlp != NULL) { 1617 while (*controlp != NULL) 1618 controlp = &(*controlp)->m_next; 1619 } 1620 } else { 1621 /* 1622 * XXXGL 1623 * 1624 * In MSG_PEEK case control is not externalized. This 1625 * means we are leaking some kernel pointers to the 1626 * userland. They are useless to a law-abiding 1627 * application, but may be useful to a malware. This 1628 * is what the historical implementation in the 1629 * soreceive_generic() did. To be improved? 1630 */ 1631 if (controlp != NULL) { 1632 *controlp = m_copym(control, 0, control->m_len, 1633 M_WAITOK); 1634 controlp = &(*controlp)->m_next; 1635 } 1636 control = STAILQ_NEXT(control, m_stailq); 1637 } 1638 } 1639 1640 for (m = first; datalen > 0; m = next) { 1641 void *data; 1642 u_int len; 1643 1644 next = STAILQ_NEXT(m, m_stailq); 1645 if (m == part) { 1646 data = peek ? 1647 mtod(m, char *) : mtod(m, char *) - partlen; 1648 len = partlen; 1649 } else { 1650 data = mtod(m, char *); 1651 len = m->m_len; 1652 } 1653 error = uiomove(data, len, uio); 1654 if (__predict_false(error)) { 1655 if (!peek) 1656 for (; m != part && datalen > 0; m = next) { 1657 next = STAILQ_NEXT(m, m_stailq); 1658 MPASS(datalen >= m->m_len); 1659 datalen -= m->m_len; 1660 m_free(m); 1661 } 1662 SOCK_IO_RECV_UNLOCK(so); 1663 return (error); 1664 } 1665 datalen -= len; 1666 if (!peek && m != part) 1667 m_free(m); 1668 } 1669 if (waitall && !(flags & MSG_EOR) && uio->uio_resid > 0) 1670 goto restart; 1671 SOCK_IO_RECV_UNLOCK(so); 1672 1673 if (flagsp != NULL) 1674 *flagsp |= flags; 1675 1676 uio->uio_td->td_ru.ru_msgrcv++; 1677 1678 return (0); 1679 } 1680 1681 static int 1682 uipc_sopoll_stream_or_seqpacket(struct socket *so, int events, 1683 struct thread *td) 1684 { 1685 struct unpcb *unp = sotounpcb(so); 1686 int revents; 1687 1688 UNP_PCB_LOCK(unp); 1689 if (SOLISTENING(so)) { 1690 /* The above check is safe, since conversion to listening uses 1691 * both protocol and socket lock. 1692 */ 1693 SOCK_LOCK(so); 1694 if (!(events & (POLLIN | POLLRDNORM))) 1695 revents = 0; 1696 else if (!TAILQ_EMPTY(&so->sol_comp)) 1697 revents = events & (POLLIN | POLLRDNORM); 1698 else if (so->so_error) 1699 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 1700 else { 1701 selrecord(td, &so->so_rdsel); 1702 revents = 0; 1703 } 1704 SOCK_UNLOCK(so); 1705 } else { 1706 if (so->so_state & SS_ISDISCONNECTED) 1707 revents = POLLHUP; 1708 else 1709 revents = 0; 1710 if (events & (POLLIN | POLLRDNORM | POLLRDHUP)) { 1711 SOCK_RECVBUF_LOCK(so); 1712 if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat || 1713 so->so_error || so->so_rerror) 1714 revents |= events & (POLLIN | POLLRDNORM); 1715 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1716 revents |= events & 1717 (POLLIN | POLLRDNORM | POLLRDHUP); 1718 if (!(revents & (POLLIN | POLLRDNORM | POLLRDHUP))) { 1719 selrecord(td, &so->so_rdsel); 1720 so->so_rcv.sb_flags |= SB_SEL; 1721 } 1722 SOCK_RECVBUF_UNLOCK(so); 1723 } 1724 if (events & (POLLOUT | POLLWRNORM)) { 1725 struct socket *so2 = so->so_rcv.uxst_peer; 1726 1727 if (so2 != NULL) { 1728 struct sockbuf *sb = &so2->so_rcv; 1729 1730 SOCK_RECVBUF_LOCK(so2); 1731 if (uipc_stream_sbspace(sb) >= sb->sb_lowat) 1732 revents |= events & 1733 (POLLOUT | POLLWRNORM); 1734 if (sb->sb_state & SBS_CANTRCVMORE) 1735 revents |= POLLHUP; 1736 if (!(revents & (POLLOUT | POLLWRNORM))) { 1737 so2->so_rcv.uxst_flags |= UXST_PEER_SEL; 1738 selrecord(td, &so->so_wrsel); 1739 } 1740 SOCK_RECVBUF_UNLOCK(so2); 1741 } else 1742 selrecord(td, &so->so_wrsel); 1743 } 1744 } 1745 UNP_PCB_UNLOCK(unp); 1746 return (revents); 1747 } 1748 1749 static void 1750 uipc_wrknl_lock(void *arg) 1751 { 1752 struct socket *so = arg; 1753 struct unpcb *unp = sotounpcb(so); 1754 1755 retry: 1756 if (SOLISTENING(so)) { 1757 SOLISTEN_LOCK(so); 1758 } else { 1759 UNP_PCB_LOCK(unp); 1760 if (__predict_false(SOLISTENING(so))) { 1761 UNP_PCB_UNLOCK(unp); 1762 goto retry; 1763 } 1764 if (so->so_rcv.uxst_peer != NULL) 1765 SOCK_RECVBUF_LOCK(so->so_rcv.uxst_peer); 1766 } 1767 } 1768 1769 static void 1770 uipc_wrknl_unlock(void *arg) 1771 { 1772 struct socket *so = arg; 1773 struct unpcb *unp = sotounpcb(so); 1774 1775 if (SOLISTENING(so)) 1776 SOLISTEN_UNLOCK(so); 1777 else { 1778 if (so->so_rcv.uxst_peer != NULL) 1779 SOCK_RECVBUF_UNLOCK(so->so_rcv.uxst_peer); 1780 UNP_PCB_UNLOCK(unp); 1781 } 1782 } 1783 1784 static void 1785 uipc_wrknl_assert_lock(void *arg, int what) 1786 { 1787 struct socket *so = arg; 1788 1789 if (SOLISTENING(so)) { 1790 if (what == LA_LOCKED) 1791 SOLISTEN_LOCK_ASSERT(so); 1792 else 1793 SOLISTEN_UNLOCK_ASSERT(so); 1794 } else { 1795 /* 1796 * The pr_soreceive method will put a note without owning the 1797 * unp lock, so we can't assert it here. But we can safely 1798 * dereference uxst_peer pointer, since receive buffer lock 1799 * is assumed to be held here. 1800 */ 1801 if (what == LA_LOCKED && so->so_rcv.uxst_peer != NULL) 1802 SOCK_RECVBUF_LOCK_ASSERT(so->so_rcv.uxst_peer); 1803 } 1804 } 1805 1806 static void 1807 uipc_filt_sowdetach(struct knote *kn) 1808 { 1809 struct socket *so = kn->kn_fp->f_data; 1810 1811 uipc_wrknl_lock(so); 1812 knlist_remove(&so->so_wrsel.si_note, kn, 1); 1813 uipc_wrknl_unlock(so); 1814 } 1815 1816 static int 1817 uipc_filt_sowrite(struct knote *kn, long hint) 1818 { 1819 struct socket *so = kn->kn_fp->f_data, *so2; 1820 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1821 1822 if (SOLISTENING(so)) 1823 return (0); 1824 1825 if (unp2 == NULL) { 1826 if (so->so_state & SS_ISDISCONNECTED) { 1827 kn->kn_flags |= EV_EOF; 1828 kn->kn_fflags = so->so_error; 1829 return (1); 1830 } else 1831 return (0); 1832 } 1833 1834 so2 = unp2->unp_socket; 1835 SOCK_RECVBUF_LOCK_ASSERT(so2); 1836 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1837 1838 if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { 1839 kn->kn_flags |= EV_EOF; 1840 return (1); 1841 } else if (kn->kn_sfflags & NOTE_LOWAT) 1842 return (kn->kn_data >= kn->kn_sdata); 1843 else 1844 return (kn->kn_data >= so2->so_rcv.sb_lowat); 1845 } 1846 1847 static int 1848 uipc_filt_soempty(struct knote *kn, long hint) 1849 { 1850 struct socket *so = kn->kn_fp->f_data, *so2; 1851 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1852 1853 if (SOLISTENING(so) || unp2 == NULL) 1854 return (1); 1855 1856 so2 = unp2->unp_socket; 1857 SOCK_RECVBUF_LOCK_ASSERT(so2); 1858 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1859 1860 return (kn->kn_data == 0 ? 1 : 0); 1861 } 1862 1863 static const struct filterops uipc_write_filtops = { 1864 .f_isfd = 1, 1865 .f_detach = uipc_filt_sowdetach, 1866 .f_event = uipc_filt_sowrite, 1867 .f_copy = knote_triv_copy, 1868 }; 1869 static const struct filterops uipc_empty_filtops = { 1870 .f_isfd = 1, 1871 .f_detach = uipc_filt_sowdetach, 1872 .f_event = uipc_filt_soempty, 1873 .f_copy = knote_triv_copy, 1874 }; 1875 1876 static int 1877 uipc_kqfilter_stream_or_seqpacket(struct socket *so, struct knote *kn) 1878 { 1879 struct unpcb *unp = sotounpcb(so); 1880 struct knlist *knl; 1881 1882 switch (kn->kn_filter) { 1883 case EVFILT_READ: 1884 return (sokqfilter_generic(so, kn)); 1885 case EVFILT_WRITE: 1886 kn->kn_fop = &uipc_write_filtops; 1887 break; 1888 case EVFILT_EMPTY: 1889 kn->kn_fop = &uipc_empty_filtops; 1890 break; 1891 default: 1892 return (EINVAL); 1893 } 1894 1895 knl = &so->so_wrsel.si_note; 1896 UNP_PCB_LOCK(unp); 1897 if (SOLISTENING(so)) { 1898 SOLISTEN_LOCK(so); 1899 knlist_add(knl, kn, 1); 1900 SOLISTEN_UNLOCK(so); 1901 } else { 1902 struct socket *so2 = so->so_rcv.uxst_peer; 1903 1904 if (so2 != NULL) 1905 SOCK_RECVBUF_LOCK(so2); 1906 knlist_add(knl, kn, 1); 1907 if (so2 != NULL) 1908 SOCK_RECVBUF_UNLOCK(so2); 1909 } 1910 UNP_PCB_UNLOCK(unp); 1911 return (0); 1912 } 1913 1914 /* PF_UNIX/SOCK_DGRAM version of sbspace() */ 1915 static inline bool 1916 uipc_dgram_sbspace(struct sockbuf *sb, u_int cc, u_int mbcnt) 1917 { 1918 u_int bleft, mleft; 1919 1920 /* 1921 * Negative space may happen if send(2) is followed by 1922 * setsockopt(SO_SNDBUF/SO_RCVBUF) that shrinks maximum. 1923 */ 1924 if (__predict_false(sb->sb_hiwat < sb->uxdg_cc || 1925 sb->sb_mbmax < sb->uxdg_mbcnt)) 1926 return (false); 1927 1928 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) 1929 return (false); 1930 1931 bleft = sb->sb_hiwat - sb->uxdg_cc; 1932 mleft = sb->sb_mbmax - sb->uxdg_mbcnt; 1933 1934 return (bleft >= cc && mleft >= mbcnt); 1935 } 1936 1937 /* 1938 * PF_UNIX/SOCK_DGRAM send 1939 * 1940 * Allocate a record consisting of 3 mbufs in the sequence of 1941 * from -> control -> data and append it to the socket buffer. 1942 * 1943 * The first mbuf carries sender's name and is a pkthdr that stores 1944 * overall length of datagram, its memory consumption and control length. 1945 */ 1946 #define ctllen PH_loc.thirtytwo[1] 1947 _Static_assert(offsetof(struct pkthdr, memlen) + sizeof(u_int) <= 1948 offsetof(struct pkthdr, ctllen), "unix/dgram can not store ctllen"); 1949 static int 1950 uipc_sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1951 struct mbuf *m, struct mbuf *c, int flags, struct thread *td) 1952 { 1953 struct unpcb *unp, *unp2; 1954 const struct sockaddr *from; 1955 struct socket *so2; 1956 struct sockbuf *sb; 1957 struct mchain cmc = MCHAIN_INITIALIZER(&cmc); 1958 struct mbuf *f; 1959 u_int cc, ctl, mbcnt; 1960 u_int dcc __diagused, dctl __diagused, dmbcnt __diagused; 1961 int error; 1962 1963 MPASS((uio != NULL && m == NULL) || (m != NULL && uio == NULL)); 1964 1965 error = 0; 1966 f = NULL; 1967 1968 if (__predict_false(flags & MSG_OOB)) { 1969 error = EOPNOTSUPP; 1970 goto out; 1971 } 1972 if (m == NULL) { 1973 if (__predict_false(uio->uio_resid > unpdg_maxdgram)) { 1974 error = EMSGSIZE; 1975 goto out; 1976 } 1977 m = m_uiotombuf(uio, M_WAITOK, 0, max_hdr, M_PKTHDR); 1978 if (__predict_false(m == NULL)) { 1979 error = EFAULT; 1980 goto out; 1981 } 1982 f = m_gethdr(M_WAITOK, MT_SONAME); 1983 cc = m->m_pkthdr.len; 1984 mbcnt = MSIZE + m->m_pkthdr.memlen; 1985 if (c != NULL && (error = unp_internalize(c, &cmc, td))) 1986 goto out; 1987 } else { 1988 struct mchain mc; 1989 1990 uipc_reset_kernel_mbuf(m, &mc); 1991 cc = mc.mc_len; 1992 mbcnt = mc.mc_mlen; 1993 if (__predict_false(m->m_pkthdr.len > unpdg_maxdgram)) { 1994 error = EMSGSIZE; 1995 goto out; 1996 } 1997 if ((f = m_gethdr(M_NOWAIT, MT_SONAME)) == NULL) { 1998 error = ENOBUFS; 1999 goto out; 2000 } 2001 } 2002 2003 unp = sotounpcb(so); 2004 MPASS(unp); 2005 2006 /* 2007 * XXXGL: would be cool to fully remove so_snd out of the equation 2008 * and avoid this lock, which is not only extraneous, but also being 2009 * released, thus still leaving possibility for a race. We can easily 2010 * handle SBS_CANTSENDMORE/SS_ISCONNECTED complement in unpcb, but it 2011 * is more difficult to invent something to handle so_error. 2012 */ 2013 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2014 if (error) 2015 goto out2; 2016 SOCK_SENDBUF_LOCK(so); 2017 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2018 SOCK_SENDBUF_UNLOCK(so); 2019 error = EPIPE; 2020 goto out3; 2021 } 2022 if (so->so_error != 0) { 2023 error = so->so_error; 2024 so->so_error = 0; 2025 SOCK_SENDBUF_UNLOCK(so); 2026 goto out3; 2027 } 2028 if (((so->so_state & SS_ISCONNECTED) == 0) && addr == NULL) { 2029 SOCK_SENDBUF_UNLOCK(so); 2030 error = EDESTADDRREQ; 2031 goto out3; 2032 } 2033 SOCK_SENDBUF_UNLOCK(so); 2034 2035 if (addr != NULL) { 2036 if ((error = unp_connectat(AT_FDCWD, so, addr, td, true))) 2037 goto out3; 2038 UNP_PCB_LOCK_ASSERT(unp); 2039 unp2 = unp->unp_conn; 2040 UNP_PCB_LOCK_ASSERT(unp2); 2041 } else { 2042 UNP_PCB_LOCK(unp); 2043 unp2 = unp_pcb_lock_peer(unp); 2044 if (unp2 == NULL) { 2045 UNP_PCB_UNLOCK(unp); 2046 error = ENOTCONN; 2047 goto out3; 2048 } 2049 } 2050 2051 if (unp2->unp_flags & UNP_WANTCRED_MASK) 2052 unp_addsockcred(td, &cmc, unp2->unp_flags); 2053 if (unp->unp_addr != NULL) 2054 from = (struct sockaddr *)unp->unp_addr; 2055 else 2056 from = &sun_noname; 2057 f->m_len = from->sa_len; 2058 MPASS(from->sa_len <= MLEN); 2059 bcopy(from, mtod(f, void *), from->sa_len); 2060 2061 /* 2062 * Concatenate mbufs: from -> control -> data. 2063 * Save overall cc and mbcnt in "from" mbuf. 2064 */ 2065 if (!STAILQ_EMPTY(&cmc.mc_q)) { 2066 f->m_next = mc_first(&cmc); 2067 mc_last(&cmc)->m_next = m; 2068 /* XXXGL: This is dirty as well as rollback after ENOBUFS. */ 2069 STAILQ_INIT(&cmc.mc_q); 2070 } else 2071 f->m_next = m; 2072 m = NULL; 2073 ctl = f->m_len + cmc.mc_len; 2074 mbcnt += cmc.mc_mlen; 2075 #ifdef INVARIANTS 2076 dcc = dctl = dmbcnt = 0; 2077 for (struct mbuf *mb = f; mb != NULL; mb = mb->m_next) { 2078 if (mb->m_type == MT_DATA) 2079 dcc += mb->m_len; 2080 else 2081 dctl += mb->m_len; 2082 dmbcnt += MSIZE; 2083 if (mb->m_flags & M_EXT) 2084 dmbcnt += mb->m_ext.ext_size; 2085 } 2086 MPASS(dcc == cc); 2087 MPASS(dctl == ctl); 2088 MPASS(dmbcnt == mbcnt); 2089 #endif 2090 f->m_pkthdr.len = cc + ctl; 2091 f->m_pkthdr.memlen = mbcnt; 2092 f->m_pkthdr.ctllen = ctl; 2093 2094 /* 2095 * Destination socket buffer selection. 2096 * 2097 * Unconnected sends, when !(so->so_state & SS_ISCONNECTED) and the 2098 * destination address is supplied, create a temporary connection for 2099 * the run time of the function (see call to unp_connectat() above and 2100 * to unp_disconnect() below). We distinguish them by condition of 2101 * (addr != NULL). We intentionally avoid adding 'bool connected' for 2102 * that condition, since, again, through the run time of this code we 2103 * are always connected. For such "unconnected" sends, the destination 2104 * buffer would be the receive buffer of destination socket so2. 2105 * 2106 * For connected sends, data lands on the send buffer of the sender's 2107 * socket "so". Then, if we just added the very first datagram 2108 * on this send buffer, we need to add the send buffer on to the 2109 * receiving socket's buffer list. We put ourselves on top of the 2110 * list. Such logic gives infrequent senders priority over frequent 2111 * senders. 2112 * 2113 * Note on byte count management. As long as event methods kevent(2), 2114 * select(2) are not protocol specific (yet), we need to maintain 2115 * meaningful values on the receive buffer. So, the receive buffer 2116 * would accumulate counters from all connected buffers potentially 2117 * having sb_ccc > sb_hiwat or sb_mbcnt > sb_mbmax. 2118 */ 2119 so2 = unp2->unp_socket; 2120 sb = (addr == NULL) ? &so->so_snd : &so2->so_rcv; 2121 SOCK_RECVBUF_LOCK(so2); 2122 if (uipc_dgram_sbspace(sb, cc + ctl, mbcnt)) { 2123 if (addr == NULL && STAILQ_EMPTY(&sb->uxdg_mb)) 2124 TAILQ_INSERT_HEAD(&so2->so_rcv.uxdg_conns, &so->so_snd, 2125 uxdg_clist); 2126 STAILQ_INSERT_TAIL(&sb->uxdg_mb, f, m_stailqpkt); 2127 sb->uxdg_cc += cc + ctl; 2128 sb->uxdg_ctl += ctl; 2129 sb->uxdg_mbcnt += mbcnt; 2130 so2->so_rcv.sb_acc += cc + ctl; 2131 so2->so_rcv.sb_ccc += cc + ctl; 2132 so2->so_rcv.sb_ctl += ctl; 2133 so2->so_rcv.sb_mbcnt += mbcnt; 2134 sorwakeup_locked(so2); 2135 f = NULL; 2136 } else { 2137 soroverflow_locked(so2); 2138 error = ENOBUFS; 2139 if (f->m_next->m_type == MT_CONTROL) { 2140 STAILQ_FIRST(&cmc.mc_q) = f->m_next; 2141 f->m_next = NULL; 2142 } 2143 } 2144 2145 if (addr != NULL) 2146 unp_disconnect(unp, unp2); 2147 else 2148 unp_pcb_unlock_pair(unp, unp2); 2149 2150 td->td_ru.ru_msgsnd++; 2151 2152 out3: 2153 SOCK_IO_SEND_UNLOCK(so); 2154 out2: 2155 if (!mc_empty(&cmc)) 2156 unp_scan(mc_first(&cmc), unp_freerights); 2157 out: 2158 if (f) 2159 m_freem(f); 2160 mc_freem(&cmc); 2161 if (m) 2162 m_freem(m); 2163 2164 return (error); 2165 } 2166 2167 /* 2168 * PF_UNIX/SOCK_DGRAM receive with MSG_PEEK. 2169 * The mbuf has already been unlinked from the uxdg_mb of socket buffer 2170 * and needs to be linked onto uxdg_peeked of receive socket buffer. 2171 */ 2172 static int 2173 uipc_peek_dgram(struct socket *so, struct mbuf *m, struct sockaddr **psa, 2174 struct uio *uio, struct mbuf **controlp, int *flagsp) 2175 { 2176 ssize_t len = 0; 2177 int error; 2178 2179 so->so_rcv.uxdg_peeked = m; 2180 so->so_rcv.uxdg_cc += m->m_pkthdr.len; 2181 so->so_rcv.uxdg_ctl += m->m_pkthdr.ctllen; 2182 so->so_rcv.uxdg_mbcnt += m->m_pkthdr.memlen; 2183 SOCK_RECVBUF_UNLOCK(so); 2184 2185 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2186 if (psa != NULL) 2187 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2188 2189 m = m->m_next; 2190 KASSERT(m, ("%s: no data or control after soname", __func__)); 2191 2192 /* 2193 * With MSG_PEEK the control isn't executed, just copied. 2194 */ 2195 while (m != NULL && m->m_type == MT_CONTROL) { 2196 if (controlp != NULL) { 2197 *controlp = m_copym(m, 0, m->m_len, M_WAITOK); 2198 controlp = &(*controlp)->m_next; 2199 } 2200 m = m->m_next; 2201 } 2202 KASSERT(m == NULL || m->m_type == MT_DATA, 2203 ("%s: not MT_DATA mbuf %p", __func__, m)); 2204 while (m != NULL && uio->uio_resid > 0) { 2205 len = uio->uio_resid; 2206 if (len > m->m_len) 2207 len = m->m_len; 2208 error = uiomove(mtod(m, char *), (int)len, uio); 2209 if (error) { 2210 SOCK_IO_RECV_UNLOCK(so); 2211 return (error); 2212 } 2213 if (len == m->m_len) 2214 m = m->m_next; 2215 } 2216 SOCK_IO_RECV_UNLOCK(so); 2217 2218 if (flagsp != NULL) { 2219 if (m != NULL) { 2220 if (*flagsp & MSG_TRUNC) { 2221 /* Report real length of the packet */ 2222 uio->uio_resid -= m_length(m, NULL) - len; 2223 } 2224 *flagsp |= MSG_TRUNC; 2225 } else 2226 *flagsp &= ~MSG_TRUNC; 2227 } 2228 2229 return (0); 2230 } 2231 2232 /* 2233 * PF_UNIX/SOCK_DGRAM receive 2234 */ 2235 static int 2236 uipc_soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2237 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2238 { 2239 struct sockbuf *sb = NULL; 2240 struct mbuf *m; 2241 int flags, error; 2242 ssize_t len = 0; 2243 bool nonblock; 2244 2245 MPASS(mp0 == NULL); 2246 2247 if (psa != NULL) 2248 *psa = NULL; 2249 if (controlp != NULL) 2250 *controlp = NULL; 2251 2252 flags = flagsp != NULL ? *flagsp : 0; 2253 nonblock = (so->so_state & SS_NBIO) || 2254 (flags & (MSG_DONTWAIT | MSG_NBIO)); 2255 2256 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 2257 if (__predict_false(error)) 2258 return (error); 2259 2260 /* 2261 * Loop blocking while waiting for a datagram. Prioritize connected 2262 * peers over unconnected sends. Set sb to selected socket buffer 2263 * containing an mbuf on exit from the wait loop. A datagram that 2264 * had already been peeked at has top priority. 2265 */ 2266 SOCK_RECVBUF_LOCK(so); 2267 while ((m = so->so_rcv.uxdg_peeked) == NULL && 2268 (sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) == NULL && 2269 (m = STAILQ_FIRST(&so->so_rcv.uxdg_mb)) == NULL) { 2270 if (so->so_error) { 2271 error = so->so_error; 2272 if (!(flags & MSG_PEEK)) 2273 so->so_error = 0; 2274 SOCK_RECVBUF_UNLOCK(so); 2275 SOCK_IO_RECV_UNLOCK(so); 2276 return (error); 2277 } 2278 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2279 uio->uio_resid == 0) { 2280 SOCK_RECVBUF_UNLOCK(so); 2281 SOCK_IO_RECV_UNLOCK(so); 2282 return (0); 2283 } 2284 if (nonblock) { 2285 SOCK_RECVBUF_UNLOCK(so); 2286 SOCK_IO_RECV_UNLOCK(so); 2287 return (EWOULDBLOCK); 2288 } 2289 error = sbwait(so, SO_RCV); 2290 if (error) { 2291 SOCK_RECVBUF_UNLOCK(so); 2292 SOCK_IO_RECV_UNLOCK(so); 2293 return (error); 2294 } 2295 } 2296 2297 if (sb == NULL) 2298 sb = &so->so_rcv; 2299 else if (m == NULL) 2300 m = STAILQ_FIRST(&sb->uxdg_mb); 2301 else 2302 MPASS(m == so->so_rcv.uxdg_peeked); 2303 2304 MPASS(sb->uxdg_cc > 0); 2305 M_ASSERTPKTHDR(m); 2306 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2307 2308 if (uio->uio_td) 2309 uio->uio_td->td_ru.ru_msgrcv++; 2310 2311 if (__predict_true(m != so->so_rcv.uxdg_peeked)) { 2312 STAILQ_REMOVE_HEAD(&sb->uxdg_mb, m_stailqpkt); 2313 if (STAILQ_EMPTY(&sb->uxdg_mb) && sb != &so->so_rcv) 2314 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 2315 } else 2316 so->so_rcv.uxdg_peeked = NULL; 2317 2318 sb->uxdg_cc -= m->m_pkthdr.len; 2319 sb->uxdg_ctl -= m->m_pkthdr.ctllen; 2320 sb->uxdg_mbcnt -= m->m_pkthdr.memlen; 2321 2322 if (__predict_false(flags & MSG_PEEK)) 2323 return (uipc_peek_dgram(so, m, psa, uio, controlp, flagsp)); 2324 2325 so->so_rcv.sb_acc -= m->m_pkthdr.len; 2326 so->so_rcv.sb_ccc -= m->m_pkthdr.len; 2327 so->so_rcv.sb_ctl -= m->m_pkthdr.ctllen; 2328 so->so_rcv.sb_mbcnt -= m->m_pkthdr.memlen; 2329 SOCK_RECVBUF_UNLOCK(so); 2330 2331 if (psa != NULL) 2332 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2333 m = m_free(m); 2334 KASSERT(m, ("%s: no data or control after soname", __func__)); 2335 2336 /* 2337 * Packet to copyout() is now in 'm' and it is disconnected from the 2338 * queue. 2339 * 2340 * Process one or more MT_CONTROL mbufs present before any data mbufs 2341 * in the first mbuf chain on the socket buffer. We call into the 2342 * unp_externalize() to perform externalization (or freeing if 2343 * controlp == NULL). In some cases there can be only MT_CONTROL mbufs 2344 * without MT_DATA mbufs. 2345 */ 2346 while (m != NULL && m->m_type == MT_CONTROL) { 2347 error = unp_externalize(m, controlp, flags); 2348 m = m_free(m); 2349 if (error != 0) { 2350 SOCK_IO_RECV_UNLOCK(so); 2351 unp_scan(m, unp_freerights); 2352 m_freem(m); 2353 return (error); 2354 } 2355 if (controlp != NULL) { 2356 while (*controlp != NULL) 2357 controlp = &(*controlp)->m_next; 2358 } 2359 } 2360 KASSERT(m == NULL || m->m_type == MT_DATA, 2361 ("%s: not MT_DATA mbuf %p", __func__, m)); 2362 while (m != NULL && uio->uio_resid > 0) { 2363 len = uio->uio_resid; 2364 if (len > m->m_len) 2365 len = m->m_len; 2366 error = uiomove(mtod(m, char *), (int)len, uio); 2367 if (error) { 2368 SOCK_IO_RECV_UNLOCK(so); 2369 m_freem(m); 2370 return (error); 2371 } 2372 if (len == m->m_len) 2373 m = m_free(m); 2374 else { 2375 m->m_data += len; 2376 m->m_len -= len; 2377 } 2378 } 2379 SOCK_IO_RECV_UNLOCK(so); 2380 2381 if (m != NULL) { 2382 if (flagsp != NULL) { 2383 if (flags & MSG_TRUNC) { 2384 /* Report real length of the packet */ 2385 uio->uio_resid -= m_length(m, NULL); 2386 } 2387 *flagsp |= MSG_TRUNC; 2388 } 2389 m_freem(m); 2390 } else if (flagsp != NULL) 2391 *flagsp &= ~MSG_TRUNC; 2392 2393 return (0); 2394 } 2395 2396 static int 2397 uipc_sendfile_wait(struct socket *so, off_t need, int *space) 2398 { 2399 struct unpcb *unp2; 2400 struct socket *so2; 2401 struct sockbuf *sb; 2402 bool nonblock, sockref; 2403 int error; 2404 2405 MPASS(so->so_type == SOCK_STREAM); 2406 MPASS(need > 0); 2407 MPASS(space != NULL); 2408 2409 nonblock = so->so_state & SS_NBIO; 2410 sockref = false; 2411 2412 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) 2413 return (ENOTCONN); 2414 2415 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2416 return (error); 2417 2418 so2 = unp2->unp_socket; 2419 sb = &so2->so_rcv; 2420 SOCK_RECVBUF_LOCK(so2); 2421 UNP_PCB_UNLOCK(unp2); 2422 while ((*space = uipc_stream_sbspace(sb)) < need && 2423 (*space < so->so_snd.sb_hiwat / 2)) { 2424 UIPC_STREAM_SBCHECK(sb); 2425 if (nonblock) { 2426 SOCK_RECVBUF_UNLOCK(so2); 2427 return (EAGAIN); 2428 } 2429 if (!sockref) 2430 soref(so2); 2431 error = uipc_stream_sbwait(so2, so->so_snd.sb_timeo); 2432 if (error == 0 && 2433 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) 2434 error = EPIPE; 2435 if (error) { 2436 SOCK_RECVBUF_UNLOCK(so2); 2437 sorele(so2); 2438 return (error); 2439 } 2440 } 2441 UIPC_STREAM_SBCHECK(sb); 2442 SOCK_RECVBUF_UNLOCK(so2); 2443 if (sockref) 2444 sorele(so2); 2445 2446 return (0); 2447 } 2448 2449 /* 2450 * Although this is a pr_send method, for unix(4) it is called only via 2451 * sendfile(2) path. This means we can be sure that mbufs are clear of 2452 * any extra flags and don't require any conditioning. 2453 */ 2454 static int 2455 uipc_sendfile(struct socket *so, int flags, struct mbuf *m, 2456 struct sockaddr *from, struct mbuf *control, struct thread *td) 2457 { 2458 struct mchain mc; 2459 struct unpcb *unp2; 2460 struct socket *so2; 2461 struct sockbuf *sb; 2462 bool notready, wakeup; 2463 int error; 2464 2465 MPASS(so->so_type == SOCK_STREAM); 2466 MPASS(from == NULL && control == NULL); 2467 KASSERT(!(m->m_flags & M_EXTPG), 2468 ("unix(4): TLS sendfile(2) not supported")); 2469 2470 notready = flags & PRUS_NOTREADY; 2471 2472 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) { 2473 error = ENOTCONN; 2474 goto out; 2475 } 2476 2477 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2478 goto out; 2479 2480 mc_init_m(&mc, m); 2481 2482 so2 = unp2->unp_socket; 2483 sb = &so2->so_rcv; 2484 SOCK_RECVBUF_LOCK(so2); 2485 UNP_PCB_UNLOCK(unp2); 2486 UIPC_STREAM_SBCHECK(sb); 2487 sb->sb_ccc += mc.mc_len; 2488 sb->sb_mbcnt += mc.mc_mlen; 2489 if (sb->uxst_fnrdy == NULL) { 2490 if (notready) { 2491 wakeup = false; 2492 STAILQ_FOREACH(m, &mc.mc_q, m_stailq) { 2493 if (m->m_flags & M_NOTREADY) { 2494 sb->uxst_fnrdy = m; 2495 break; 2496 } else { 2497 sb->sb_acc += m->m_len; 2498 wakeup = true; 2499 } 2500 } 2501 } else { 2502 wakeup = true; 2503 sb->sb_acc += mc.mc_len; 2504 } 2505 } else { 2506 wakeup = false; 2507 } 2508 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 2509 UIPC_STREAM_SBCHECK(sb); 2510 if (wakeup) 2511 sorwakeup_locked(so2); 2512 else 2513 SOCK_RECVBUF_UNLOCK(so2); 2514 2515 return (0); 2516 out: 2517 /* 2518 * In case of not ready data, uipc_ready() is responsible 2519 * for freeing memory. 2520 */ 2521 if (m != NULL && !notready) 2522 m_freem(m); 2523 2524 return (error); 2525 } 2526 2527 static int 2528 uipc_sbready(struct sockbuf *sb, struct mbuf *m, int count) 2529 { 2530 bool blocker; 2531 2532 /* assert locked */ 2533 2534 blocker = (sb->uxst_fnrdy == m); 2535 STAILQ_FOREACH_FROM(m, &sb->uxst_mbq, m_stailq) { 2536 if (count > 0) { 2537 MPASS(m->m_flags & M_NOTREADY); 2538 m->m_flags &= ~M_NOTREADY; 2539 if (blocker) 2540 sb->sb_acc += m->m_len; 2541 count--; 2542 } else if (m->m_flags & M_NOTREADY) 2543 break; 2544 else if (blocker) 2545 sb->sb_acc += m->m_len; 2546 } 2547 if (blocker) { 2548 sb->uxst_fnrdy = m; 2549 return (0); 2550 } else 2551 return (EINPROGRESS); 2552 } 2553 2554 static bool 2555 uipc_ready_scan(struct socket *so, struct mbuf *m, int count, int *errorp) 2556 { 2557 struct mbuf *mb; 2558 struct sockbuf *sb; 2559 2560 SOCK_LOCK(so); 2561 if (SOLISTENING(so)) { 2562 SOCK_UNLOCK(so); 2563 return (false); 2564 } 2565 mb = NULL; 2566 sb = &so->so_rcv; 2567 SOCK_RECVBUF_LOCK(so); 2568 if (sb->uxst_fnrdy != NULL) { 2569 STAILQ_FOREACH(mb, &sb->uxst_mbq, m_stailq) { 2570 if (mb == m) { 2571 *errorp = uipc_sbready(sb, m, count); 2572 break; 2573 } 2574 } 2575 } 2576 SOCK_RECVBUF_UNLOCK(so); 2577 SOCK_UNLOCK(so); 2578 return (mb != NULL); 2579 } 2580 2581 static int 2582 uipc_ready(struct socket *so, struct mbuf *m, int count) 2583 { 2584 struct unpcb *unp, *unp2; 2585 int error; 2586 2587 MPASS(so->so_type == SOCK_STREAM); 2588 2589 if (__predict_true(uipc_lock_peer(so, &unp2) == 0)) { 2590 struct socket *so2; 2591 struct sockbuf *sb; 2592 2593 so2 = unp2->unp_socket; 2594 sb = &so2->so_rcv; 2595 SOCK_RECVBUF_LOCK(so2); 2596 UNP_PCB_UNLOCK(unp2); 2597 UIPC_STREAM_SBCHECK(sb); 2598 error = uipc_sbready(sb, m, count); 2599 UIPC_STREAM_SBCHECK(sb); 2600 if (error == 0) 2601 sorwakeup_locked(so2); 2602 else 2603 SOCK_RECVBUF_UNLOCK(so2); 2604 } else { 2605 /* 2606 * The receiving socket has been disconnected, but may still 2607 * be valid. In this case, the not-ready mbufs are still 2608 * present in its socket buffer, so perform an exhaustive 2609 * search before giving up and freeing the mbufs. 2610 */ 2611 UNP_LINK_RLOCK(); 2612 LIST_FOREACH(unp, &unp_shead, unp_link) { 2613 if (uipc_ready_scan(unp->unp_socket, m, count, &error)) 2614 break; 2615 } 2616 UNP_LINK_RUNLOCK(); 2617 2618 if (unp == NULL) { 2619 for (int i = 0; i < count; i++) 2620 m = m_free(m); 2621 return (ECONNRESET); 2622 } 2623 } 2624 return (error); 2625 } 2626 2627 static int 2628 uipc_sense(struct socket *so, struct stat *sb) 2629 { 2630 struct unpcb *unp; 2631 2632 unp = sotounpcb(so); 2633 KASSERT(unp != NULL, ("uipc_sense: unp == NULL")); 2634 2635 sb->st_blksize = so->so_snd.sb_hiwat; 2636 sb->st_dev = NODEV; 2637 sb->st_ino = unp->unp_ino; 2638 return (0); 2639 } 2640 2641 static int 2642 uipc_shutdown(struct socket *so, enum shutdown_how how) 2643 { 2644 struct unpcb *unp = sotounpcb(so); 2645 int error; 2646 2647 SOCK_LOCK(so); 2648 if (SOLISTENING(so)) { 2649 if (how != SHUT_WR) { 2650 so->so_error = ECONNABORTED; 2651 solisten_wakeup(so); /* unlocks so */ 2652 } else 2653 SOCK_UNLOCK(so); 2654 return (ENOTCONN); 2655 } else if ((so->so_state & 2656 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2657 /* 2658 * POSIX mandates us to just return ENOTCONN when shutdown(2) is 2659 * invoked on a datagram sockets, however historically we would 2660 * actually tear socket down. This is known to be leveraged by 2661 * some applications to unblock process waiting in recv(2) by 2662 * other process that it shares that socket with. Try to meet 2663 * both backward-compatibility and POSIX requirements by forcing 2664 * ENOTCONN but still flushing buffers and performing wakeup(9). 2665 * 2666 * XXXGL: it remains unknown what applications expect this 2667 * behavior and is this isolated to unix/dgram or inet/dgram or 2668 * both. See: D10351, D3039. 2669 */ 2670 error = ENOTCONN; 2671 if (so->so_type != SOCK_DGRAM) { 2672 SOCK_UNLOCK(so); 2673 return (error); 2674 } 2675 } else 2676 error = 0; 2677 SOCK_UNLOCK(so); 2678 2679 switch (how) { 2680 case SHUT_RD: 2681 if (so->so_type == SOCK_DGRAM) 2682 socantrcvmore(so); 2683 else 2684 uipc_cantrcvmore(so); 2685 unp_dispose(so); 2686 break; 2687 case SHUT_RDWR: 2688 if (so->so_type == SOCK_DGRAM) 2689 socantrcvmore(so); 2690 else 2691 uipc_cantrcvmore(so); 2692 unp_dispose(so); 2693 /* FALLTHROUGH */ 2694 case SHUT_WR: 2695 if (so->so_type == SOCK_DGRAM) { 2696 socantsendmore(so); 2697 } else { 2698 UNP_PCB_LOCK(unp); 2699 if (unp->unp_conn != NULL) 2700 uipc_cantrcvmore(unp->unp_conn->unp_socket); 2701 UNP_PCB_UNLOCK(unp); 2702 } 2703 } 2704 wakeup(&so->so_timeo); 2705 2706 return (error); 2707 } 2708 2709 static int 2710 uipc_sockaddr(struct socket *so, struct sockaddr *ret) 2711 { 2712 struct unpcb *unp; 2713 const struct sockaddr *sa; 2714 2715 unp = sotounpcb(so); 2716 KASSERT(unp != NULL, ("uipc_sockaddr: unp == NULL")); 2717 2718 UNP_PCB_LOCK(unp); 2719 if (unp->unp_addr != NULL) 2720 sa = (struct sockaddr *) unp->unp_addr; 2721 else 2722 sa = &sun_noname; 2723 bcopy(sa, ret, sa->sa_len); 2724 UNP_PCB_UNLOCK(unp); 2725 return (0); 2726 } 2727 2728 static int 2729 uipc_ctloutput(struct socket *so, struct sockopt *sopt) 2730 { 2731 struct unpcb *unp; 2732 struct xucred xu; 2733 int error, optval; 2734 2735 if (sopt->sopt_level != SOL_LOCAL) 2736 return (EINVAL); 2737 2738 unp = sotounpcb(so); 2739 KASSERT(unp != NULL, ("uipc_ctloutput: unp == NULL")); 2740 error = 0; 2741 switch (sopt->sopt_dir) { 2742 case SOPT_GET: 2743 switch (sopt->sopt_name) { 2744 case LOCAL_PEERCRED: 2745 UNP_PCB_LOCK(unp); 2746 if (unp->unp_flags & UNP_HAVEPC) 2747 xu = unp->unp_peercred; 2748 else { 2749 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2750 error = ENOTCONN; 2751 else 2752 error = EINVAL; 2753 } 2754 UNP_PCB_UNLOCK(unp); 2755 if (error == 0) 2756 error = sooptcopyout(sopt, &xu, sizeof(xu)); 2757 break; 2758 2759 case LOCAL_CREDS: 2760 /* Unlocked read. */ 2761 optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0; 2762 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2763 break; 2764 2765 case LOCAL_CREDS_PERSISTENT: 2766 /* Unlocked read. */ 2767 optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0; 2768 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2769 break; 2770 2771 default: 2772 error = EOPNOTSUPP; 2773 break; 2774 } 2775 break; 2776 2777 case SOPT_SET: 2778 switch (sopt->sopt_name) { 2779 case LOCAL_CREDS: 2780 case LOCAL_CREDS_PERSISTENT: 2781 error = sooptcopyin(sopt, &optval, sizeof(optval), 2782 sizeof(optval)); 2783 if (error) 2784 break; 2785 2786 #define OPTSET(bit, exclusive) do { \ 2787 UNP_PCB_LOCK(unp); \ 2788 if (optval) { \ 2789 if ((unp->unp_flags & (exclusive)) != 0) { \ 2790 UNP_PCB_UNLOCK(unp); \ 2791 error = EINVAL; \ 2792 break; \ 2793 } \ 2794 unp->unp_flags |= (bit); \ 2795 } else \ 2796 unp->unp_flags &= ~(bit); \ 2797 UNP_PCB_UNLOCK(unp); \ 2798 } while (0) 2799 2800 switch (sopt->sopt_name) { 2801 case LOCAL_CREDS: 2802 OPTSET(UNP_WANTCRED_ONESHOT, UNP_WANTCRED_ALWAYS); 2803 break; 2804 2805 case LOCAL_CREDS_PERSISTENT: 2806 OPTSET(UNP_WANTCRED_ALWAYS, UNP_WANTCRED_ONESHOT); 2807 break; 2808 2809 default: 2810 break; 2811 } 2812 break; 2813 #undef OPTSET 2814 default: 2815 error = ENOPROTOOPT; 2816 break; 2817 } 2818 break; 2819 2820 default: 2821 error = EOPNOTSUPP; 2822 break; 2823 } 2824 return (error); 2825 } 2826 2827 static int 2828 unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 2829 { 2830 2831 return (unp_connectat(AT_FDCWD, so, nam, td, false)); 2832 } 2833 2834 static int 2835 unp_connectat(int fd, struct socket *so, struct sockaddr *nam, 2836 struct thread *td, bool return_locked) 2837 { 2838 struct mtx *vplock; 2839 struct sockaddr_un *soun; 2840 struct vnode *vp; 2841 struct socket *so2; 2842 struct unpcb *unp, *unp2, *unp3; 2843 struct nameidata nd; 2844 char buf[SOCK_MAXADDRLEN]; 2845 struct sockaddr *sa; 2846 cap_rights_t rights; 2847 int error, len; 2848 bool connreq; 2849 2850 CURVNET_ASSERT_SET(); 2851 2852 if (nam->sa_family != AF_UNIX) 2853 return (EAFNOSUPPORT); 2854 if (nam->sa_len > sizeof(struct sockaddr_un)) 2855 return (EINVAL); 2856 len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); 2857 if (len <= 0) 2858 return (EINVAL); 2859 soun = (struct sockaddr_un *)nam; 2860 bcopy(soun->sun_path, buf, len); 2861 buf[len] = 0; 2862 2863 error = 0; 2864 unp = sotounpcb(so); 2865 UNP_PCB_LOCK(unp); 2866 for (;;) { 2867 /* 2868 * Wait for connection state to stabilize. If a connection 2869 * already exists, give up. For datagram sockets, which permit 2870 * multiple consecutive connect(2) calls, upper layers are 2871 * responsible for disconnecting in advance of a subsequent 2872 * connect(2), but this is not synchronized with PCB connection 2873 * state. 2874 * 2875 * Also make sure that no threads are currently attempting to 2876 * lock the peer socket, to ensure that unp_conn cannot 2877 * transition between two valid sockets while locks are dropped. 2878 */ 2879 if (SOLISTENING(so)) 2880 error = EOPNOTSUPP; 2881 else if (unp->unp_conn != NULL) 2882 error = EISCONN; 2883 else if ((unp->unp_flags & UNP_CONNECTING) != 0) { 2884 error = EALREADY; 2885 } 2886 if (error != 0) { 2887 UNP_PCB_UNLOCK(unp); 2888 return (error); 2889 } 2890 if (unp->unp_pairbusy > 0) { 2891 unp->unp_flags |= UNP_WAITING; 2892 mtx_sleep(unp, UNP_PCB_LOCKPTR(unp), 0, "unpeer", 0); 2893 continue; 2894 } 2895 break; 2896 } 2897 unp->unp_flags |= UNP_CONNECTING; 2898 UNP_PCB_UNLOCK(unp); 2899 2900 connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0; 2901 if (connreq) 2902 sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK); 2903 else 2904 sa = NULL; 2905 NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, 2906 UIO_SYSSPACE, buf, fd, cap_rights_init_one(&rights, CAP_CONNECTAT)); 2907 error = namei(&nd); 2908 if (error) 2909 vp = NULL; 2910 else 2911 vp = nd.ni_vp; 2912 ASSERT_VOP_LOCKED(vp, "unp_connect"); 2913 if (error) 2914 goto bad; 2915 NDFREE_PNBUF(&nd); 2916 2917 if (vp->v_type != VSOCK) { 2918 error = ENOTSOCK; 2919 goto bad; 2920 } 2921 #ifdef MAC 2922 error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD); 2923 if (error) 2924 goto bad; 2925 #endif 2926 error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td); 2927 if (error) 2928 goto bad; 2929 2930 unp = sotounpcb(so); 2931 KASSERT(unp != NULL, ("unp_connect: unp == NULL")); 2932 2933 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 2934 mtx_lock(vplock); 2935 VOP_UNP_CONNECT(vp, &unp2); 2936 if (unp2 == NULL) { 2937 error = ECONNREFUSED; 2938 goto bad2; 2939 } 2940 so2 = unp2->unp_socket; 2941 if (so->so_type != so2->so_type) { 2942 error = EPROTOTYPE; 2943 goto bad2; 2944 } 2945 if (connreq) { 2946 if (SOLISTENING(so2)) 2947 so2 = solisten_clone(so2); 2948 else 2949 so2 = NULL; 2950 if (so2 == NULL) { 2951 error = ECONNREFUSED; 2952 goto bad2; 2953 } 2954 if ((error = uipc_attach(so2, 0, NULL)) != 0) { 2955 sodealloc(so2); 2956 goto bad2; 2957 } 2958 unp3 = sotounpcb(so2); 2959 unp_pcb_lock_pair(unp2, unp3); 2960 if (unp2->unp_addr != NULL) { 2961 bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); 2962 unp3->unp_addr = (struct sockaddr_un *) sa; 2963 sa = NULL; 2964 } 2965 2966 unp_copy_peercred(td, unp3, unp, unp2); 2967 2968 UNP_PCB_UNLOCK(unp2); 2969 unp2 = unp3; 2970 2971 /* 2972 * It is safe to block on the PCB lock here since unp2 is 2973 * nascent and cannot be connected to any other sockets. 2974 */ 2975 UNP_PCB_LOCK(unp); 2976 #ifdef MAC 2977 mac_socketpeer_set_from_socket(so, so2); 2978 mac_socketpeer_set_from_socket(so2, so); 2979 #endif 2980 } else { 2981 unp_pcb_lock_pair(unp, unp2); 2982 } 2983 KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && 2984 sotounpcb(so2) == unp2, 2985 ("%s: unp2 %p so2 %p", __func__, unp2, so2)); 2986 unp_connect2(so, so2, connreq); 2987 if (connreq) 2988 (void)solisten_enqueue(so2, SS_ISCONNECTED); 2989 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 2990 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 2991 unp->unp_flags &= ~UNP_CONNECTING; 2992 if (!return_locked) 2993 unp_pcb_unlock_pair(unp, unp2); 2994 bad2: 2995 mtx_unlock(vplock); 2996 bad: 2997 if (vp != NULL) { 2998 /* 2999 * If we are returning locked (called via uipc_sosend_dgram()), 3000 * we need to be sure that vput() won't sleep. This is 3001 * guaranteed by VOP_UNP_CONNECT() call above and unp2 lock. 3002 * SOCK_STREAM/SEQPACKET can't request return_locked (yet). 3003 */ 3004 MPASS(!(return_locked && connreq)); 3005 vput(vp); 3006 } 3007 free(sa, M_SONAME); 3008 if (__predict_false(error)) { 3009 UNP_PCB_LOCK(unp); 3010 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 3011 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 3012 unp->unp_flags &= ~UNP_CONNECTING; 3013 UNP_PCB_UNLOCK(unp); 3014 } 3015 return (error); 3016 } 3017 3018 /* 3019 * Set socket peer credentials at connection time. 3020 * 3021 * The client's PCB credentials are copied from its process structure. The 3022 * server's PCB credentials are copied from the socket on which it called 3023 * listen(2). uipc_listen cached that process's credentials at the time. 3024 */ 3025 void 3026 unp_copy_peercred(struct thread *td, struct unpcb *client_unp, 3027 struct unpcb *server_unp, struct unpcb *listen_unp) 3028 { 3029 cru2xt(td, &client_unp->unp_peercred); 3030 client_unp->unp_flags |= UNP_HAVEPC; 3031 3032 memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred, 3033 sizeof(server_unp->unp_peercred)); 3034 server_unp->unp_flags |= UNP_HAVEPC; 3035 client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK); 3036 } 3037 3038 /* 3039 * unix/stream & unix/seqpacket version of soisconnected(). 3040 * 3041 * The crucial thing we are doing here is setting up the uxst_peer linkage, 3042 * holding unp and receive buffer locks of the both sockets. The disconnect 3043 * procedure does the same. This gives as a safe way to access the peer in the 3044 * send(2) and recv(2) during the socket lifetime. 3045 * 3046 * The less important thing is event notification of the fact that a socket is 3047 * now connected. It is unusual for a software to put a socket into event 3048 * mechanism before connect(2), but is supposed to be supported. Note that 3049 * there can not be any sleeping I/O on the socket, yet, only presence in the 3050 * select/poll/kevent. 3051 * 3052 * This function can be called via two call paths: 3053 * 1) socketpair(2) - in this case socket has not been yet reported to userland 3054 * and just can't have any event notifications mechanisms set up. The 3055 * 'wakeup' boolean is always false. 3056 * 2) connect(2) of existing socket to a recent clone of a listener: 3057 * 2.1) Socket that connect(2)s will have 'wakeup' true. An application 3058 * could have already put it into event mechanism, is it shall be 3059 * reported as readable and as writable. 3060 * 2.2) Socket that was just cloned with solisten_clone(). Same as 1). 3061 */ 3062 static void 3063 unp_soisconnected(struct socket *so, bool wakeup) 3064 { 3065 struct socket *so2 = sotounpcb(so)->unp_conn->unp_socket; 3066 struct sockbuf *sb; 3067 3068 SOCK_LOCK_ASSERT(so); 3069 UNP_PCB_LOCK_ASSERT(sotounpcb(so)); 3070 UNP_PCB_LOCK_ASSERT(sotounpcb(so2)); 3071 SOCK_RECVBUF_LOCK_ASSERT(so); 3072 SOCK_RECVBUF_LOCK_ASSERT(so2); 3073 3074 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3075 MPASS((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 3076 SS_ISDISCONNECTING)) == 0); 3077 MPASS(so->so_qstate == SQ_NONE); 3078 3079 so->so_state &= ~SS_ISDISCONNECTED; 3080 so->so_state |= SS_ISCONNECTED; 3081 3082 sb = &so2->so_rcv; 3083 sb->uxst_peer = so; 3084 3085 if (wakeup) { 3086 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); 3087 sb = &so->so_rcv; 3088 selwakeuppri(sb->sb_sel, PSOCK); 3089 SOCK_SENDBUF_LOCK_ASSERT(so); 3090 sb = &so->so_snd; 3091 selwakeuppri(sb->sb_sel, PSOCK); 3092 SOCK_SENDBUF_UNLOCK(so); 3093 } 3094 } 3095 3096 static void 3097 unp_connect2(struct socket *so, struct socket *so2, bool wakeup) 3098 { 3099 struct unpcb *unp; 3100 struct unpcb *unp2; 3101 3102 MPASS(so2->so_type == so->so_type); 3103 unp = sotounpcb(so); 3104 KASSERT(unp != NULL, ("unp_connect2: unp == NULL")); 3105 unp2 = sotounpcb(so2); 3106 KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL")); 3107 3108 UNP_PCB_LOCK_ASSERT(unp); 3109 UNP_PCB_LOCK_ASSERT(unp2); 3110 KASSERT(unp->unp_conn == NULL, 3111 ("%s: socket %p is already connected", __func__, unp)); 3112 3113 unp->unp_conn = unp2; 3114 unp_pcb_hold(unp2); 3115 unp_pcb_hold(unp); 3116 switch (so->so_type) { 3117 case SOCK_DGRAM: 3118 UNP_REF_LIST_LOCK(); 3119 LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); 3120 UNP_REF_LIST_UNLOCK(); 3121 soisconnected(so); 3122 break; 3123 3124 case SOCK_STREAM: 3125 case SOCK_SEQPACKET: 3126 KASSERT(unp2->unp_conn == NULL, 3127 ("%s: socket %p is already connected", __func__, unp2)); 3128 unp2->unp_conn = unp; 3129 SOCK_LOCK(so); 3130 SOCK_LOCK(so2); 3131 if (wakeup) /* Avoid LOR with receive buffer lock. */ 3132 SOCK_SENDBUF_LOCK(so); 3133 SOCK_RECVBUF_LOCK(so); 3134 SOCK_RECVBUF_LOCK(so2); 3135 unp_soisconnected(so, wakeup); /* Will unlock send buffer. */ 3136 unp_soisconnected(so2, false); 3137 SOCK_RECVBUF_UNLOCK(so); 3138 SOCK_RECVBUF_UNLOCK(so2); 3139 SOCK_UNLOCK(so); 3140 SOCK_UNLOCK(so2); 3141 break; 3142 3143 default: 3144 panic("unp_connect2"); 3145 } 3146 } 3147 3148 static void 3149 unp_soisdisconnected(struct socket *so) 3150 { 3151 SOCK_LOCK_ASSERT(so); 3152 SOCK_RECVBUF_LOCK_ASSERT(so); 3153 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3154 MPASS(!SOLISTENING(so)); 3155 MPASS((so->so_state & (SS_ISCONNECTING | SS_ISDISCONNECTING | 3156 SS_ISDISCONNECTED)) == 0); 3157 MPASS(so->so_state & SS_ISCONNECTED); 3158 3159 so->so_state |= SS_ISDISCONNECTED; 3160 so->so_state &= ~SS_ISCONNECTED; 3161 so->so_rcv.uxst_peer = NULL; 3162 socantrcvmore_locked(so); 3163 } 3164 3165 static void 3166 unp_disconnect(struct unpcb *unp, struct unpcb *unp2) 3167 { 3168 struct socket *so, *so2; 3169 struct mbuf *m = NULL; 3170 #ifdef INVARIANTS 3171 struct unpcb *unptmp; 3172 #endif 3173 3174 UNP_PCB_LOCK_ASSERT(unp); 3175 UNP_PCB_LOCK_ASSERT(unp2); 3176 KASSERT(unp->unp_conn == unp2, 3177 ("%s: unpcb %p is not connected to %p", __func__, unp, unp2)); 3178 3179 unp->unp_conn = NULL; 3180 so = unp->unp_socket; 3181 so2 = unp2->unp_socket; 3182 switch (unp->unp_socket->so_type) { 3183 case SOCK_DGRAM: 3184 /* 3185 * Remove our send socket buffer from the peer's receive buffer. 3186 * Move the data to the receive buffer only if it is empty. 3187 * This is a protection against a scenario where a peer 3188 * connects, floods and disconnects, effectively blocking 3189 * sendto() from unconnected sockets. 3190 */ 3191 SOCK_RECVBUF_LOCK(so2); 3192 if (!STAILQ_EMPTY(&so->so_snd.uxdg_mb)) { 3193 TAILQ_REMOVE(&so2->so_rcv.uxdg_conns, &so->so_snd, 3194 uxdg_clist); 3195 if (__predict_true((so2->so_rcv.sb_state & 3196 SBS_CANTRCVMORE) == 0) && 3197 STAILQ_EMPTY(&so2->so_rcv.uxdg_mb)) { 3198 STAILQ_CONCAT(&so2->so_rcv.uxdg_mb, 3199 &so->so_snd.uxdg_mb); 3200 so2->so_rcv.uxdg_cc += so->so_snd.uxdg_cc; 3201 so2->so_rcv.uxdg_ctl += so->so_snd.uxdg_ctl; 3202 so2->so_rcv.uxdg_mbcnt += so->so_snd.uxdg_mbcnt; 3203 } else { 3204 m = STAILQ_FIRST(&so->so_snd.uxdg_mb); 3205 STAILQ_INIT(&so->so_snd.uxdg_mb); 3206 so2->so_rcv.sb_acc -= so->so_snd.uxdg_cc; 3207 so2->so_rcv.sb_ccc -= so->so_snd.uxdg_cc; 3208 so2->so_rcv.sb_ctl -= so->so_snd.uxdg_ctl; 3209 so2->so_rcv.sb_mbcnt -= so->so_snd.uxdg_mbcnt; 3210 } 3211 /* Note: so may reconnect. */ 3212 so->so_snd.uxdg_cc = 0; 3213 so->so_snd.uxdg_ctl = 0; 3214 so->so_snd.uxdg_mbcnt = 0; 3215 } 3216 SOCK_RECVBUF_UNLOCK(so2); 3217 UNP_REF_LIST_LOCK(); 3218 #ifdef INVARIANTS 3219 LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) { 3220 if (unptmp == unp) 3221 break; 3222 } 3223 KASSERT(unptmp != NULL, 3224 ("%s: %p not found in reflist of %p", __func__, unp, unp2)); 3225 #endif 3226 LIST_REMOVE(unp, unp_reflink); 3227 UNP_REF_LIST_UNLOCK(); 3228 SOCK_LOCK(so); 3229 so->so_state &= ~SS_ISCONNECTED; 3230 SOCK_UNLOCK(so); 3231 break; 3232 3233 case SOCK_STREAM: 3234 case SOCK_SEQPACKET: 3235 SOCK_LOCK(so); 3236 SOCK_LOCK(so2); 3237 SOCK_RECVBUF_LOCK(so); 3238 SOCK_RECVBUF_LOCK(so2); 3239 unp_soisdisconnected(so); 3240 MPASS(unp2->unp_conn == unp); 3241 unp2->unp_conn = NULL; 3242 unp_soisdisconnected(so2); 3243 SOCK_UNLOCK(so); 3244 SOCK_UNLOCK(so2); 3245 break; 3246 } 3247 3248 if (unp == unp2) { 3249 unp_pcb_rele_notlast(unp); 3250 if (!unp_pcb_rele(unp)) 3251 UNP_PCB_UNLOCK(unp); 3252 } else { 3253 if (!unp_pcb_rele(unp)) 3254 UNP_PCB_UNLOCK(unp); 3255 if (!unp_pcb_rele(unp2)) 3256 UNP_PCB_UNLOCK(unp2); 3257 } 3258 3259 if (m != NULL) { 3260 unp_scan(m, unp_freerights); 3261 m_freemp(m); 3262 } 3263 } 3264 3265 /* 3266 * unp_pcblist() walks the global list of struct unpcb's to generate a 3267 * pointer list, bumping the refcount on each unpcb. It then copies them out 3268 * sequentially, validating the generation number on each to see if it has 3269 * been detached. All of this is necessary because copyout() may sleep on 3270 * disk I/O. 3271 */ 3272 static int 3273 unp_pcblist(SYSCTL_HANDLER_ARGS) 3274 { 3275 struct unpcb *unp, **unp_list; 3276 unp_gen_t gencnt; 3277 struct xunpgen *xug; 3278 struct unp_head *head; 3279 struct xunpcb *xu; 3280 u_int i; 3281 int error, n; 3282 3283 switch ((intptr_t)arg1) { 3284 case SOCK_STREAM: 3285 head = &unp_shead; 3286 break; 3287 3288 case SOCK_DGRAM: 3289 head = &unp_dhead; 3290 break; 3291 3292 case SOCK_SEQPACKET: 3293 head = &unp_sphead; 3294 break; 3295 3296 default: 3297 panic("unp_pcblist: arg1 %d", (int)(intptr_t)arg1); 3298 } 3299 3300 /* 3301 * The process of preparing the PCB list is too time-consuming and 3302 * resource-intensive to repeat twice on every request. 3303 */ 3304 if (req->oldptr == NULL) { 3305 n = unp_count; 3306 req->oldidx = 2 * (sizeof *xug) 3307 + (n + n/8) * sizeof(struct xunpcb); 3308 return (0); 3309 } 3310 3311 if (req->newptr != NULL) 3312 return (EPERM); 3313 3314 /* 3315 * OK, now we're committed to doing something. 3316 */ 3317 xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK | M_ZERO); 3318 UNP_LINK_RLOCK(); 3319 gencnt = unp_gencnt; 3320 n = unp_count; 3321 UNP_LINK_RUNLOCK(); 3322 3323 xug->xug_len = sizeof *xug; 3324 xug->xug_count = n; 3325 xug->xug_gen = gencnt; 3326 xug->xug_sogen = so_gencnt; 3327 error = SYSCTL_OUT(req, xug, sizeof *xug); 3328 if (error) { 3329 free(xug, M_TEMP); 3330 return (error); 3331 } 3332 3333 unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK); 3334 3335 UNP_LINK_RLOCK(); 3336 for (unp = LIST_FIRST(head), i = 0; unp && i < n; 3337 unp = LIST_NEXT(unp, unp_link)) { 3338 UNP_PCB_LOCK(unp); 3339 if (unp->unp_gencnt <= gencnt) { 3340 if (cr_cansee(req->td->td_ucred, 3341 unp->unp_socket->so_cred)) { 3342 UNP_PCB_UNLOCK(unp); 3343 continue; 3344 } 3345 unp_list[i++] = unp; 3346 unp_pcb_hold(unp); 3347 } 3348 UNP_PCB_UNLOCK(unp); 3349 } 3350 UNP_LINK_RUNLOCK(); 3351 n = i; /* In case we lost some during malloc. */ 3352 3353 error = 0; 3354 xu = malloc(sizeof(*xu), M_TEMP, M_WAITOK | M_ZERO); 3355 for (i = 0; i < n; i++) { 3356 unp = unp_list[i]; 3357 UNP_PCB_LOCK(unp); 3358 if (unp_pcb_rele(unp)) 3359 continue; 3360 3361 if (unp->unp_gencnt <= gencnt) { 3362 xu->xu_len = sizeof *xu; 3363 xu->xu_unpp = (uintptr_t)unp; 3364 /* 3365 * XXX - need more locking here to protect against 3366 * connect/disconnect races for SMP. 3367 */ 3368 if (unp->unp_addr != NULL) 3369 bcopy(unp->unp_addr, &xu->xu_addr, 3370 unp->unp_addr->sun_len); 3371 else 3372 bzero(&xu->xu_addr, sizeof(xu->xu_addr)); 3373 if (unp->unp_conn != NULL && 3374 unp->unp_conn->unp_addr != NULL) 3375 bcopy(unp->unp_conn->unp_addr, 3376 &xu->xu_caddr, 3377 unp->unp_conn->unp_addr->sun_len); 3378 else 3379 bzero(&xu->xu_caddr, sizeof(xu->xu_caddr)); 3380 xu->unp_vnode = (uintptr_t)unp->unp_vnode; 3381 xu->unp_conn = (uintptr_t)unp->unp_conn; 3382 xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs); 3383 xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink); 3384 xu->unp_gencnt = unp->unp_gencnt; 3385 sotoxsocket(unp->unp_socket, &xu->xu_socket); 3386 UNP_PCB_UNLOCK(unp); 3387 error = SYSCTL_OUT(req, xu, sizeof *xu); 3388 } else { 3389 UNP_PCB_UNLOCK(unp); 3390 } 3391 } 3392 free(xu, M_TEMP); 3393 if (!error) { 3394 /* 3395 * Give the user an updated idea of our state. If the 3396 * generation differs from what we told her before, she knows 3397 * that something happened while we were processing this 3398 * request, and it might be necessary to retry. 3399 */ 3400 xug->xug_gen = unp_gencnt; 3401 xug->xug_sogen = so_gencnt; 3402 xug->xug_count = unp_count; 3403 error = SYSCTL_OUT(req, xug, sizeof *xug); 3404 } 3405 free(unp_list, M_TEMP); 3406 free(xug, M_TEMP); 3407 return (error); 3408 } 3409 3410 SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, 3411 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3412 (void *)(intptr_t)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb", 3413 "List of active local datagram sockets"); 3414 SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, 3415 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3416 (void *)(intptr_t)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb", 3417 "List of active local stream sockets"); 3418 SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist, 3419 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3420 (void *)(intptr_t)SOCK_SEQPACKET, 0, unp_pcblist, "S,xunpcb", 3421 "List of active local seqpacket sockets"); 3422 3423 static void 3424 unp_drop(struct unpcb *unp) 3425 { 3426 struct socket *so; 3427 struct unpcb *unp2; 3428 3429 /* 3430 * Regardless of whether the socket's peer dropped the connection 3431 * with this socket by aborting or disconnecting, POSIX requires 3432 * that ECONNRESET is returned on next connected send(2) in case of 3433 * a SOCK_DGRAM socket and EPIPE for SOCK_STREAM. 3434 */ 3435 UNP_PCB_LOCK(unp); 3436 if ((so = unp->unp_socket) != NULL) 3437 so->so_error = 3438 so->so_proto->pr_type == SOCK_DGRAM ? ECONNRESET : EPIPE; 3439 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) { 3440 /* Last reference dropped in unp_disconnect(). */ 3441 unp_pcb_rele_notlast(unp); 3442 unp_disconnect(unp, unp2); 3443 } else if (!unp_pcb_rele(unp)) { 3444 UNP_PCB_UNLOCK(unp); 3445 } 3446 } 3447 3448 static void 3449 unp_freerights(struct filedescent **fdep, int fdcount) 3450 { 3451 struct file *fp; 3452 int i; 3453 3454 KASSERT(fdcount > 0, ("%s: fdcount %d", __func__, fdcount)); 3455 3456 for (i = 0; i < fdcount; i++) { 3457 fp = fdep[i]->fde_file; 3458 filecaps_free(&fdep[i]->fde_caps); 3459 unp_discard(fp); 3460 } 3461 free(fdep[0], M_FILECAPS); 3462 } 3463 3464 static bool 3465 restrict_rights(struct file *fp, struct thread *td) 3466 { 3467 struct prison *prison1, *prison2; 3468 3469 prison1 = fp->f_cred->cr_prison; 3470 prison2 = td->td_ucred->cr_prison; 3471 return (prison1 != prison2 && prison1->pr_root != prison2->pr_root && 3472 prison2 != &prison0); 3473 } 3474 3475 static int 3476 unp_externalize(struct mbuf *control, struct mbuf **controlp, int flags) 3477 { 3478 struct thread *td = curthread; /* XXX */ 3479 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 3480 int *fdp; 3481 struct filedesc *fdesc = td->td_proc->p_fd; 3482 struct filedescent **fdep; 3483 void *data; 3484 socklen_t clen = control->m_len, datalen; 3485 int error, fdflags, newfds; 3486 u_int newlen; 3487 3488 UNP_LINK_UNLOCK_ASSERT(); 3489 3490 fdflags = ((flags & MSG_CMSG_CLOEXEC) ? O_CLOEXEC : 0) | 3491 ((flags & MSG_CMSG_CLOFORK) ? O_CLOFORK : 0); 3492 3493 error = 0; 3494 if (controlp != NULL) /* controlp == NULL => free control messages */ 3495 *controlp = NULL; 3496 while (cm != NULL) { 3497 MPASS(clen >= sizeof(*cm) && clen >= cm->cmsg_len); 3498 3499 data = CMSG_DATA(cm); 3500 datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; 3501 if (cm->cmsg_level == SOL_SOCKET 3502 && cm->cmsg_type == SCM_RIGHTS) { 3503 newfds = datalen / sizeof(*fdep); 3504 if (newfds == 0) 3505 goto next; 3506 fdep = data; 3507 3508 /* If we're not outputting the descriptors free them. */ 3509 if (error || controlp == NULL) { 3510 unp_freerights(fdep, newfds); 3511 goto next; 3512 } 3513 FILEDESC_XLOCK(fdesc); 3514 3515 /* 3516 * Now change each pointer to an fd in the global 3517 * table to an integer that is the index to the local 3518 * fd table entry that we set up to point to the 3519 * global one we are transferring. 3520 */ 3521 newlen = newfds * sizeof(int); 3522 *controlp = sbcreatecontrol(NULL, newlen, 3523 SCM_RIGHTS, SOL_SOCKET, M_WAITOK); 3524 3525 fdp = (int *) 3526 CMSG_DATA(mtod(*controlp, struct cmsghdr *)); 3527 if ((error = fdallocn(td, 0, fdp, newfds))) { 3528 FILEDESC_XUNLOCK(fdesc); 3529 unp_freerights(fdep, newfds); 3530 m_freem(*controlp); 3531 *controlp = NULL; 3532 goto next; 3533 } 3534 for (int i = 0; i < newfds; i++, fdp++) { 3535 struct file *fp; 3536 3537 fp = fdep[i]->fde_file; 3538 _finstall(fdesc, fp, *fdp, fdflags | 3539 (restrict_rights(fp, td) ? 3540 O_RESOLVE_BENEATH : 0), &fdep[i]->fde_caps); 3541 unp_externalize_fp(fp); 3542 } 3543 3544 /* 3545 * The new type indicates that the mbuf data refers to 3546 * kernel resources that may need to be released before 3547 * the mbuf is freed. 3548 */ 3549 m_chtype(*controlp, MT_EXTCONTROL); 3550 FILEDESC_XUNLOCK(fdesc); 3551 free(fdep[0], M_FILECAPS); 3552 } else { 3553 /* We can just copy anything else across. */ 3554 if (error || controlp == NULL) 3555 goto next; 3556 *controlp = sbcreatecontrol(NULL, datalen, 3557 cm->cmsg_type, cm->cmsg_level, M_WAITOK); 3558 bcopy(data, 3559 CMSG_DATA(mtod(*controlp, struct cmsghdr *)), 3560 datalen); 3561 } 3562 controlp = &(*controlp)->m_next; 3563 3564 next: 3565 if (CMSG_SPACE(datalen) < clen) { 3566 clen -= CMSG_SPACE(datalen); 3567 cm = (struct cmsghdr *) 3568 ((caddr_t)cm + CMSG_SPACE(datalen)); 3569 } else { 3570 clen = 0; 3571 cm = NULL; 3572 } 3573 } 3574 3575 return (error); 3576 } 3577 3578 static void 3579 unp_zone_change(void *tag) 3580 { 3581 3582 uma_zone_set_max(unp_zone, maxsockets); 3583 } 3584 3585 #ifdef INVARIANTS 3586 static void 3587 unp_zdtor(void *mem, int size __unused, void *arg __unused) 3588 { 3589 struct unpcb *unp; 3590 3591 unp = mem; 3592 3593 KASSERT(LIST_EMPTY(&unp->unp_refs), 3594 ("%s: unpcb %p has lingering refs", __func__, unp)); 3595 KASSERT(unp->unp_socket == NULL, 3596 ("%s: unpcb %p has socket backpointer", __func__, unp)); 3597 KASSERT(unp->unp_vnode == NULL, 3598 ("%s: unpcb %p has vnode references", __func__, unp)); 3599 KASSERT(unp->unp_conn == NULL, 3600 ("%s: unpcb %p is still connected", __func__, unp)); 3601 KASSERT(unp->unp_addr == NULL, 3602 ("%s: unpcb %p has leaked addr", __func__, unp)); 3603 } 3604 #endif 3605 3606 static void 3607 unp_init(void *arg __unused) 3608 { 3609 uma_dtor dtor; 3610 3611 #ifdef INVARIANTS 3612 dtor = unp_zdtor; 3613 #else 3614 dtor = NULL; 3615 #endif 3616 unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, dtor, 3617 NULL, NULL, UMA_ALIGN_CACHE, 0); 3618 uma_zone_set_max(unp_zone, maxsockets); 3619 uma_zone_set_warning(unp_zone, "kern.ipc.maxsockets limit reached"); 3620 EVENTHANDLER_REGISTER(maxsockets_change, unp_zone_change, 3621 NULL, EVENTHANDLER_PRI_ANY); 3622 LIST_INIT(&unp_dhead); 3623 LIST_INIT(&unp_shead); 3624 LIST_INIT(&unp_sphead); 3625 SLIST_INIT(&unp_defers); 3626 TIMEOUT_TASK_INIT(taskqueue_thread, &unp_gc_task, 0, unp_gc, NULL); 3627 TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL); 3628 UNP_LINK_LOCK_INIT(); 3629 UNP_DEFERRED_LOCK_INIT(); 3630 unp_vp_mtxpool = mtx_pool_create("unp vp mtxpool", 32, MTX_DEF); 3631 } 3632 SYSINIT(unp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_SECOND, unp_init, NULL); 3633 3634 static void 3635 unp_internalize_cleanup_rights(struct mbuf *control) 3636 { 3637 struct cmsghdr *cp; 3638 struct mbuf *m; 3639 void *data; 3640 socklen_t datalen; 3641 3642 for (m = control; m != NULL; m = m->m_next) { 3643 cp = mtod(m, struct cmsghdr *); 3644 if (cp->cmsg_level != SOL_SOCKET || 3645 cp->cmsg_type != SCM_RIGHTS) 3646 continue; 3647 data = CMSG_DATA(cp); 3648 datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data; 3649 unp_freerights(data, datalen / sizeof(struct filedesc *)); 3650 } 3651 } 3652 3653 static int 3654 unp_internalize(struct mbuf *control, struct mchain *mc, struct thread *td) 3655 { 3656 struct proc *p; 3657 struct filedesc *fdesc; 3658 struct bintime *bt; 3659 struct cmsghdr *cm; 3660 struct cmsgcred *cmcred; 3661 struct mbuf *m; 3662 struct filedescent *fde, **fdep, *fdev; 3663 struct file *fp; 3664 struct timeval *tv; 3665 struct timespec *ts; 3666 void *data; 3667 socklen_t clen, datalen; 3668 int i, j, error, *fdp, oldfds; 3669 u_int newlen; 3670 3671 MPASS(control->m_next == NULL); /* COMPAT_OLDSOCK may violate */ 3672 UNP_LINK_UNLOCK_ASSERT(); 3673 3674 p = td->td_proc; 3675 fdesc = p->p_fd; 3676 error = 0; 3677 *mc = MCHAIN_INITIALIZER(mc); 3678 for (clen = control->m_len, cm = mtod(control, struct cmsghdr *), 3679 data = CMSG_DATA(cm); 3680 3681 clen >= sizeof(*cm) && cm->cmsg_level == SOL_SOCKET && 3682 clen >= cm->cmsg_len && cm->cmsg_len >= sizeof(*cm) && 3683 (char *)cm + cm->cmsg_len >= (char *)data; 3684 3685 clen -= min(CMSG_SPACE(datalen), clen), 3686 cm = (struct cmsghdr *) ((char *)cm + CMSG_SPACE(datalen)), 3687 data = CMSG_DATA(cm)) { 3688 datalen = (char *)cm + cm->cmsg_len - (char *)data; 3689 switch (cm->cmsg_type) { 3690 case SCM_CREDS: 3691 m = sbcreatecontrol(NULL, sizeof(*cmcred), SCM_CREDS, 3692 SOL_SOCKET, M_WAITOK); 3693 cmcred = (struct cmsgcred *) 3694 CMSG_DATA(mtod(m, struct cmsghdr *)); 3695 cmcred->cmcred_pid = p->p_pid; 3696 cmcred->cmcred_uid = td->td_ucred->cr_ruid; 3697 cmcred->cmcred_gid = td->td_ucred->cr_rgid; 3698 cmcred->cmcred_euid = td->td_ucred->cr_uid; 3699 _Static_assert(CMGROUP_MAX >= 1, 3700 "Room needed for the effective GID."); 3701 cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups + 1, 3702 CMGROUP_MAX); 3703 cmcred->cmcred_groups[0] = td->td_ucred->cr_gid; 3704 for (i = 1; i < cmcred->cmcred_ngroups; i++) 3705 cmcred->cmcred_groups[i] = 3706 td->td_ucred->cr_groups[i - 1]; 3707 break; 3708 3709 case SCM_RIGHTS: 3710 oldfds = datalen / sizeof (int); 3711 if (oldfds == 0) 3712 continue; 3713 /* On some machines sizeof pointer is bigger than 3714 * sizeof int, so we need to check if data fits into 3715 * single mbuf. We could allocate several mbufs, and 3716 * unp_externalize() should even properly handle that. 3717 * But it is not worth to complicate the code for an 3718 * insane scenario of passing over 200 file descriptors 3719 * at once. 3720 */ 3721 newlen = oldfds * sizeof(fdep[0]); 3722 if (CMSG_SPACE(newlen) > MCLBYTES) { 3723 error = EMSGSIZE; 3724 goto out; 3725 } 3726 /* 3727 * Check that all the FDs passed in refer to legal 3728 * files. If not, reject the entire operation. 3729 */ 3730 fdp = data; 3731 FILEDESC_SLOCK(fdesc); 3732 for (i = 0; i < oldfds; i++, fdp++) { 3733 fp = fget_noref(fdesc, *fdp); 3734 if (fp == NULL) { 3735 FILEDESC_SUNLOCK(fdesc); 3736 error = EBADF; 3737 goto out; 3738 } 3739 if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) { 3740 FILEDESC_SUNLOCK(fdesc); 3741 error = EOPNOTSUPP; 3742 goto out; 3743 } 3744 } 3745 3746 /* 3747 * Now replace the integer FDs with pointers to the 3748 * file structure and capability rights. 3749 */ 3750 m = sbcreatecontrol(NULL, newlen, SCM_RIGHTS, 3751 SOL_SOCKET, M_WAITOK); 3752 fdp = data; 3753 for (i = 0; i < oldfds; i++, fdp++) { 3754 if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) { 3755 fdp = data; 3756 for (j = 0; j < i; j++, fdp++) { 3757 fdrop(fdesc->fd_ofiles[*fdp]. 3758 fde_file, td); 3759 } 3760 FILEDESC_SUNLOCK(fdesc); 3761 error = EBADF; 3762 goto out; 3763 } 3764 } 3765 fdp = data; 3766 fdep = (struct filedescent **) 3767 CMSG_DATA(mtod(m, struct cmsghdr *)); 3768 fdev = malloc(sizeof(*fdev) * oldfds, M_FILECAPS, 3769 M_WAITOK); 3770 for (i = 0; i < oldfds; i++, fdev++, fdp++) { 3771 fde = &fdesc->fd_ofiles[*fdp]; 3772 fdep[i] = fdev; 3773 fdep[i]->fde_file = fde->fde_file; 3774 filecaps_copy(&fde->fde_caps, 3775 &fdep[i]->fde_caps, true); 3776 unp_internalize_fp(fdep[i]->fde_file); 3777 } 3778 FILEDESC_SUNLOCK(fdesc); 3779 break; 3780 3781 case SCM_TIMESTAMP: 3782 m = sbcreatecontrol(NULL, sizeof(*tv), SCM_TIMESTAMP, 3783 SOL_SOCKET, M_WAITOK); 3784 tv = (struct timeval *) 3785 CMSG_DATA(mtod(m, struct cmsghdr *)); 3786 microtime(tv); 3787 break; 3788 3789 case SCM_BINTIME: 3790 m = sbcreatecontrol(NULL, sizeof(*bt), SCM_BINTIME, 3791 SOL_SOCKET, M_WAITOK); 3792 bt = (struct bintime *) 3793 CMSG_DATA(mtod(m, struct cmsghdr *)); 3794 bintime(bt); 3795 break; 3796 3797 case SCM_REALTIME: 3798 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_REALTIME, 3799 SOL_SOCKET, M_WAITOK); 3800 ts = (struct timespec *) 3801 CMSG_DATA(mtod(m, struct cmsghdr *)); 3802 nanotime(ts); 3803 break; 3804 3805 case SCM_MONOTONIC: 3806 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_MONOTONIC, 3807 SOL_SOCKET, M_WAITOK); 3808 ts = (struct timespec *) 3809 CMSG_DATA(mtod(m, struct cmsghdr *)); 3810 nanouptime(ts); 3811 break; 3812 3813 default: 3814 error = EINVAL; 3815 goto out; 3816 } 3817 3818 mc_append(mc, m); 3819 } 3820 if (clen > 0) 3821 error = EINVAL; 3822 3823 out: 3824 if (error != 0) 3825 unp_internalize_cleanup_rights(mc_first(mc)); 3826 m_freem(control); 3827 return (error); 3828 } 3829 3830 static void 3831 unp_addsockcred(struct thread *td, struct mchain *mc, int mode) 3832 { 3833 struct mbuf *m, *n, *n_prev; 3834 const struct cmsghdr *cm; 3835 int ngroups, i, cmsgtype; 3836 size_t ctrlsz; 3837 3838 ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX); 3839 if (mode & UNP_WANTCRED_ALWAYS) { 3840 ctrlsz = SOCKCRED2SIZE(ngroups); 3841 cmsgtype = SCM_CREDS2; 3842 } else { 3843 ctrlsz = SOCKCREDSIZE(ngroups); 3844 cmsgtype = SCM_CREDS; 3845 } 3846 3847 /* XXXGL: uipc_sosend_*() need to be improved so that we can M_WAITOK */ 3848 m = sbcreatecontrol(NULL, ctrlsz, cmsgtype, SOL_SOCKET, M_NOWAIT); 3849 if (m == NULL) 3850 return; 3851 MPASS((m->m_flags & M_EXT) == 0 && m->m_next == NULL); 3852 3853 if (mode & UNP_WANTCRED_ALWAYS) { 3854 struct sockcred2 *sc; 3855 3856 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3857 sc->sc_version = 0; 3858 sc->sc_pid = td->td_proc->p_pid; 3859 sc->sc_uid = td->td_ucred->cr_ruid; 3860 sc->sc_euid = td->td_ucred->cr_uid; 3861 sc->sc_gid = td->td_ucred->cr_rgid; 3862 sc->sc_egid = td->td_ucred->cr_gid; 3863 sc->sc_ngroups = ngroups; 3864 for (i = 0; i < sc->sc_ngroups; i++) 3865 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3866 } else { 3867 struct sockcred *sc; 3868 3869 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3870 sc->sc_uid = td->td_ucred->cr_ruid; 3871 sc->sc_euid = td->td_ucred->cr_uid; 3872 sc->sc_gid = td->td_ucred->cr_rgid; 3873 sc->sc_egid = td->td_ucred->cr_gid; 3874 sc->sc_ngroups = ngroups; 3875 for (i = 0; i < sc->sc_ngroups; i++) 3876 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3877 } 3878 3879 /* 3880 * Unlink SCM_CREDS control messages (struct cmsgcred), since just 3881 * created SCM_CREDS control message (struct sockcred) has another 3882 * format. 3883 */ 3884 if (!STAILQ_EMPTY(&mc->mc_q) && cmsgtype == SCM_CREDS) 3885 STAILQ_FOREACH_SAFE(n, &mc->mc_q, m_stailq, n_prev) { 3886 cm = mtod(n, struct cmsghdr *); 3887 if (cm->cmsg_level == SOL_SOCKET && 3888 cm->cmsg_type == SCM_CREDS) { 3889 mc_remove(mc, n); 3890 m_free(n); 3891 } 3892 } 3893 3894 /* Prepend it to the head. */ 3895 mc_prepend(mc, m); 3896 } 3897 3898 static struct unpcb * 3899 fptounp(struct file *fp) 3900 { 3901 struct socket *so; 3902 3903 if (fp->f_type != DTYPE_SOCKET) 3904 return (NULL); 3905 if ((so = fp->f_data) == NULL) 3906 return (NULL); 3907 if (so->so_proto->pr_domain != &localdomain) 3908 return (NULL); 3909 return sotounpcb(so); 3910 } 3911 3912 static void 3913 unp_discard(struct file *fp) 3914 { 3915 struct unp_defer *dr; 3916 3917 if (unp_externalize_fp(fp)) { 3918 dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK); 3919 dr->ud_fp = fp; 3920 UNP_DEFERRED_LOCK(); 3921 SLIST_INSERT_HEAD(&unp_defers, dr, ud_link); 3922 UNP_DEFERRED_UNLOCK(); 3923 atomic_add_int(&unp_defers_count, 1); 3924 taskqueue_enqueue(taskqueue_thread, &unp_defer_task); 3925 } else 3926 closef_nothread(fp); 3927 } 3928 3929 static void 3930 unp_process_defers(void *arg __unused, int pending) 3931 { 3932 struct unp_defer *dr; 3933 SLIST_HEAD(, unp_defer) drl; 3934 int count; 3935 3936 SLIST_INIT(&drl); 3937 for (;;) { 3938 UNP_DEFERRED_LOCK(); 3939 if (SLIST_FIRST(&unp_defers) == NULL) { 3940 UNP_DEFERRED_UNLOCK(); 3941 break; 3942 } 3943 SLIST_SWAP(&unp_defers, &drl, unp_defer); 3944 UNP_DEFERRED_UNLOCK(); 3945 count = 0; 3946 while ((dr = SLIST_FIRST(&drl)) != NULL) { 3947 SLIST_REMOVE_HEAD(&drl, ud_link); 3948 closef_nothread(dr->ud_fp); 3949 free(dr, M_TEMP); 3950 count++; 3951 } 3952 atomic_add_int(&unp_defers_count, -count); 3953 } 3954 } 3955 3956 static void 3957 unp_internalize_fp(struct file *fp) 3958 { 3959 struct unpcb *unp; 3960 3961 UNP_LINK_WLOCK(); 3962 if ((unp = fptounp(fp)) != NULL) { 3963 unp->unp_file = fp; 3964 unp->unp_msgcount++; 3965 } 3966 unp_rights++; 3967 UNP_LINK_WUNLOCK(); 3968 } 3969 3970 static int 3971 unp_externalize_fp(struct file *fp) 3972 { 3973 struct unpcb *unp; 3974 int ret; 3975 3976 UNP_LINK_WLOCK(); 3977 if ((unp = fptounp(fp)) != NULL) { 3978 unp->unp_msgcount--; 3979 ret = 1; 3980 } else 3981 ret = 0; 3982 unp_rights--; 3983 UNP_LINK_WUNLOCK(); 3984 return (ret); 3985 } 3986 3987 /* 3988 * unp_defer indicates whether additional work has been defered for a future 3989 * pass through unp_gc(). It is thread local and does not require explicit 3990 * synchronization. 3991 */ 3992 static int unp_marked; 3993 3994 static void 3995 unp_remove_dead_ref(struct filedescent **fdep, int fdcount) 3996 { 3997 struct unpcb *unp; 3998 struct file *fp; 3999 int i; 4000 4001 /* 4002 * This function can only be called from the gc task. 4003 */ 4004 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4005 ("%s: not on gc callout", __func__)); 4006 UNP_LINK_LOCK_ASSERT(); 4007 4008 for (i = 0; i < fdcount; i++) { 4009 fp = fdep[i]->fde_file; 4010 if ((unp = fptounp(fp)) == NULL) 4011 continue; 4012 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4013 continue; 4014 unp->unp_gcrefs--; 4015 } 4016 } 4017 4018 static void 4019 unp_restore_undead_ref(struct filedescent **fdep, int fdcount) 4020 { 4021 struct unpcb *unp; 4022 struct file *fp; 4023 int i; 4024 4025 /* 4026 * This function can only be called from the gc task. 4027 */ 4028 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4029 ("%s: not on gc callout", __func__)); 4030 UNP_LINK_LOCK_ASSERT(); 4031 4032 for (i = 0; i < fdcount; i++) { 4033 fp = fdep[i]->fde_file; 4034 if ((unp = fptounp(fp)) == NULL) 4035 continue; 4036 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4037 continue; 4038 unp->unp_gcrefs++; 4039 unp_marked++; 4040 } 4041 } 4042 4043 static void 4044 unp_scan_socket(struct socket *so, void (*op)(struct filedescent **, int)) 4045 { 4046 struct sockbuf *sb; 4047 4048 SOCK_LOCK_ASSERT(so); 4049 4050 if (sotounpcb(so)->unp_gcflag & UNPGC_IGNORE_RIGHTS) 4051 return; 4052 4053 SOCK_RECVBUF_LOCK(so); 4054 switch (so->so_type) { 4055 case SOCK_DGRAM: 4056 unp_scan(STAILQ_FIRST(&so->so_rcv.uxdg_mb), op); 4057 unp_scan(so->so_rcv.uxdg_peeked, op); 4058 TAILQ_FOREACH(sb, &so->so_rcv.uxdg_conns, uxdg_clist) 4059 unp_scan(STAILQ_FIRST(&sb->uxdg_mb), op); 4060 break; 4061 case SOCK_STREAM: 4062 case SOCK_SEQPACKET: 4063 unp_scan(STAILQ_FIRST(&so->so_rcv.uxst_mbq), op); 4064 break; 4065 } 4066 SOCK_RECVBUF_UNLOCK(so); 4067 } 4068 4069 static void 4070 unp_gc_scan(struct unpcb *unp, void (*op)(struct filedescent **, int)) 4071 { 4072 struct socket *so, *soa; 4073 4074 so = unp->unp_socket; 4075 SOCK_LOCK(so); 4076 if (SOLISTENING(so)) { 4077 /* 4078 * Mark all sockets in our accept queue. 4079 */ 4080 TAILQ_FOREACH(soa, &so->sol_comp, so_list) 4081 unp_scan_socket(soa, op); 4082 } else { 4083 /* 4084 * Mark all sockets we reference with RIGHTS. 4085 */ 4086 unp_scan_socket(so, op); 4087 } 4088 SOCK_UNLOCK(so); 4089 } 4090 4091 static int unp_recycled; 4092 SYSCTL_INT(_net_local, OID_AUTO, recycled, CTLFLAG_RD, &unp_recycled, 0, 4093 "Number of unreachable sockets claimed by the garbage collector."); 4094 4095 static int unp_taskcount; 4096 SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0, 4097 "Number of times the garbage collector has run."); 4098 4099 SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0, 4100 "Number of active local sockets."); 4101 4102 static void 4103 unp_gc(__unused void *arg, int pending) 4104 { 4105 struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead, 4106 NULL }; 4107 struct unp_head **head; 4108 struct unp_head unp_deadhead; /* List of potentially-dead sockets. */ 4109 struct file *f, **unref; 4110 struct unpcb *unp, *unptmp; 4111 int i, total, unp_unreachable; 4112 4113 LIST_INIT(&unp_deadhead); 4114 unp_taskcount++; 4115 UNP_LINK_RLOCK(); 4116 /* 4117 * First determine which sockets may be in cycles. 4118 */ 4119 unp_unreachable = 0; 4120 4121 for (head = heads; *head != NULL; head++) 4122 LIST_FOREACH(unp, *head, unp_link) { 4123 KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0, 4124 ("%s: unp %p has unexpected gc flags 0x%x", 4125 __func__, unp, (unsigned int)unp->unp_gcflag)); 4126 4127 f = unp->unp_file; 4128 4129 /* 4130 * Check for an unreachable socket potentially in a 4131 * cycle. It must be in a queue as indicated by 4132 * msgcount, and this must equal the file reference 4133 * count. Note that when msgcount is 0 the file is 4134 * NULL. 4135 */ 4136 if (f != NULL && unp->unp_msgcount != 0 && 4137 refcount_load(&f->f_count) == unp->unp_msgcount) { 4138 LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead); 4139 unp->unp_gcflag |= UNPGC_DEAD; 4140 unp->unp_gcrefs = unp->unp_msgcount; 4141 unp_unreachable++; 4142 } 4143 } 4144 4145 /* 4146 * Scan all sockets previously marked as potentially being in a cycle 4147 * and remove the references each socket holds on any UNPGC_DEAD 4148 * sockets in its queue. After this step, all remaining references on 4149 * sockets marked UNPGC_DEAD should not be part of any cycle. 4150 */ 4151 LIST_FOREACH(unp, &unp_deadhead, unp_dead) 4152 unp_gc_scan(unp, unp_remove_dead_ref); 4153 4154 /* 4155 * If a socket still has a non-negative refcount, it cannot be in a 4156 * cycle. In this case increment refcount of all children iteratively. 4157 * Stop the scan once we do a complete loop without discovering 4158 * a new reachable socket. 4159 */ 4160 do { 4161 unp_marked = 0; 4162 LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp) 4163 if (unp->unp_gcrefs > 0) { 4164 unp->unp_gcflag &= ~UNPGC_DEAD; 4165 LIST_REMOVE(unp, unp_dead); 4166 KASSERT(unp_unreachable > 0, 4167 ("%s: unp_unreachable underflow.", 4168 __func__)); 4169 unp_unreachable--; 4170 unp_gc_scan(unp, unp_restore_undead_ref); 4171 } 4172 } while (unp_marked); 4173 4174 UNP_LINK_RUNLOCK(); 4175 4176 if (unp_unreachable == 0) 4177 return; 4178 4179 /* 4180 * Allocate space for a local array of dead unpcbs. 4181 * TODO: can this path be simplified by instead using the local 4182 * dead list at unp_deadhead, after taking out references 4183 * on the file object and/or unpcb and dropping the link lock? 4184 */ 4185 unref = malloc(unp_unreachable * sizeof(struct file *), 4186 M_TEMP, M_WAITOK); 4187 4188 /* 4189 * Iterate looking for sockets which have been specifically marked 4190 * as unreachable and store them locally. 4191 */ 4192 UNP_LINK_RLOCK(); 4193 total = 0; 4194 LIST_FOREACH(unp, &unp_deadhead, unp_dead) { 4195 KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0, 4196 ("%s: unp %p not marked UNPGC_DEAD", __func__, unp)); 4197 unp->unp_gcflag &= ~UNPGC_DEAD; 4198 f = unp->unp_file; 4199 if (unp->unp_msgcount == 0 || f == NULL || 4200 refcount_load(&f->f_count) != unp->unp_msgcount || 4201 !fhold(f)) 4202 continue; 4203 unref[total++] = f; 4204 KASSERT(total <= unp_unreachable, 4205 ("%s: incorrect unreachable count.", __func__)); 4206 } 4207 UNP_LINK_RUNLOCK(); 4208 4209 /* 4210 * Now flush all sockets, free'ing rights. This will free the 4211 * struct files associated with these sockets but leave each socket 4212 * with one remaining ref. 4213 */ 4214 for (i = 0; i < total; i++) { 4215 struct socket *so; 4216 4217 so = unref[i]->f_data; 4218 if (!SOLISTENING(so)) { 4219 CURVNET_SET(so->so_vnet); 4220 socantrcvmore(so); 4221 unp_dispose(so); 4222 CURVNET_RESTORE(); 4223 } 4224 } 4225 4226 /* 4227 * And finally release the sockets so they can be reclaimed. 4228 */ 4229 for (i = 0; i < total; i++) 4230 fdrop(unref[i], NULL); 4231 unp_recycled += total; 4232 free(unref, M_TEMP); 4233 } 4234 4235 /* 4236 * Synchronize against unp_gc, which can trip over data as we are freeing it. 4237 */ 4238 static void 4239 unp_dispose(struct socket *so) 4240 { 4241 struct sockbuf *sb; 4242 struct unpcb *unp; 4243 struct mbuf *m; 4244 int error __diagused; 4245 4246 MPASS(!SOLISTENING(so)); 4247 4248 unp = sotounpcb(so); 4249 UNP_LINK_WLOCK(); 4250 unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS; 4251 UNP_LINK_WUNLOCK(); 4252 4253 /* 4254 * Grab our special mbufs before calling sbrelease(). 4255 */ 4256 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 4257 MPASS(!error); 4258 SOCK_RECVBUF_LOCK(so); 4259 switch (so->so_type) { 4260 case SOCK_DGRAM: 4261 while ((sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) != NULL) { 4262 STAILQ_CONCAT(&so->so_rcv.uxdg_mb, &sb->uxdg_mb); 4263 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 4264 /* Note: socket of sb may reconnect. */ 4265 sb->uxdg_cc = sb->uxdg_ctl = sb->uxdg_mbcnt = 0; 4266 } 4267 sb = &so->so_rcv; 4268 if (sb->uxdg_peeked != NULL) { 4269 STAILQ_INSERT_HEAD(&sb->uxdg_mb, sb->uxdg_peeked, 4270 m_stailqpkt); 4271 sb->uxdg_peeked = NULL; 4272 } 4273 m = STAILQ_FIRST(&sb->uxdg_mb); 4274 STAILQ_INIT(&sb->uxdg_mb); 4275 break; 4276 case SOCK_STREAM: 4277 case SOCK_SEQPACKET: 4278 sb = &so->so_rcv; 4279 m = STAILQ_FIRST(&sb->uxst_mbq); 4280 STAILQ_INIT(&sb->uxst_mbq); 4281 sb->sb_acc = sb->sb_ccc = sb->sb_ctl = sb->sb_mbcnt = 0; 4282 /* 4283 * Trim M_NOTREADY buffers from the free list. They are 4284 * referenced by the I/O thread. 4285 */ 4286 if (sb->uxst_fnrdy != NULL) { 4287 struct mbuf *n, *prev; 4288 4289 while (m != NULL && m->m_flags & M_NOTREADY) 4290 m = m->m_next; 4291 for (prev = n = m; n != NULL; n = n->m_next) { 4292 if (n->m_flags & M_NOTREADY) 4293 prev->m_next = n->m_next; 4294 else 4295 prev = n; 4296 } 4297 sb->uxst_fnrdy = NULL; 4298 } 4299 break; 4300 } 4301 /* 4302 * Mark sb with SBS_CANTRCVMORE. This is needed to prevent 4303 * uipc_sosend_*() or unp_disconnect() adding more data to the socket. 4304 * We came here either through shutdown(2) or from the final sofree(). 4305 * The sofree() case is simple as it guarantees that no more sends will 4306 * happen, however we can race with unp_disconnect() from our peer. 4307 * The shutdown(2) case is more exotic. It would call into 4308 * unp_dispose() only if socket is SS_ISCONNECTED. This is possible if 4309 * we did connect(2) on this socket and we also had it bound with 4310 * bind(2) and receive connections from other sockets. Because 4311 * uipc_shutdown() violates POSIX (see comment there) this applies to 4312 * SOCK_DGRAM as well. For SOCK_DGRAM this SBS_CANTRCVMORE will have 4313 * affect not only on the peer we connect(2)ed to, but also on all of 4314 * the peers who had connect(2)ed to us. Their sends would end up 4315 * with ENOBUFS. 4316 */ 4317 sb->sb_state |= SBS_CANTRCVMORE; 4318 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 4319 RLIM_INFINITY); 4320 SOCK_RECVBUF_UNLOCK(so); 4321 SOCK_IO_RECV_UNLOCK(so); 4322 4323 if (m != NULL) { 4324 unp_scan(m, unp_freerights); 4325 m_freemp(m); 4326 } 4327 } 4328 4329 static void 4330 unp_scan(struct mbuf *m0, void (*op)(struct filedescent **, int)) 4331 { 4332 struct mbuf *m; 4333 struct cmsghdr *cm; 4334 void *data; 4335 socklen_t clen, datalen; 4336 4337 while (m0 != NULL) { 4338 for (m = m0; m; m = m->m_next) { 4339 if (m->m_type != MT_CONTROL) 4340 continue; 4341 4342 cm = mtod(m, struct cmsghdr *); 4343 clen = m->m_len; 4344 4345 while (cm != NULL) { 4346 if (sizeof(*cm) > clen || cm->cmsg_len > clen) 4347 break; 4348 4349 data = CMSG_DATA(cm); 4350 datalen = (caddr_t)cm + cm->cmsg_len 4351 - (caddr_t)data; 4352 4353 if (cm->cmsg_level == SOL_SOCKET && 4354 cm->cmsg_type == SCM_RIGHTS) { 4355 (*op)(data, datalen / 4356 sizeof(struct filedescent *)); 4357 } 4358 4359 if (CMSG_SPACE(datalen) < clen) { 4360 clen -= CMSG_SPACE(datalen); 4361 cm = (struct cmsghdr *) 4362 ((caddr_t)cm + CMSG_SPACE(datalen)); 4363 } else { 4364 clen = 0; 4365 cm = NULL; 4366 } 4367 } 4368 } 4369 m0 = m0->m_nextpkt; 4370 } 4371 } 4372 4373 /* 4374 * Definitions of protocols supported in the LOCAL domain. 4375 */ 4376 static struct protosw streamproto = { 4377 .pr_type = SOCK_STREAM, 4378 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4379 .pr_ctloutput = &uipc_ctloutput, 4380 .pr_abort = uipc_abort, 4381 .pr_accept = uipc_peeraddr, 4382 .pr_attach = uipc_attach, 4383 .pr_bind = uipc_bind, 4384 .pr_bindat = uipc_bindat, 4385 .pr_connect = uipc_connect, 4386 .pr_connectat = uipc_connectat, 4387 .pr_connect2 = uipc_connect2, 4388 .pr_detach = uipc_detach, 4389 .pr_disconnect = uipc_disconnect, 4390 .pr_fdclose = uipc_fdclose, 4391 .pr_listen = uipc_listen, 4392 .pr_peeraddr = uipc_peeraddr, 4393 .pr_send = uipc_sendfile, 4394 .pr_sendfile_wait = uipc_sendfile_wait, 4395 .pr_ready = uipc_ready, 4396 .pr_sense = uipc_sense, 4397 .pr_shutdown = uipc_shutdown, 4398 .pr_sockaddr = uipc_sockaddr, 4399 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4400 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4401 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4402 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4403 .pr_close = uipc_close, 4404 .pr_chmod = uipc_chmod, 4405 }; 4406 4407 static struct protosw dgramproto = { 4408 .pr_type = SOCK_DGRAM, 4409 .pr_flags = PR_ATOMIC | PR_ADDR | PR_CAPATTACH | PR_SOCKBUF, 4410 .pr_ctloutput = &uipc_ctloutput, 4411 .pr_abort = uipc_abort, 4412 .pr_accept = uipc_peeraddr, 4413 .pr_attach = uipc_attach, 4414 .pr_bind = uipc_bind, 4415 .pr_bindat = uipc_bindat, 4416 .pr_connect = uipc_connect, 4417 .pr_connectat = uipc_connectat, 4418 .pr_connect2 = uipc_connect2, 4419 .pr_detach = uipc_detach, 4420 .pr_disconnect = uipc_disconnect, 4421 .pr_fdclose = uipc_fdclose, 4422 .pr_peeraddr = uipc_peeraddr, 4423 .pr_sosend = uipc_sosend_dgram, 4424 .pr_sense = uipc_sense, 4425 .pr_shutdown = uipc_shutdown, 4426 .pr_sockaddr = uipc_sockaddr, 4427 .pr_soreceive = uipc_soreceive_dgram, 4428 .pr_close = uipc_close, 4429 .pr_chmod = uipc_chmod, 4430 }; 4431 4432 static struct protosw seqpacketproto = { 4433 .pr_type = SOCK_SEQPACKET, 4434 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4435 .pr_ctloutput = &uipc_ctloutput, 4436 .pr_abort = uipc_abort, 4437 .pr_accept = uipc_peeraddr, 4438 .pr_attach = uipc_attach, 4439 .pr_bind = uipc_bind, 4440 .pr_bindat = uipc_bindat, 4441 .pr_connect = uipc_connect, 4442 .pr_connectat = uipc_connectat, 4443 .pr_connect2 = uipc_connect2, 4444 .pr_detach = uipc_detach, 4445 .pr_disconnect = uipc_disconnect, 4446 .pr_fdclose = uipc_fdclose, 4447 .pr_listen = uipc_listen, 4448 .pr_peeraddr = uipc_peeraddr, 4449 .pr_sense = uipc_sense, 4450 .pr_shutdown = uipc_shutdown, 4451 .pr_sockaddr = uipc_sockaddr, 4452 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4453 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4454 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4455 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4456 .pr_close = uipc_close, 4457 .pr_chmod = uipc_chmod, 4458 }; 4459 4460 static struct domain localdomain = { 4461 .dom_family = AF_LOCAL, 4462 .dom_name = "local", 4463 .dom_nprotosw = 3, 4464 .dom_protosw = { 4465 &streamproto, 4466 &dgramproto, 4467 &seqpacketproto, 4468 } 4469 }; 4470 DOMAIN_SET(local); 4471 4472 /* 4473 * A helper function called by VFS before socket-type vnode reclamation. 4474 * For an active vnode it clears unp_vnode pointer and decrements unp_vnode 4475 * use count. 4476 */ 4477 void 4478 vfs_unp_reclaim(struct vnode *vp) 4479 { 4480 struct unpcb *unp; 4481 int active; 4482 struct mtx *vplock; 4483 4484 ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim"); 4485 KASSERT(vp->v_type == VSOCK, 4486 ("vfs_unp_reclaim: vp->v_type != VSOCK")); 4487 4488 active = 0; 4489 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 4490 mtx_lock(vplock); 4491 VOP_UNP_CONNECT(vp, &unp); 4492 if (unp == NULL) 4493 goto done; 4494 UNP_PCB_LOCK(unp); 4495 if (unp->unp_vnode == vp) { 4496 VOP_UNP_DETACH(vp); 4497 unp->unp_vnode = NULL; 4498 active = 1; 4499 } 4500 UNP_PCB_UNLOCK(unp); 4501 done: 4502 mtx_unlock(vplock); 4503 if (active) 4504 vunref(vp); 4505 } 4506 4507 #ifdef DDB 4508 static void 4509 db_print_indent(int indent) 4510 { 4511 int i; 4512 4513 for (i = 0; i < indent; i++) 4514 db_printf(" "); 4515 } 4516 4517 static void 4518 db_print_unpflags(int unp_flags) 4519 { 4520 int comma; 4521 4522 comma = 0; 4523 if (unp_flags & UNP_HAVEPC) { 4524 db_printf("%sUNP_HAVEPC", comma ? ", " : ""); 4525 comma = 1; 4526 } 4527 if (unp_flags & UNP_WANTCRED_ALWAYS) { 4528 db_printf("%sUNP_WANTCRED_ALWAYS", comma ? ", " : ""); 4529 comma = 1; 4530 } 4531 if (unp_flags & UNP_WANTCRED_ONESHOT) { 4532 db_printf("%sUNP_WANTCRED_ONESHOT", comma ? ", " : ""); 4533 comma = 1; 4534 } 4535 if (unp_flags & UNP_CONNECTING) { 4536 db_printf("%sUNP_CONNECTING", comma ? ", " : ""); 4537 comma = 1; 4538 } 4539 if (unp_flags & UNP_BINDING) { 4540 db_printf("%sUNP_BINDING", comma ? ", " : ""); 4541 comma = 1; 4542 } 4543 } 4544 4545 static void 4546 db_print_xucred(int indent, struct xucred *xu) 4547 { 4548 int comma, i; 4549 4550 db_print_indent(indent); 4551 db_printf("cr_version: %u cr_uid: %u cr_pid: %d cr_ngroups: %d\n", 4552 xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups); 4553 db_print_indent(indent); 4554 db_printf("cr_groups: "); 4555 comma = 0; 4556 for (i = 0; i < xu->cr_ngroups; i++) { 4557 db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]); 4558 comma = 1; 4559 } 4560 db_printf("\n"); 4561 } 4562 4563 static void 4564 db_print_unprefs(int indent, struct unp_head *uh) 4565 { 4566 struct unpcb *unp; 4567 int counter; 4568 4569 counter = 0; 4570 LIST_FOREACH(unp, uh, unp_reflink) { 4571 if (counter % 4 == 0) 4572 db_print_indent(indent); 4573 db_printf("%p ", unp); 4574 if (counter % 4 == 3) 4575 db_printf("\n"); 4576 counter++; 4577 } 4578 if (counter != 0 && counter % 4 != 0) 4579 db_printf("\n"); 4580 } 4581 4582 DB_SHOW_COMMAND(unpcb, db_show_unpcb) 4583 { 4584 struct unpcb *unp; 4585 4586 if (!have_addr) { 4587 db_printf("usage: show unpcb <addr>\n"); 4588 return; 4589 } 4590 unp = (struct unpcb *)addr; 4591 4592 db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket, 4593 unp->unp_vnode); 4594 4595 db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino, 4596 unp->unp_conn); 4597 4598 db_printf("unp_refs:\n"); 4599 db_print_unprefs(2, &unp->unp_refs); 4600 4601 /* XXXRW: Would be nice to print the full address, if any. */ 4602 db_printf("unp_addr: %p\n", unp->unp_addr); 4603 4604 db_printf("unp_gencnt: %llu\n", 4605 (unsigned long long)unp->unp_gencnt); 4606 4607 db_printf("unp_flags: %x (", unp->unp_flags); 4608 db_print_unpflags(unp->unp_flags); 4609 db_printf(")\n"); 4610 4611 db_printf("unp_peercred:\n"); 4612 db_print_xucred(2, &unp->unp_peercred); 4613 4614 db_printf("unp_refcount: %u\n", unp->unp_refcount); 4615 } 4616 #endif 4617