1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1989, 1991, 1993 5 * The Regents of the University of California. All Rights Reserved. 6 * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved. 7 * Copyright (c) 2018 Matthew Macy 8 * Copyright (c) 2022-2025 Gleb Smirnoff <glebius@FreeBSD.org> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 /* 36 * UNIX Domain (Local) Sockets 37 * 38 * This is an implementation of UNIX (local) domain sockets. Each socket has 39 * an associated struct unpcb (UNIX protocol control block). Stream sockets 40 * may be connected to 0 or 1 other socket. Datagram sockets may be 41 * connected to 0, 1, or many other sockets. Sockets may be created and 42 * connected in pairs (socketpair(2)), or bound/connected to using the file 43 * system name space. For most purposes, only the receive socket buffer is 44 * used, as sending on one socket delivers directly to the receive socket 45 * buffer of a second socket. 46 * 47 * The implementation is substantially complicated by the fact that 48 * "ancillary data", such as file descriptors or credentials, may be passed 49 * across UNIX domain sockets. The potential for passing UNIX domain sockets 50 * over other UNIX domain sockets requires the implementation of a simple 51 * garbage collector to find and tear down cycles of disconnected sockets. 52 * 53 * TODO: 54 * RDM 55 * rethink name space problems 56 * need a proper out-of-band 57 */ 58 59 #include "opt_ddb.h" 60 61 #include <sys/param.h> 62 #include <sys/capsicum.h> 63 #include <sys/domain.h> 64 #include <sys/eventhandler.h> 65 #include <sys/fcntl.h> 66 #include <sys/file.h> 67 #include <sys/filedesc.h> 68 #include <sys/jail.h> 69 #include <sys/kernel.h> 70 #include <sys/lock.h> 71 #include <sys/malloc.h> 72 #include <sys/mbuf.h> 73 #include <sys/mount.h> 74 #include <sys/mutex.h> 75 #include <sys/namei.h> 76 #include <sys/poll.h> 77 #include <sys/proc.h> 78 #include <sys/protosw.h> 79 #include <sys/queue.h> 80 #include <sys/resourcevar.h> 81 #include <sys/rwlock.h> 82 #include <sys/socket.h> 83 #include <sys/socketvar.h> 84 #include <sys/signalvar.h> 85 #include <sys/stat.h> 86 #include <sys/sx.h> 87 #include <sys/sysctl.h> 88 #include <sys/systm.h> 89 #include <sys/taskqueue.h> 90 #include <sys/un.h> 91 #include <sys/unpcb.h> 92 #include <sys/vnode.h> 93 94 #include <net/vnet.h> 95 96 #ifdef DDB 97 #include <ddb/ddb.h> 98 #endif 99 100 #include <security/mac/mac_framework.h> 101 102 #include <vm/uma.h> 103 104 MALLOC_DECLARE(M_FILECAPS); 105 106 static struct domain localdomain; 107 108 static uma_zone_t unp_zone; 109 static unp_gen_t unp_gencnt; /* (l) */ 110 static u_int unp_count; /* (l) Count of local sockets. */ 111 static ino_t unp_ino; /* Prototype for fake inode numbers. */ 112 static int unp_rights; /* (g) File descriptors in flight. */ 113 static struct unp_head unp_shead; /* (l) List of stream sockets. */ 114 static struct unp_head unp_dhead; /* (l) List of datagram sockets. */ 115 static struct unp_head unp_sphead; /* (l) List of seqpacket sockets. */ 116 static struct mtx_pool *unp_vp_mtxpool; 117 118 struct unp_defer { 119 SLIST_ENTRY(unp_defer) ud_link; 120 struct file *ud_fp; 121 }; 122 static SLIST_HEAD(, unp_defer) unp_defers; 123 static int unp_defers_count; 124 125 static const struct sockaddr sun_noname = { 126 .sa_len = sizeof(sun_noname), 127 .sa_family = AF_LOCAL, 128 }; 129 130 /* 131 * Garbage collection of cyclic file descriptor/socket references occurs 132 * asynchronously in a taskqueue context in order to avoid recursion and 133 * reentrance in the UNIX domain socket, file descriptor, and socket layer 134 * code. See unp_gc() for a full description. 135 */ 136 static struct timeout_task unp_gc_task; 137 138 /* 139 * The close of unix domain sockets attached as SCM_RIGHTS is 140 * postponed to the taskqueue, to avoid arbitrary recursion depth. 141 * The attached sockets might have another sockets attached. 142 */ 143 static struct task unp_defer_task; 144 145 /* 146 * SOCK_STREAM and SOCK_SEQPACKET unix(4) sockets fully bypass the send buffer, 147 * however the notion of send buffer still makes sense with them. Its size is 148 * the amount of space that a send(2) syscall may copyin(9) before checking 149 * with the receive buffer of a peer. Although not linked anywhere yet, 150 * pointed to by a stack variable, effectively it is a buffer that needs to be 151 * sized. 152 * 153 * SOCK_DGRAM sockets really use the sendspace as the maximum datagram size, 154 * and don't really want to reserve the sendspace. Their recvspace should be 155 * large enough for at least one max-size datagram plus address. 156 */ 157 static u_long unpst_sendspace = 64*1024; 158 static u_long unpst_recvspace = 64*1024; 159 static u_long unpdg_maxdgram = 8*1024; /* support 8KB syslog msgs */ 160 static u_long unpdg_recvspace = 16*1024; 161 static u_long unpsp_sendspace = 64*1024; 162 static u_long unpsp_recvspace = 64*1024; 163 164 static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 165 "Local domain"); 166 static SYSCTL_NODE(_net_local, SOCK_STREAM, stream, 167 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 168 "SOCK_STREAM"); 169 static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram, 170 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 171 "SOCK_DGRAM"); 172 static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket, 173 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 174 "SOCK_SEQPACKET"); 175 176 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW, 177 &unpst_sendspace, 0, "Default stream send space."); 178 SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW, 179 &unpst_recvspace, 0, "Default stream receive space."); 180 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW, 181 &unpdg_maxdgram, 0, "Maximum datagram size."); 182 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW, 183 &unpdg_recvspace, 0, "Default datagram receive space."); 184 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW, 185 &unpsp_sendspace, 0, "Default seqpacket send space."); 186 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW, 187 &unpsp_recvspace, 0, "Default seqpacket receive space."); 188 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0, 189 "File descriptors in flight."); 190 SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD, 191 &unp_defers_count, 0, 192 "File descriptors deferred to taskqueue for close."); 193 194 /* 195 * Locking and synchronization: 196 * 197 * Several types of locks exist in the local domain socket implementation: 198 * - a global linkage lock 199 * - a global connection list lock 200 * - the mtxpool lock 201 * - per-unpcb mutexes 202 * 203 * The linkage lock protects the global socket lists, the generation number 204 * counter and garbage collector state. 205 * 206 * The connection list lock protects the list of referring sockets in a datagram 207 * socket PCB. This lock is also overloaded to protect a global list of 208 * sockets whose buffers contain socket references in the form of SCM_RIGHTS 209 * messages. To avoid recursion, such references are released by a dedicated 210 * thread. 211 * 212 * The mtxpool lock protects the vnode from being modified while referenced. 213 * Lock ordering rules require that it be acquired before any PCB locks. 214 * 215 * The unpcb lock (unp_mtx) protects the most commonly referenced fields in the 216 * unpcb. This includes the unp_conn field, which either links two connected 217 * PCBs together (for connected socket types) or points at the destination 218 * socket (for connectionless socket types). The operations of creating or 219 * destroying a connection therefore involve locking multiple PCBs. To avoid 220 * lock order reversals, in some cases this involves dropping a PCB lock and 221 * using a reference counter to maintain liveness. 222 * 223 * UNIX domain sockets each have an unpcb hung off of their so_pcb pointer, 224 * allocated in pr_attach() and freed in pr_detach(). The validity of that 225 * pointer is an invariant, so no lock is required to dereference the so_pcb 226 * pointer if a valid socket reference is held by the caller. In practice, 227 * this is always true during operations performed on a socket. Each unpcb 228 * has a back-pointer to its socket, unp_socket, which will be stable under 229 * the same circumstances. 230 * 231 * This pointer may only be safely dereferenced as long as a valid reference 232 * to the unpcb is held. Typically, this reference will be from the socket, 233 * or from another unpcb when the referring unpcb's lock is held (in order 234 * that the reference not be invalidated during use). For example, to follow 235 * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee 236 * that detach is not run clearing unp_socket. 237 * 238 * Blocking with UNIX domain sockets is a tricky issue: unlike most network 239 * protocols, bind() is a non-atomic operation, and connect() requires 240 * potential sleeping in the protocol, due to potentially waiting on local or 241 * distributed file systems. We try to separate "lookup" operations, which 242 * may sleep, and the IPC operations themselves, which typically can occur 243 * with relative atomicity as locks can be held over the entire operation. 244 * 245 * Another tricky issue is simultaneous multi-threaded or multi-process 246 * access to a single UNIX domain socket. These are handled by the flags 247 * UNP_CONNECTING and UNP_BINDING, which prevent concurrent connecting or 248 * binding, both of which involve dropping UNIX domain socket locks in order 249 * to perform namei() and other file system operations. 250 */ 251 static struct rwlock unp_link_rwlock; 252 static struct mtx unp_defers_lock; 253 254 #define UNP_LINK_LOCK_INIT() rw_init(&unp_link_rwlock, \ 255 "unp_link_rwlock") 256 257 #define UNP_LINK_LOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 258 RA_LOCKED) 259 #define UNP_LINK_UNLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 260 RA_UNLOCKED) 261 262 #define UNP_LINK_RLOCK() rw_rlock(&unp_link_rwlock) 263 #define UNP_LINK_RUNLOCK() rw_runlock(&unp_link_rwlock) 264 #define UNP_LINK_WLOCK() rw_wlock(&unp_link_rwlock) 265 #define UNP_LINK_WUNLOCK() rw_wunlock(&unp_link_rwlock) 266 #define UNP_LINK_WLOCK_ASSERT() rw_assert(&unp_link_rwlock, \ 267 RA_WLOCKED) 268 #define UNP_LINK_WOWNED() rw_wowned(&unp_link_rwlock) 269 270 #define UNP_DEFERRED_LOCK_INIT() mtx_init(&unp_defers_lock, \ 271 "unp_defer", NULL, MTX_DEF) 272 #define UNP_DEFERRED_LOCK() mtx_lock(&unp_defers_lock) 273 #define UNP_DEFERRED_UNLOCK() mtx_unlock(&unp_defers_lock) 274 275 #define UNP_REF_LIST_LOCK() UNP_DEFERRED_LOCK(); 276 #define UNP_REF_LIST_UNLOCK() UNP_DEFERRED_UNLOCK(); 277 278 #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \ 279 "unp", "unp", \ 280 MTX_DUPOK|MTX_DEF) 281 #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx) 282 #define UNP_PCB_LOCKPTR(unp) (&(unp)->unp_mtx) 283 #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx) 284 #define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx) 285 #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx) 286 #define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx) 287 #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED) 288 #define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED) 289 290 static int uipc_connect2(struct socket *, struct socket *); 291 static int uipc_ctloutput(struct socket *, struct sockopt *); 292 static int unp_connect(struct socket *, struct sockaddr *, 293 struct thread *); 294 static int unp_connectat(int, struct socket *, struct sockaddr *, 295 struct thread *, bool); 296 static void unp_connect2(struct socket *, struct socket *, bool); 297 static void unp_disconnect(struct unpcb *unp, struct unpcb *unp2); 298 static void unp_dispose(struct socket *so); 299 static void unp_drop(struct unpcb *); 300 static void unp_gc(__unused void *, int); 301 static void unp_scan(struct mbuf *, void (*)(struct filedescent **, int)); 302 static void unp_discard(struct file *); 303 static void unp_freerights(struct filedescent **, int); 304 static int unp_internalize(struct mbuf *, struct mchain *, 305 struct thread *); 306 static void unp_internalize_fp(struct file *); 307 static int unp_externalize(struct mbuf *, struct mbuf **, int); 308 static int unp_externalize_fp(struct file *); 309 static void unp_addsockcred(struct thread *, struct mchain *, int); 310 static void unp_process_defers(void * __unused, int); 311 312 static void uipc_wrknl_lock(void *); 313 static void uipc_wrknl_unlock(void *); 314 static void uipc_wrknl_assert_lock(void *, int); 315 316 static void 317 unp_pcb_hold(struct unpcb *unp) 318 { 319 u_int old __unused; 320 321 old = refcount_acquire(&unp->unp_refcount); 322 KASSERT(old > 0, ("%s: unpcb %p has no references", __func__, unp)); 323 } 324 325 static __result_use_check bool 326 unp_pcb_rele(struct unpcb *unp) 327 { 328 bool ret; 329 330 UNP_PCB_LOCK_ASSERT(unp); 331 332 if ((ret = refcount_release(&unp->unp_refcount))) { 333 UNP_PCB_UNLOCK(unp); 334 UNP_PCB_LOCK_DESTROY(unp); 335 uma_zfree(unp_zone, unp); 336 } 337 return (ret); 338 } 339 340 static void 341 unp_pcb_rele_notlast(struct unpcb *unp) 342 { 343 bool ret __unused; 344 345 ret = refcount_release(&unp->unp_refcount); 346 KASSERT(!ret, ("%s: unpcb %p has no references", __func__, unp)); 347 } 348 349 static void 350 unp_pcb_lock_pair(struct unpcb *unp, struct unpcb *unp2) 351 { 352 UNP_PCB_UNLOCK_ASSERT(unp); 353 UNP_PCB_UNLOCK_ASSERT(unp2); 354 355 if (unp == unp2) { 356 UNP_PCB_LOCK(unp); 357 } else if ((uintptr_t)unp2 > (uintptr_t)unp) { 358 UNP_PCB_LOCK(unp); 359 UNP_PCB_LOCK(unp2); 360 } else { 361 UNP_PCB_LOCK(unp2); 362 UNP_PCB_LOCK(unp); 363 } 364 } 365 366 static void 367 unp_pcb_unlock_pair(struct unpcb *unp, struct unpcb *unp2) 368 { 369 UNP_PCB_UNLOCK(unp); 370 if (unp != unp2) 371 UNP_PCB_UNLOCK(unp2); 372 } 373 374 /* 375 * Try to lock the connected peer of an already locked socket. In some cases 376 * this requires that we unlock the current socket. The pairbusy counter is 377 * used to block concurrent connection attempts while the lock is dropped. The 378 * caller must be careful to revalidate PCB state. 379 */ 380 static struct unpcb * 381 unp_pcb_lock_peer(struct unpcb *unp) 382 { 383 struct unpcb *unp2; 384 385 UNP_PCB_LOCK_ASSERT(unp); 386 unp2 = unp->unp_conn; 387 if (unp2 == NULL) 388 return (NULL); 389 if (__predict_false(unp == unp2)) 390 return (unp); 391 392 UNP_PCB_UNLOCK_ASSERT(unp2); 393 394 if (__predict_true(UNP_PCB_TRYLOCK(unp2))) 395 return (unp2); 396 if ((uintptr_t)unp2 > (uintptr_t)unp) { 397 UNP_PCB_LOCK(unp2); 398 return (unp2); 399 } 400 unp->unp_pairbusy++; 401 unp_pcb_hold(unp2); 402 UNP_PCB_UNLOCK(unp); 403 404 UNP_PCB_LOCK(unp2); 405 UNP_PCB_LOCK(unp); 406 KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL, 407 ("%s: socket %p was reconnected", __func__, unp)); 408 if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) { 409 unp->unp_flags &= ~UNP_WAITING; 410 wakeup(unp); 411 } 412 if (unp_pcb_rele(unp2)) { 413 /* unp2 is unlocked. */ 414 return (NULL); 415 } 416 if (unp->unp_conn == NULL) { 417 UNP_PCB_UNLOCK(unp2); 418 return (NULL); 419 } 420 return (unp2); 421 } 422 423 /* 424 * Try to lock peer of our socket for purposes of sending data to it. 425 */ 426 static int 427 uipc_lock_peer(struct socket *so, struct unpcb **unp2) 428 { 429 struct unpcb *unp; 430 int error; 431 432 unp = sotounpcb(so); 433 UNP_PCB_LOCK(unp); 434 *unp2 = unp_pcb_lock_peer(unp); 435 if (__predict_false(so->so_error != 0)) { 436 error = so->so_error; 437 so->so_error = 0; 438 UNP_PCB_UNLOCK(unp); 439 if (*unp2 != NULL) 440 UNP_PCB_UNLOCK(*unp2); 441 return (error); 442 } 443 if (__predict_false(*unp2 == NULL)) { 444 /* 445 * Different error code for a previously connected socket and 446 * a never connected one. The SS_ISDISCONNECTED is set in the 447 * unp_soisdisconnected() and is synchronized by the pcb lock. 448 */ 449 error = so->so_state & SS_ISDISCONNECTED ? EPIPE : ENOTCONN; 450 UNP_PCB_UNLOCK(unp); 451 return (error); 452 } 453 UNP_PCB_UNLOCK(unp); 454 455 return (0); 456 } 457 458 static void 459 uipc_abort(struct socket *so) 460 { 461 struct unpcb *unp, *unp2; 462 463 unp = sotounpcb(so); 464 KASSERT(unp != NULL, ("uipc_abort: unp == NULL")); 465 UNP_PCB_UNLOCK_ASSERT(unp); 466 467 UNP_PCB_LOCK(unp); 468 unp2 = unp->unp_conn; 469 if (unp2 != NULL) { 470 unp_pcb_hold(unp2); 471 UNP_PCB_UNLOCK(unp); 472 unp_drop(unp2); 473 } else 474 UNP_PCB_UNLOCK(unp); 475 } 476 477 static int 478 uipc_attach(struct socket *so, int proto, struct thread *td) 479 { 480 u_long sendspace, recvspace; 481 struct unpcb *unp; 482 int error; 483 bool locked; 484 485 KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); 486 switch (so->so_type) { 487 case SOCK_DGRAM: 488 STAILQ_INIT(&so->so_rcv.uxdg_mb); 489 STAILQ_INIT(&so->so_snd.uxdg_mb); 490 TAILQ_INIT(&so->so_rcv.uxdg_conns); 491 /* 492 * Since send buffer is either bypassed or is a part 493 * of one-to-many receive buffer, we assign both space 494 * limits to unpdg_recvspace. 495 */ 496 sendspace = recvspace = unpdg_recvspace; 497 break; 498 499 case SOCK_STREAM: 500 sendspace = unpst_sendspace; 501 recvspace = unpst_recvspace; 502 goto common; 503 504 case SOCK_SEQPACKET: 505 sendspace = unpsp_sendspace; 506 recvspace = unpsp_recvspace; 507 common: 508 /* 509 * XXXGL: we need to initialize the mutex with MTX_DUPOK. 510 * Ideally, protocols that have PR_SOCKBUF should be 511 * responsible for mutex initialization officially, and then 512 * this uglyness with mtx_destroy(); mtx_init(); would go away. 513 */ 514 mtx_destroy(&so->so_rcv_mtx); 515 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF | MTX_DUPOK); 516 knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, 517 uipc_wrknl_unlock, uipc_wrknl_assert_lock); 518 STAILQ_INIT(&so->so_rcv.uxst_mbq); 519 break; 520 default: 521 panic("uipc_attach"); 522 } 523 error = soreserve(so, sendspace, recvspace); 524 if (error) 525 return (error); 526 unp = uma_zalloc(unp_zone, M_NOWAIT | M_ZERO); 527 if (unp == NULL) 528 return (ENOBUFS); 529 LIST_INIT(&unp->unp_refs); 530 UNP_PCB_LOCK_INIT(unp); 531 unp->unp_socket = so; 532 so->so_pcb = unp; 533 refcount_init(&unp->unp_refcount, 1); 534 unp->unp_mode = ACCESSPERMS; 535 536 if ((locked = UNP_LINK_WOWNED()) == false) 537 UNP_LINK_WLOCK(); 538 539 unp->unp_gencnt = ++unp_gencnt; 540 unp->unp_ino = ++unp_ino; 541 unp_count++; 542 switch (so->so_type) { 543 case SOCK_STREAM: 544 LIST_INSERT_HEAD(&unp_shead, unp, unp_link); 545 break; 546 547 case SOCK_DGRAM: 548 LIST_INSERT_HEAD(&unp_dhead, unp, unp_link); 549 break; 550 551 case SOCK_SEQPACKET: 552 LIST_INSERT_HEAD(&unp_sphead, unp, unp_link); 553 break; 554 555 default: 556 panic("uipc_attach"); 557 } 558 559 if (locked == false) 560 UNP_LINK_WUNLOCK(); 561 562 return (0); 563 } 564 565 static int 566 uipc_bindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 567 { 568 struct sockaddr_un *soun = (struct sockaddr_un *)nam; 569 struct vattr vattr; 570 int error, namelen; 571 struct nameidata nd; 572 struct unpcb *unp; 573 struct vnode *vp; 574 struct mount *mp; 575 cap_rights_t rights; 576 char *buf; 577 mode_t mode; 578 579 if (nam->sa_family != AF_UNIX) 580 return (EAFNOSUPPORT); 581 582 unp = sotounpcb(so); 583 KASSERT(unp != NULL, ("uipc_bind: unp == NULL")); 584 585 if (soun->sun_len > sizeof(struct sockaddr_un)) 586 return (EINVAL); 587 namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); 588 if (namelen <= 0) 589 return (EINVAL); 590 591 /* 592 * We don't allow simultaneous bind() calls on a single UNIX domain 593 * socket, so flag in-progress operations, and return an error if an 594 * operation is already in progress. 595 * 596 * Historically, we have not allowed a socket to be rebound, so this 597 * also returns an error. Not allowing re-binding simplifies the 598 * implementation and avoids a great many possible failure modes. 599 */ 600 UNP_PCB_LOCK(unp); 601 if (unp->unp_vnode != NULL) { 602 UNP_PCB_UNLOCK(unp); 603 return (EINVAL); 604 } 605 if (unp->unp_flags & UNP_BINDING) { 606 UNP_PCB_UNLOCK(unp); 607 return (EALREADY); 608 } 609 unp->unp_flags |= UNP_BINDING; 610 mode = unp->unp_mode & ~td->td_proc->p_pd->pd_cmask; 611 UNP_PCB_UNLOCK(unp); 612 613 buf = malloc(namelen + 1, M_TEMP, M_WAITOK); 614 bcopy(soun->sun_path, buf, namelen); 615 buf[namelen] = 0; 616 617 restart: 618 NDINIT_ATRIGHTS(&nd, CREATE, NOFOLLOW | LOCKPARENT | NOCACHE, 619 UIO_SYSSPACE, buf, fd, cap_rights_init_one(&rights, CAP_BINDAT)); 620 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */ 621 error = namei(&nd); 622 if (error) 623 goto error; 624 vp = nd.ni_vp; 625 if (vp != NULL || vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) { 626 NDFREE_PNBUF(&nd); 627 if (nd.ni_dvp == vp) 628 vrele(nd.ni_dvp); 629 else 630 vput(nd.ni_dvp); 631 if (vp != NULL) { 632 vrele(vp); 633 error = EADDRINUSE; 634 goto error; 635 } 636 error = vn_start_write(NULL, &mp, V_XSLEEP | V_PCATCH); 637 if (error) 638 goto error; 639 goto restart; 640 } 641 VATTR_NULL(&vattr); 642 vattr.va_type = VSOCK; 643 vattr.va_mode = mode; 644 #ifdef MAC 645 error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd, 646 &vattr); 647 #endif 648 if (error == 0) { 649 /* 650 * The prior lookup may have left LK_SHARED in cn_lkflags, 651 * and VOP_CREATE technically only requires the new vnode to 652 * be locked shared. Most filesystems will return the new vnode 653 * locked exclusive regardless, but we should explicitly 654 * specify that here since we require it and assert to that 655 * effect below. 656 */ 657 nd.ni_cnd.cn_lkflags = (nd.ni_cnd.cn_lkflags & ~LK_SHARED) | 658 LK_EXCLUSIVE; 659 error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr); 660 } 661 NDFREE_PNBUF(&nd); 662 if (error) { 663 VOP_VPUT_PAIR(nd.ni_dvp, NULL, true); 664 vn_finished_write(mp); 665 if (error == ERELOOKUP) 666 goto restart; 667 goto error; 668 } 669 vp = nd.ni_vp; 670 ASSERT_VOP_ELOCKED(vp, "uipc_bind"); 671 soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK); 672 673 UNP_PCB_LOCK(unp); 674 VOP_UNP_BIND(vp, unp); 675 unp->unp_vnode = vp; 676 unp->unp_addr = soun; 677 unp->unp_flags &= ~UNP_BINDING; 678 UNP_PCB_UNLOCK(unp); 679 vref(vp); 680 VOP_VPUT_PAIR(nd.ni_dvp, &vp, true); 681 vn_finished_write(mp); 682 free(buf, M_TEMP); 683 return (0); 684 685 error: 686 UNP_PCB_LOCK(unp); 687 unp->unp_flags &= ~UNP_BINDING; 688 UNP_PCB_UNLOCK(unp); 689 free(buf, M_TEMP); 690 return (error); 691 } 692 693 static int 694 uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td) 695 { 696 697 return (uipc_bindat(AT_FDCWD, so, nam, td)); 698 } 699 700 static int 701 uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 702 { 703 int error; 704 705 KASSERT(td == curthread, ("uipc_connect: td != curthread")); 706 error = unp_connect(so, nam, td); 707 return (error); 708 } 709 710 static int 711 uipc_connectat(int fd, struct socket *so, struct sockaddr *nam, 712 struct thread *td) 713 { 714 int error; 715 716 KASSERT(td == curthread, ("uipc_connectat: td != curthread")); 717 error = unp_connectat(fd, so, nam, td, false); 718 return (error); 719 } 720 721 static void 722 uipc_close(struct socket *so) 723 { 724 struct unpcb *unp, *unp2; 725 struct vnode *vp = NULL; 726 struct mtx *vplock; 727 728 unp = sotounpcb(so); 729 KASSERT(unp != NULL, ("uipc_close: unp == NULL")); 730 731 vplock = NULL; 732 if ((vp = unp->unp_vnode) != NULL) { 733 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 734 mtx_lock(vplock); 735 } 736 UNP_PCB_LOCK(unp); 737 if (vp && unp->unp_vnode == NULL) { 738 mtx_unlock(vplock); 739 vp = NULL; 740 } 741 if (vp != NULL) { 742 VOP_UNP_DETACH(vp); 743 unp->unp_vnode = NULL; 744 } 745 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 746 unp_disconnect(unp, unp2); 747 else 748 UNP_PCB_UNLOCK(unp); 749 if (vp) { 750 mtx_unlock(vplock); 751 vrele(vp); 752 } 753 } 754 755 static int 756 uipc_chmod(struct socket *so, mode_t mode, struct ucred *cred __unused, 757 struct thread *td __unused) 758 { 759 struct unpcb *unp; 760 int error; 761 762 if ((mode & ~ACCESSPERMS) != 0) 763 return (EINVAL); 764 765 error = 0; 766 unp = sotounpcb(so); 767 UNP_PCB_LOCK(unp); 768 if (unp->unp_vnode != NULL || (unp->unp_flags & UNP_BINDING) != 0) 769 error = EINVAL; 770 else 771 unp->unp_mode = mode; 772 UNP_PCB_UNLOCK(unp); 773 return (error); 774 } 775 776 static int 777 uipc_connect2(struct socket *so1, struct socket *so2) 778 { 779 struct unpcb *unp, *unp2; 780 781 if (so1->so_type != so2->so_type) 782 return (EPROTOTYPE); 783 784 unp = so1->so_pcb; 785 KASSERT(unp != NULL, ("uipc_connect2: unp == NULL")); 786 unp2 = so2->so_pcb; 787 KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL")); 788 unp_pcb_lock_pair(unp, unp2); 789 unp_connect2(so1, so2, false); 790 unp_pcb_unlock_pair(unp, unp2); 791 792 return (0); 793 } 794 795 static void 796 uipc_detach(struct socket *so) 797 { 798 struct unpcb *unp, *unp2; 799 struct mtx *vplock; 800 struct vnode *vp; 801 int local_unp_rights; 802 803 unp = sotounpcb(so); 804 KASSERT(unp != NULL, ("uipc_detach: unp == NULL")); 805 806 vp = NULL; 807 vplock = NULL; 808 809 if (!SOLISTENING(so)) 810 unp_dispose(so); 811 812 UNP_LINK_WLOCK(); 813 LIST_REMOVE(unp, unp_link); 814 if (unp->unp_gcflag & UNPGC_DEAD) 815 LIST_REMOVE(unp, unp_dead); 816 unp->unp_gencnt = ++unp_gencnt; 817 --unp_count; 818 UNP_LINK_WUNLOCK(); 819 820 UNP_PCB_UNLOCK_ASSERT(unp); 821 restart: 822 if ((vp = unp->unp_vnode) != NULL) { 823 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 824 mtx_lock(vplock); 825 } 826 UNP_PCB_LOCK(unp); 827 if (unp->unp_vnode != vp && unp->unp_vnode != NULL) { 828 if (vplock) 829 mtx_unlock(vplock); 830 UNP_PCB_UNLOCK(unp); 831 goto restart; 832 } 833 if ((vp = unp->unp_vnode) != NULL) { 834 VOP_UNP_DETACH(vp); 835 unp->unp_vnode = NULL; 836 } 837 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 838 unp_disconnect(unp, unp2); 839 else 840 UNP_PCB_UNLOCK(unp); 841 842 UNP_REF_LIST_LOCK(); 843 while (!LIST_EMPTY(&unp->unp_refs)) { 844 struct unpcb *ref = LIST_FIRST(&unp->unp_refs); 845 846 unp_pcb_hold(ref); 847 UNP_REF_LIST_UNLOCK(); 848 849 MPASS(ref != unp); 850 UNP_PCB_UNLOCK_ASSERT(ref); 851 unp_drop(ref); 852 UNP_REF_LIST_LOCK(); 853 } 854 UNP_REF_LIST_UNLOCK(); 855 856 UNP_PCB_LOCK(unp); 857 local_unp_rights = unp_rights; 858 unp->unp_socket->so_pcb = NULL; 859 unp->unp_socket = NULL; 860 free(unp->unp_addr, M_SONAME); 861 unp->unp_addr = NULL; 862 if (!unp_pcb_rele(unp)) 863 UNP_PCB_UNLOCK(unp); 864 if (vp) { 865 mtx_unlock(vplock); 866 vrele(vp); 867 } 868 if (local_unp_rights) 869 taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); 870 871 switch (so->so_type) { 872 case SOCK_STREAM: 873 case SOCK_SEQPACKET: 874 MPASS(SOLISTENING(so) || (STAILQ_EMPTY(&so->so_rcv.uxst_mbq) && 875 so->so_rcv.uxst_peer == NULL)); 876 break; 877 case SOCK_DGRAM: 878 /* 879 * Everything should have been unlinked/freed by unp_dispose() 880 * and/or unp_disconnect(). 881 */ 882 MPASS(so->so_rcv.uxdg_peeked == NULL); 883 MPASS(STAILQ_EMPTY(&so->so_rcv.uxdg_mb)); 884 MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); 885 MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); 886 } 887 } 888 889 static int 890 uipc_disconnect(struct socket *so) 891 { 892 struct unpcb *unp, *unp2; 893 894 unp = sotounpcb(so); 895 KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL")); 896 897 UNP_PCB_LOCK(unp); 898 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) 899 unp_disconnect(unp, unp2); 900 else 901 UNP_PCB_UNLOCK(unp); 902 return (0); 903 } 904 905 static int 906 uipc_listen(struct socket *so, int backlog, struct thread *td) 907 { 908 struct unpcb *unp; 909 int error; 910 911 MPASS(so->so_type != SOCK_DGRAM); 912 913 /* 914 * Synchronize with concurrent connection attempts. 915 */ 916 error = 0; 917 unp = sotounpcb(so); 918 UNP_PCB_LOCK(unp); 919 if (unp->unp_conn != NULL || (unp->unp_flags & UNP_CONNECTING) != 0) 920 error = EINVAL; 921 else if (unp->unp_vnode == NULL) 922 error = EDESTADDRREQ; 923 if (error != 0) { 924 UNP_PCB_UNLOCK(unp); 925 return (error); 926 } 927 928 SOCK_LOCK(so); 929 error = solisten_proto_check(so); 930 if (error == 0) { 931 cru2xt(td, &unp->unp_peercred); 932 if (!SOLISTENING(so)) { 933 (void)chgsbsize(so->so_cred->cr_uidinfo, 934 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 935 (void)chgsbsize(so->so_cred->cr_uidinfo, 936 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 937 } 938 solisten_proto(so, backlog); 939 } 940 SOCK_UNLOCK(so); 941 UNP_PCB_UNLOCK(unp); 942 return (error); 943 } 944 945 static int 946 uipc_peeraddr(struct socket *so, struct sockaddr *ret) 947 { 948 struct unpcb *unp, *unp2; 949 const struct sockaddr *sa; 950 951 unp = sotounpcb(so); 952 KASSERT(unp != NULL, ("uipc_peeraddr: unp == NULL")); 953 954 UNP_PCB_LOCK(unp); 955 unp2 = unp_pcb_lock_peer(unp); 956 if (unp2 != NULL) { 957 if (unp2->unp_addr != NULL) 958 sa = (struct sockaddr *)unp2->unp_addr; 959 else 960 sa = &sun_noname; 961 bcopy(sa, ret, sa->sa_len); 962 unp_pcb_unlock_pair(unp, unp2); 963 } else { 964 UNP_PCB_UNLOCK(unp); 965 sa = &sun_noname; 966 bcopy(sa, ret, sa->sa_len); 967 } 968 return (0); 969 } 970 971 /* 972 * pr_sosend() called with mbuf instead of uio is a kernel thread. NFS, 973 * netgraph(4) and other subsystems can call into socket code. The 974 * function will condition the mbuf so that it can be safely put onto socket 975 * buffer and calculate its char count and mbuf count. 976 * 977 * Note: we don't support receiving control data from a kernel thread. Our 978 * pr_sosend methods have MPASS() to check that. This may change. 979 */ 980 static void 981 uipc_reset_kernel_mbuf(struct mbuf *m, struct mchain *mc) 982 { 983 984 M_ASSERTPKTHDR(m); 985 986 m_clrprotoflags(m); 987 m_tag_delete_chain(m, NULL); 988 m->m_pkthdr.rcvif = NULL; 989 m->m_pkthdr.flowid = 0; 990 m->m_pkthdr.csum_flags = 0; 991 m->m_pkthdr.fibnum = 0; 992 m->m_pkthdr.rsstype = 0; 993 994 mc_init_m(mc, m); 995 MPASS(m->m_pkthdr.len == mc->mc_len); 996 } 997 998 #ifdef SOCKBUF_DEBUG 999 static inline void 1000 uipc_stream_sbcheck(struct sockbuf *sb) 1001 { 1002 struct mbuf *d; 1003 u_int dacc, dccc, dctl, dmbcnt; 1004 bool notready = false; 1005 1006 dacc = dccc = dctl = dmbcnt = 0; 1007 STAILQ_FOREACH(d, &sb->uxst_mbq, m_stailq) { 1008 if (d == sb->uxst_fnrdy) { 1009 MPASS(d->m_flags & M_NOTREADY); 1010 notready = true; 1011 } 1012 if (d->m_type == MT_CONTROL) 1013 dctl += d->m_len; 1014 else if (d->m_type == MT_DATA) { 1015 dccc += d->m_len; 1016 if (!notready) 1017 dacc += d->m_len; 1018 } else 1019 MPASS(0); 1020 dmbcnt += MSIZE; 1021 if (d->m_flags & M_EXT) 1022 dmbcnt += d->m_ext.ext_size; 1023 if (d->m_stailq.stqe_next == NULL) 1024 MPASS(sb->uxst_mbq.stqh_last == &d->m_stailq.stqe_next); 1025 } 1026 MPASS(sb->uxst_fnrdy == NULL || notready); 1027 MPASS(dacc == sb->sb_acc); 1028 MPASS(dccc == sb->sb_ccc); 1029 MPASS(dctl == sb->sb_ctl); 1030 MPASS(dmbcnt == sb->sb_mbcnt); 1031 (void)STAILQ_EMPTY(&sb->uxst_mbq); 1032 } 1033 #define UIPC_STREAM_SBCHECK(sb) uipc_stream_sbcheck(sb) 1034 #else 1035 #define UIPC_STREAM_SBCHECK(sb) do {} while (0) 1036 #endif 1037 1038 /* 1039 * uipc_stream_sbspace() returns how much a writer can send, limited by char 1040 * count or mbuf memory use, whatever ends first. 1041 * 1042 * An obvious and legitimate reason for a socket having more data than allowed, 1043 * is lowering the limit with setsockopt(SO_RCVBUF) on already full buffer. 1044 * Also, sb_mbcnt may overcommit sb_mbmax in case if previous write observed 1045 * 'space < mbspace', but mchain allocated to hold 'space' bytes of data ended 1046 * up with 'mc_mlen > mbspace'. A typical scenario would be a full buffer with 1047 * writer trying to push in a large write, and a slow reader, that reads just 1048 * a few bytes at a time. In that case writer will keep creating new mbufs 1049 * with mc_split(). These mbufs will carry little chars, but will all point at 1050 * the same cluster, thus each adding cluster size to sb_mbcnt. This means we 1051 * will count same cluster many times potentially underutilizing socket buffer. 1052 * We aren't optimizing towards ineffective readers. Classic socket buffer had 1053 * the same "feature". 1054 */ 1055 static inline u_int 1056 uipc_stream_sbspace(struct sockbuf *sb) 1057 { 1058 u_int space, mbspace; 1059 1060 if (__predict_true(sb->sb_hiwat >= sb->sb_ccc + sb->sb_ctl)) 1061 space = sb->sb_hiwat - sb->sb_ccc - sb->sb_ctl; 1062 else 1063 return (0); 1064 if (__predict_true(sb->sb_mbmax >= sb->sb_mbcnt)) 1065 mbspace = sb->sb_mbmax - sb->sb_mbcnt; 1066 else 1067 return (0); 1068 1069 return (min(space, mbspace)); 1070 } 1071 1072 /* 1073 * UNIX version of generic sbwait() for writes. We wait on peer's receive 1074 * buffer, using our timeout. 1075 */ 1076 static int 1077 uipc_stream_sbwait(struct socket *so, sbintime_t timeo) 1078 { 1079 struct sockbuf *sb = &so->so_rcv; 1080 1081 SOCK_RECVBUF_LOCK_ASSERT(so); 1082 sb->sb_flags |= SB_WAIT; 1083 return (msleep_sbt(&sb->sb_acc, SOCK_RECVBUF_MTX(so), PSOCK | PCATCH, 1084 "sbwait", timeo, 0, 0)); 1085 } 1086 1087 static int 1088 uipc_sosend_stream_or_seqpacket(struct socket *so, struct sockaddr *addr, 1089 struct uio *uio0, struct mbuf *m, struct mbuf *c, int flags, 1090 struct thread *td) 1091 { 1092 struct unpcb *unp2; 1093 struct socket *so2; 1094 struct sockbuf *sb; 1095 struct uio *uio; 1096 struct mchain mc, cmc; 1097 size_t resid, sent; 1098 bool nonblock, eor, aio; 1099 int error; 1100 1101 MPASS((uio0 != NULL && m == NULL) || (m != NULL && uio0 == NULL)); 1102 MPASS(m == NULL || c == NULL); 1103 1104 if (__predict_false(flags & MSG_OOB)) 1105 return (EOPNOTSUPP); 1106 1107 nonblock = (so->so_state & SS_NBIO) || 1108 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1109 eor = flags & MSG_EOR; 1110 1111 mc = MCHAIN_INITIALIZER(&mc); 1112 cmc = MCHAIN_INITIALIZER(&cmc); 1113 sent = 0; 1114 aio = false; 1115 1116 if (m == NULL) { 1117 if (c != NULL && (error = unp_internalize(c, &cmc, td))) 1118 goto out; 1119 /* 1120 * This function may read more data from the uio than it would 1121 * then place on socket. That would leave uio inconsistent 1122 * upon return. Normally uio is allocated on the stack of the 1123 * syscall thread and we don't care about leaving it consistent. 1124 * However, aio(9) will allocate a uio as part of job and will 1125 * use it to track progress. We detect aio(9) checking the 1126 * SB_AIO_RUNNING flag. It is safe to check it without lock 1127 * cause it is set and cleared in the same taskqueue thread. 1128 * 1129 * This check can also produce a false positive: there is 1130 * aio(9) job and also there is a syscall we are serving now. 1131 * No sane software does that, it would leave to a mess in 1132 * the socket buffer, as aio(9) doesn't grab the I/O sx(9). 1133 * But syzkaller can create this mess. For such false positive 1134 * our goal is just don't panic or leak memory. 1135 */ 1136 if (__predict_false(so->so_snd.sb_flags & SB_AIO_RUNNING)) { 1137 uio = cloneuio(uio0); 1138 aio = true; 1139 } else { 1140 uio = uio0; 1141 resid = uio->uio_resid; 1142 } 1143 /* 1144 * Optimization for a case when our send fits into the receive 1145 * buffer - do the copyin before taking any locks, sized to our 1146 * send buffer. Later copyins will also take into account 1147 * space in the peer's receive buffer. 1148 */ 1149 error = mc_uiotomc(&mc, uio, so->so_snd.sb_hiwat, 0, M_WAITOK, 1150 eor ? M_EOR : 0); 1151 if (__predict_false(error)) 1152 goto out2; 1153 } else 1154 uipc_reset_kernel_mbuf(m, &mc); 1155 1156 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 1157 if (error) 1158 goto out2; 1159 1160 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 1161 goto out3; 1162 1163 if (unp2->unp_flags & UNP_WANTCRED_MASK) { 1164 /* 1165 * Credentials are passed only once on SOCK_STREAM and 1166 * SOCK_SEQPACKET (LOCAL_CREDS => WANTCRED_ONESHOT), or 1167 * forever (LOCAL_CREDS_PERSISTENT => WANTCRED_ALWAYS). 1168 */ 1169 unp_addsockcred(td, &cmc, unp2->unp_flags); 1170 unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT; 1171 } 1172 1173 /* 1174 * Cycle through the data to send and available space in the peer's 1175 * receive buffer. Put a reference on the peer socket, so that it 1176 * doesn't get freed while we sbwait(). If peer goes away, we will 1177 * observe the SBS_CANTRCVMORE and our sorele() will finalize peer's 1178 * socket destruction. 1179 */ 1180 so2 = unp2->unp_socket; 1181 soref(so2); 1182 UNP_PCB_UNLOCK(unp2); 1183 sb = &so2->so_rcv; 1184 while (mc.mc_len + cmc.mc_len > 0) { 1185 struct mchain mcnext = MCHAIN_INITIALIZER(&mcnext); 1186 u_int space; 1187 1188 SOCK_RECVBUF_LOCK(so2); 1189 restart: 1190 UIPC_STREAM_SBCHECK(sb); 1191 if (__predict_false(cmc.mc_len > sb->sb_hiwat)) { 1192 SOCK_RECVBUF_UNLOCK(so2); 1193 error = EMSGSIZE; 1194 goto out4; 1195 } 1196 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { 1197 SOCK_RECVBUF_UNLOCK(so2); 1198 error = EPIPE; 1199 goto out4; 1200 } 1201 /* 1202 * Wait on the peer socket receive buffer until we have enough 1203 * space to put at least control. The data is a stream and can 1204 * be put partially, but control is really a datagram. 1205 */ 1206 space = uipc_stream_sbspace(sb); 1207 if (space < sb->sb_lowat || space < cmc.mc_len) { 1208 if (nonblock) { 1209 if (aio) 1210 sb->uxst_flags |= UXST_PEER_AIO; 1211 SOCK_RECVBUF_UNLOCK(so2); 1212 if (aio) { 1213 SOCK_SENDBUF_LOCK(so); 1214 so->so_snd.sb_ccc = 1215 so->so_snd.sb_hiwat - space; 1216 SOCK_SENDBUF_UNLOCK(so); 1217 } 1218 error = EWOULDBLOCK; 1219 goto out4; 1220 } 1221 if ((error = uipc_stream_sbwait(so2, 1222 so->so_snd.sb_timeo)) != 0) { 1223 SOCK_RECVBUF_UNLOCK(so2); 1224 goto out4; 1225 } else 1226 goto restart; 1227 } 1228 MPASS(space >= cmc.mc_len); 1229 space -= cmc.mc_len; 1230 if (space == 0) { 1231 /* There is space only to send control. */ 1232 MPASS(!STAILQ_EMPTY(&cmc.mc_q)); 1233 mcnext = mc; 1234 mc = MCHAIN_INITIALIZER(&mc); 1235 } else if (space < mc.mc_len) { 1236 /* Not enough space. */ 1237 if (__predict_false(mc_split(&mc, &mcnext, space, 1238 M_NOWAIT) == ENOMEM)) { 1239 /* 1240 * If allocation failed use M_WAITOK and merge 1241 * the chain back. Next time mc_split() will 1242 * easily split at the same place. Only if we 1243 * race with setsockopt(SO_RCVBUF) shrinking 1244 * sb_hiwat can this happen more than once. 1245 */ 1246 SOCK_RECVBUF_UNLOCK(so2); 1247 (void)mc_split(&mc, &mcnext, space, M_WAITOK); 1248 mc_concat(&mc, &mcnext); 1249 SOCK_RECVBUF_LOCK(so2); 1250 goto restart; 1251 } 1252 MPASS(mc.mc_len == space); 1253 } 1254 if (!STAILQ_EMPTY(&cmc.mc_q)) { 1255 STAILQ_CONCAT(&sb->uxst_mbq, &cmc.mc_q); 1256 sb->sb_ctl += cmc.mc_len; 1257 sb->sb_mbcnt += cmc.mc_mlen; 1258 cmc.mc_len = 0; 1259 } 1260 sent += mc.mc_len; 1261 if (sb->uxst_fnrdy == NULL) 1262 sb->sb_acc += mc.mc_len; 1263 sb->sb_ccc += mc.mc_len; 1264 sb->sb_mbcnt += mc.mc_mlen; 1265 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 1266 UIPC_STREAM_SBCHECK(sb); 1267 space = uipc_stream_sbspace(sb); 1268 sorwakeup_locked(so2); 1269 if (!STAILQ_EMPTY(&mcnext.mc_q)) { 1270 /* 1271 * Such assignment is unsafe in general, but it is 1272 * safe with !STAILQ_EMPTY(&mcnext.mc_q). In C++ we 1273 * could reload = for STAILQs :) 1274 */ 1275 mc = mcnext; 1276 } else if (uio != NULL && uio->uio_resid > 0) { 1277 /* 1278 * Copyin sum of peer's receive buffer space and our 1279 * sb_hiwat, which is our virtual send buffer size. 1280 * See comment above unpst_sendspace declaration. 1281 * We are reading sb_hiwat locklessly, cause a) we 1282 * don't care about an application that does send(2) 1283 * and setsockopt(2) racing internally, and for an 1284 * application that does this in sequence we will see 1285 * the correct value cause sbsetopt() uses buffer lock 1286 * and we also have already acquired it at least once. 1287 */ 1288 error = mc_uiotomc(&mc, uio, space + 1289 atomic_load_int(&so->so_snd.sb_hiwat), 0, M_WAITOK, 1290 eor ? M_EOR : 0); 1291 if (__predict_false(error)) 1292 goto out4; 1293 } else 1294 mc = MCHAIN_INITIALIZER(&mc); 1295 } 1296 1297 MPASS(STAILQ_EMPTY(&mc.mc_q)); 1298 1299 td->td_ru.ru_msgsnd++; 1300 out4: 1301 sorele(so2); 1302 out3: 1303 SOCK_IO_SEND_UNLOCK(so); 1304 out2: 1305 if (aio) { 1306 freeuio(uio); 1307 uioadvance(uio0, sent); 1308 } else if (uio != NULL) 1309 uio->uio_resid = resid - sent; 1310 if (!mc_empty(&cmc)) 1311 unp_scan(mc_first(&cmc), unp_freerights); 1312 out: 1313 mc_freem(&mc); 1314 mc_freem(&cmc); 1315 1316 return (error); 1317 } 1318 1319 /* 1320 * Wakeup a writer, used by recv(2) and shutdown(2). 1321 * 1322 * @param so Points to a connected stream socket with receive buffer locked 1323 * 1324 * In a blocking mode peer is sleeping on our receive buffer, and we need just 1325 * wakeup(9) on it. But to wake up various event engines, we need to reach 1326 * over to peer's selinfo. This can be safely done as the socket buffer 1327 * receive lock is protecting us from the peer going away. 1328 */ 1329 static void 1330 uipc_wakeup_writer(struct socket *so) 1331 { 1332 struct sockbuf *sb = &so->so_rcv; 1333 struct selinfo *sel; 1334 1335 SOCK_RECVBUF_LOCK_ASSERT(so); 1336 MPASS(sb->uxst_peer != NULL); 1337 1338 sel = &sb->uxst_peer->so_wrsel; 1339 1340 if (sb->uxst_flags & UXST_PEER_SEL) { 1341 selwakeuppri(sel, PSOCK); 1342 /* 1343 * XXXGL: sowakeup() does SEL_WAITING() without locks. 1344 */ 1345 if (!SEL_WAITING(sel)) 1346 sb->uxst_flags &= ~UXST_PEER_SEL; 1347 } 1348 if (sb->sb_flags & SB_WAIT) { 1349 sb->sb_flags &= ~SB_WAIT; 1350 wakeup(&sb->sb_acc); 1351 } 1352 KNOTE_LOCKED(&sel->si_note, 0); 1353 SOCK_RECVBUF_UNLOCK(so); 1354 } 1355 1356 static void 1357 uipc_cantrcvmore(struct socket *so) 1358 { 1359 1360 SOCK_RECVBUF_LOCK(so); 1361 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 1362 selwakeuppri(&so->so_rdsel, PSOCK); 1363 KNOTE_LOCKED(&so->so_rdsel.si_note, 0); 1364 if (so->so_rcv.uxst_peer != NULL) 1365 uipc_wakeup_writer(so); 1366 else 1367 SOCK_RECVBUF_UNLOCK(so); 1368 } 1369 1370 static int 1371 uipc_soreceive_stream_or_seqpacket(struct socket *so, struct sockaddr **psa, 1372 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1373 { 1374 struct sockbuf *sb = &so->so_rcv; 1375 struct mbuf *control, *m, *first, *last, *next; 1376 u_int ctl, space, datalen, mbcnt, lastlen; 1377 int error, flags; 1378 bool nonblock, waitall, peek; 1379 1380 MPASS(mp0 == NULL); 1381 1382 if (psa != NULL) 1383 *psa = NULL; 1384 if (controlp != NULL) 1385 *controlp = NULL; 1386 1387 flags = flagsp != NULL ? *flagsp : 0; 1388 nonblock = (so->so_state & SS_NBIO) || 1389 (flags & (MSG_DONTWAIT | MSG_NBIO)); 1390 peek = flags & MSG_PEEK; 1391 waitall = (flags & MSG_WAITALL) && !peek; 1392 1393 /* 1394 * This check may fail only on a socket that never went through 1395 * connect(2). We can check this locklessly, cause: a) for a new born 1396 * socket we don't care about applications that may race internally 1397 * between connect(2) and recv(2), and b) for a dying socket if we 1398 * miss update by unp_sosidisconnected(), we would still get the check 1399 * correct. For dying socket we would observe SBS_CANTRCVMORE later. 1400 */ 1401 if (__predict_false((atomic_load_short(&so->so_state) & 1402 (SS_ISCONNECTED|SS_ISDISCONNECTED)) == 0)) 1403 return (ENOTCONN); 1404 1405 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 1406 if (__predict_false(error)) 1407 return (error); 1408 1409 restart: 1410 SOCK_RECVBUF_LOCK(so); 1411 UIPC_STREAM_SBCHECK(sb); 1412 while (sb->sb_acc < sb->sb_lowat && 1413 (sb->sb_ctl == 0 || controlp == NULL)) { 1414 if (so->so_error) { 1415 error = so->so_error; 1416 if (!peek) 1417 so->so_error = 0; 1418 SOCK_RECVBUF_UNLOCK(so); 1419 SOCK_IO_RECV_UNLOCK(so); 1420 return (error); 1421 } 1422 if (sb->sb_state & SBS_CANTRCVMORE) { 1423 SOCK_RECVBUF_UNLOCK(so); 1424 SOCK_IO_RECV_UNLOCK(so); 1425 return (0); 1426 } 1427 if (nonblock) { 1428 SOCK_RECVBUF_UNLOCK(so); 1429 SOCK_IO_RECV_UNLOCK(so); 1430 return (EWOULDBLOCK); 1431 } 1432 error = sbwait(so, SO_RCV); 1433 if (error) { 1434 SOCK_RECVBUF_UNLOCK(so); 1435 SOCK_IO_RECV_UNLOCK(so); 1436 return (error); 1437 } 1438 } 1439 1440 MPASS(STAILQ_FIRST(&sb->uxst_mbq)); 1441 MPASS(sb->sb_acc > 0 || sb->sb_ctl > 0); 1442 1443 mbcnt = 0; 1444 ctl = 0; 1445 first = STAILQ_FIRST(&sb->uxst_mbq); 1446 if (first->m_type == MT_CONTROL) { 1447 control = first; 1448 STAILQ_FOREACH_FROM(first, &sb->uxst_mbq, m_stailq) { 1449 if (first->m_type != MT_CONTROL) 1450 break; 1451 ctl += first->m_len; 1452 mbcnt += MSIZE; 1453 if (first->m_flags & M_EXT) 1454 mbcnt += first->m_ext.ext_size; 1455 } 1456 } else 1457 control = NULL; 1458 1459 /* 1460 * Find split point for the next copyout. On exit from the loop: 1461 * last == NULL - socket to be flushed 1462 * last != NULL 1463 * lastlen > last->m_len - uio to be filled, last to be adjusted 1464 * lastlen == 0 - MT_CONTROL, M_EOR or M_NOTREADY encountered 1465 */ 1466 space = uio->uio_resid; 1467 datalen = 0; 1468 for (m = first, last = sb->uxst_fnrdy, lastlen = 0; 1469 m != sb->uxst_fnrdy; 1470 m = STAILQ_NEXT(m, m_stailq)) { 1471 if (m->m_type != MT_DATA) { 1472 last = m; 1473 lastlen = 0; 1474 break; 1475 } 1476 if (space >= m->m_len) { 1477 space -= m->m_len; 1478 datalen += m->m_len; 1479 mbcnt += MSIZE; 1480 if (m->m_flags & M_EXT) 1481 mbcnt += m->m_ext.ext_size; 1482 if (m->m_flags & M_EOR) { 1483 last = STAILQ_NEXT(m, m_stailq); 1484 lastlen = 0; 1485 flags |= MSG_EOR; 1486 break; 1487 } 1488 } else { 1489 datalen += space; 1490 last = m; 1491 lastlen = space; 1492 break; 1493 } 1494 } 1495 1496 UIPC_STREAM_SBCHECK(sb); 1497 if (!peek) { 1498 if (last == NULL) 1499 STAILQ_INIT(&sb->uxst_mbq); 1500 else { 1501 STAILQ_FIRST(&sb->uxst_mbq) = last; 1502 MPASS(last->m_len > lastlen); 1503 last->m_len -= lastlen; 1504 last->m_data += lastlen; 1505 } 1506 MPASS(sb->sb_acc >= datalen); 1507 sb->sb_acc -= datalen; 1508 sb->sb_ccc -= datalen; 1509 MPASS(sb->sb_ctl >= ctl); 1510 sb->sb_ctl -= ctl; 1511 MPASS(sb->sb_mbcnt >= mbcnt); 1512 sb->sb_mbcnt -= mbcnt; 1513 UIPC_STREAM_SBCHECK(sb); 1514 if (__predict_true(sb->uxst_peer != NULL)) { 1515 struct unpcb *unp2; 1516 bool aio; 1517 1518 if ((aio = sb->uxst_flags & UXST_PEER_AIO)) 1519 sb->uxst_flags &= ~UXST_PEER_AIO; 1520 1521 uipc_wakeup_writer(so); 1522 /* 1523 * XXXGL: need to go through uipc_lock_peer() after 1524 * the receive buffer lock dropped, it was protecting 1525 * us from unp_soisdisconnected(). The aio workarounds 1526 * should be refactored to the aio(4) side. 1527 */ 1528 if (aio && uipc_lock_peer(so, &unp2) == 0) { 1529 struct socket *so2 = unp2->unp_socket; 1530 1531 SOCK_SENDBUF_LOCK(so2); 1532 so2->so_snd.sb_ccc -= datalen; 1533 sowakeup_aio(so2, SO_SND); 1534 SOCK_SENDBUF_UNLOCK(so2); 1535 UNP_PCB_UNLOCK(unp2); 1536 } 1537 } else 1538 SOCK_RECVBUF_UNLOCK(so); 1539 } else 1540 SOCK_RECVBUF_UNLOCK(so); 1541 1542 while (control != NULL && control->m_type == MT_CONTROL) { 1543 if (!peek) { 1544 /* 1545 * unp_externalize() failure must abort entire read(2). 1546 * Such failure should also free the problematic 1547 * control, but link back the remaining data to the head 1548 * of the buffer, so that socket is not left in a state 1549 * where it can't progress forward with reading. 1550 * Probability of such a failure is really low, so it 1551 * is fine that we need to perform pretty complex 1552 * operation here to reconstruct the buffer. 1553 */ 1554 error = unp_externalize(control, controlp, flags); 1555 control = m_free(control); 1556 if (__predict_false(error && control != NULL)) { 1557 struct mchain cmc; 1558 1559 mc_init_m(&cmc, control); 1560 1561 SOCK_RECVBUF_LOCK(so); 1562 if (__predict_false( 1563 (sb->sb_state & SBS_CANTRCVMORE) || 1564 cmc.mc_len + sb->sb_ccc + sb->sb_ctl > 1565 sb->sb_hiwat)) { 1566 /* 1567 * While the lock was dropped and we 1568 * were failing in unp_externalize(), 1569 * the peer could has a) disconnected, 1570 * b) filled the buffer so that we 1571 * can't prepend data back. 1572 * These are two edge conditions that 1573 * we just can't handle, so lose the 1574 * data and return the error. 1575 */ 1576 SOCK_RECVBUF_UNLOCK(so); 1577 SOCK_IO_RECV_UNLOCK(so); 1578 unp_scan(mc_first(&cmc), 1579 unp_freerights); 1580 mc_freem(&cmc); 1581 return (error); 1582 } 1583 1584 UIPC_STREAM_SBCHECK(sb); 1585 /* XXXGL: STAILQ_PREPEND */ 1586 STAILQ_CONCAT(&cmc.mc_q, &sb->uxst_mbq); 1587 STAILQ_SWAP(&cmc.mc_q, &sb->uxst_mbq, mbuf); 1588 1589 sb->sb_ctl = sb->sb_acc = sb->sb_ccc = 1590 sb->sb_mbcnt = 0; 1591 STAILQ_FOREACH(m, &sb->uxst_mbq, m_stailq) { 1592 if (m->m_type == MT_DATA) { 1593 sb->sb_acc += m->m_len; 1594 sb->sb_ccc += m->m_len; 1595 } else { 1596 sb->sb_ctl += m->m_len; 1597 } 1598 sb->sb_mbcnt += MSIZE; 1599 if (m->m_flags & M_EXT) 1600 sb->sb_mbcnt += 1601 m->m_ext.ext_size; 1602 } 1603 UIPC_STREAM_SBCHECK(sb); 1604 SOCK_RECVBUF_UNLOCK(so); 1605 SOCK_IO_RECV_UNLOCK(so); 1606 return (error); 1607 } 1608 if (controlp != NULL) { 1609 while (*controlp != NULL) 1610 controlp = &(*controlp)->m_next; 1611 } 1612 } else { 1613 /* 1614 * XXXGL 1615 * 1616 * In MSG_PEEK case control is not externalized. This 1617 * means we are leaking some kernel pointers to the 1618 * userland. They are useless to a law-abiding 1619 * application, but may be useful to a malware. This 1620 * is what the historical implementation in the 1621 * soreceive_generic() did. To be improved? 1622 */ 1623 if (controlp != NULL) { 1624 *controlp = m_copym(control, 0, control->m_len, 1625 M_WAITOK); 1626 controlp = &(*controlp)->m_next; 1627 } 1628 control = STAILQ_NEXT(control, m_stailq); 1629 } 1630 } 1631 1632 for (m = first; m != last; m = next) { 1633 next = STAILQ_NEXT(m, m_stailq); 1634 error = uiomove(mtod(m, char *), m->m_len, uio); 1635 if (__predict_false(error)) { 1636 SOCK_IO_RECV_UNLOCK(so); 1637 if (!peek) 1638 for (; m != last; m = next) { 1639 next = STAILQ_NEXT(m, m_stailq); 1640 m_free(m); 1641 } 1642 return (error); 1643 } 1644 if (!peek) 1645 m_free(m); 1646 } 1647 if (last != NULL && lastlen > 0) { 1648 if (!peek) { 1649 MPASS(!(m->m_flags & M_PKTHDR)); 1650 MPASS(last->m_data - M_START(last) >= lastlen); 1651 error = uiomove(mtod(last, char *) - lastlen, 1652 lastlen, uio); 1653 } else 1654 error = uiomove(mtod(last, char *), lastlen, uio); 1655 if (__predict_false(error)) { 1656 SOCK_IO_RECV_UNLOCK(so); 1657 return (error); 1658 } 1659 } 1660 if (waitall && !(flags & MSG_EOR) && uio->uio_resid > 0) 1661 goto restart; 1662 SOCK_IO_RECV_UNLOCK(so); 1663 1664 if (flagsp != NULL) 1665 *flagsp |= flags; 1666 1667 uio->uio_td->td_ru.ru_msgrcv++; 1668 1669 return (0); 1670 } 1671 1672 static int 1673 uipc_sopoll_stream_or_seqpacket(struct socket *so, int events, 1674 struct thread *td) 1675 { 1676 struct unpcb *unp = sotounpcb(so); 1677 int revents; 1678 1679 UNP_PCB_LOCK(unp); 1680 if (SOLISTENING(so)) { 1681 /* The above check is safe, since conversion to listening uses 1682 * both protocol and socket lock. 1683 */ 1684 SOCK_LOCK(so); 1685 if (!(events & (POLLIN | POLLRDNORM))) 1686 revents = 0; 1687 else if (!TAILQ_EMPTY(&so->sol_comp)) 1688 revents = events & (POLLIN | POLLRDNORM); 1689 else if (so->so_error) 1690 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP; 1691 else { 1692 selrecord(td, &so->so_rdsel); 1693 revents = 0; 1694 } 1695 SOCK_UNLOCK(so); 1696 } else { 1697 if (so->so_state & SS_ISDISCONNECTED) 1698 revents = POLLHUP; 1699 else 1700 revents = 0; 1701 if (events & (POLLIN | POLLRDNORM | POLLRDHUP)) { 1702 SOCK_RECVBUF_LOCK(so); 1703 if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat || 1704 so->so_error || so->so_rerror) 1705 revents |= events & (POLLIN | POLLRDNORM); 1706 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 1707 revents |= events & 1708 (POLLIN | POLLRDNORM | POLLRDHUP); 1709 if (!(revents & (POLLIN | POLLRDNORM | POLLRDHUP))) { 1710 selrecord(td, &so->so_rdsel); 1711 so->so_rcv.sb_flags |= SB_SEL; 1712 } 1713 SOCK_RECVBUF_UNLOCK(so); 1714 } 1715 if (events & (POLLOUT | POLLWRNORM)) { 1716 struct socket *so2 = so->so_rcv.uxst_peer; 1717 1718 if (so2 != NULL) { 1719 struct sockbuf *sb = &so2->so_rcv; 1720 1721 SOCK_RECVBUF_LOCK(so2); 1722 if (uipc_stream_sbspace(sb) >= sb->sb_lowat) 1723 revents |= events & 1724 (POLLOUT | POLLWRNORM); 1725 if (sb->sb_state & SBS_CANTRCVMORE) 1726 revents |= POLLHUP; 1727 if (!(revents & (POLLOUT | POLLWRNORM))) { 1728 so2->so_rcv.uxst_flags |= UXST_PEER_SEL; 1729 selrecord(td, &so->so_wrsel); 1730 } 1731 SOCK_RECVBUF_UNLOCK(so2); 1732 } else 1733 selrecord(td, &so->so_wrsel); 1734 } 1735 } 1736 UNP_PCB_UNLOCK(unp); 1737 return (revents); 1738 } 1739 1740 static void 1741 uipc_wrknl_lock(void *arg) 1742 { 1743 struct socket *so = arg; 1744 struct unpcb *unp = sotounpcb(so); 1745 1746 retry: 1747 if (SOLISTENING(so)) { 1748 SOLISTEN_LOCK(so); 1749 } else { 1750 UNP_PCB_LOCK(unp); 1751 if (__predict_false(SOLISTENING(so))) { 1752 UNP_PCB_UNLOCK(unp); 1753 goto retry; 1754 } 1755 if (so->so_rcv.uxst_peer != NULL) 1756 SOCK_RECVBUF_LOCK(so->so_rcv.uxst_peer); 1757 } 1758 } 1759 1760 static void 1761 uipc_wrknl_unlock(void *arg) 1762 { 1763 struct socket *so = arg; 1764 struct unpcb *unp = sotounpcb(so); 1765 1766 if (SOLISTENING(so)) 1767 SOLISTEN_UNLOCK(so); 1768 else { 1769 if (so->so_rcv.uxst_peer != NULL) 1770 SOCK_RECVBUF_UNLOCK(so->so_rcv.uxst_peer); 1771 UNP_PCB_UNLOCK(unp); 1772 } 1773 } 1774 1775 static void 1776 uipc_wrknl_assert_lock(void *arg, int what) 1777 { 1778 struct socket *so = arg; 1779 1780 if (SOLISTENING(so)) { 1781 if (what == LA_LOCKED) 1782 SOLISTEN_LOCK_ASSERT(so); 1783 else 1784 SOLISTEN_UNLOCK_ASSERT(so); 1785 } else { 1786 /* 1787 * The pr_soreceive method will put a note without owning the 1788 * unp lock, so we can't assert it here. But we can safely 1789 * dereference uxst_peer pointer, since receive buffer lock 1790 * is assumed to be held here. 1791 */ 1792 if (what == LA_LOCKED && so->so_rcv.uxst_peer != NULL) 1793 SOCK_RECVBUF_LOCK_ASSERT(so->so_rcv.uxst_peer); 1794 } 1795 } 1796 1797 static void 1798 uipc_filt_sowdetach(struct knote *kn) 1799 { 1800 struct socket *so = kn->kn_fp->f_data; 1801 1802 uipc_wrknl_lock(so); 1803 knlist_remove(&so->so_wrsel.si_note, kn, 1); 1804 uipc_wrknl_unlock(so); 1805 } 1806 1807 static int 1808 uipc_filt_sowrite(struct knote *kn, long hint) 1809 { 1810 struct socket *so = kn->kn_fp->f_data, *so2; 1811 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1812 1813 if (SOLISTENING(so)) 1814 return (0); 1815 1816 if (unp2 == NULL) { 1817 if (so->so_state & SS_ISDISCONNECTED) { 1818 kn->kn_flags |= EV_EOF; 1819 kn->kn_fflags = so->so_error; 1820 return (1); 1821 } else 1822 return (0); 1823 } 1824 1825 so2 = unp2->unp_socket; 1826 SOCK_RECVBUF_LOCK_ASSERT(so2); 1827 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1828 1829 if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { 1830 kn->kn_flags |= EV_EOF; 1831 return (1); 1832 } else if (kn->kn_sfflags & NOTE_LOWAT) 1833 return (kn->kn_data >= kn->kn_sdata); 1834 else 1835 return (kn->kn_data >= so2->so_rcv.sb_lowat); 1836 } 1837 1838 static int 1839 uipc_filt_soempty(struct knote *kn, long hint) 1840 { 1841 struct socket *so = kn->kn_fp->f_data, *so2; 1842 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; 1843 1844 if (SOLISTENING(so) || unp2 == NULL) 1845 return (1); 1846 1847 so2 = unp2->unp_socket; 1848 SOCK_RECVBUF_LOCK_ASSERT(so2); 1849 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); 1850 1851 return (kn->kn_data == 0 ? 1 : 0); 1852 } 1853 1854 static const struct filterops uipc_write_filtops = { 1855 .f_isfd = 1, 1856 .f_detach = uipc_filt_sowdetach, 1857 .f_event = uipc_filt_sowrite, 1858 .f_copy = knote_triv_copy, 1859 }; 1860 static const struct filterops uipc_empty_filtops = { 1861 .f_isfd = 1, 1862 .f_detach = uipc_filt_sowdetach, 1863 .f_event = uipc_filt_soempty, 1864 .f_copy = knote_triv_copy, 1865 }; 1866 1867 static int 1868 uipc_kqfilter_stream_or_seqpacket(struct socket *so, struct knote *kn) 1869 { 1870 struct unpcb *unp = sotounpcb(so); 1871 struct knlist *knl; 1872 1873 switch (kn->kn_filter) { 1874 case EVFILT_READ: 1875 return (sokqfilter_generic(so, kn)); 1876 case EVFILT_WRITE: 1877 kn->kn_fop = &uipc_write_filtops; 1878 break; 1879 case EVFILT_EMPTY: 1880 kn->kn_fop = &uipc_empty_filtops; 1881 break; 1882 default: 1883 return (EINVAL); 1884 } 1885 1886 knl = &so->so_wrsel.si_note; 1887 UNP_PCB_LOCK(unp); 1888 if (SOLISTENING(so)) { 1889 SOLISTEN_LOCK(so); 1890 knlist_add(knl, kn, 1); 1891 SOLISTEN_UNLOCK(so); 1892 } else { 1893 struct socket *so2 = so->so_rcv.uxst_peer; 1894 1895 if (so2 != NULL) 1896 SOCK_RECVBUF_LOCK(so2); 1897 knlist_add(knl, kn, 1); 1898 if (so2 != NULL) 1899 SOCK_RECVBUF_UNLOCK(so2); 1900 } 1901 UNP_PCB_UNLOCK(unp); 1902 return (0); 1903 } 1904 1905 /* PF_UNIX/SOCK_DGRAM version of sbspace() */ 1906 static inline bool 1907 uipc_dgram_sbspace(struct sockbuf *sb, u_int cc, u_int mbcnt) 1908 { 1909 u_int bleft, mleft; 1910 1911 /* 1912 * Negative space may happen if send(2) is followed by 1913 * setsockopt(SO_SNDBUF/SO_RCVBUF) that shrinks maximum. 1914 */ 1915 if (__predict_false(sb->sb_hiwat < sb->uxdg_cc || 1916 sb->sb_mbmax < sb->uxdg_mbcnt)) 1917 return (false); 1918 1919 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) 1920 return (false); 1921 1922 bleft = sb->sb_hiwat - sb->uxdg_cc; 1923 mleft = sb->sb_mbmax - sb->uxdg_mbcnt; 1924 1925 return (bleft >= cc && mleft >= mbcnt); 1926 } 1927 1928 /* 1929 * PF_UNIX/SOCK_DGRAM send 1930 * 1931 * Allocate a record consisting of 3 mbufs in the sequence of 1932 * from -> control -> data and append it to the socket buffer. 1933 * 1934 * The first mbuf carries sender's name and is a pkthdr that stores 1935 * overall length of datagram, its memory consumption and control length. 1936 */ 1937 #define ctllen PH_loc.thirtytwo[1] 1938 _Static_assert(offsetof(struct pkthdr, memlen) + sizeof(u_int) <= 1939 offsetof(struct pkthdr, ctllen), "unix/dgram can not store ctllen"); 1940 static int 1941 uipc_sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1942 struct mbuf *m, struct mbuf *c, int flags, struct thread *td) 1943 { 1944 struct unpcb *unp, *unp2; 1945 const struct sockaddr *from; 1946 struct socket *so2; 1947 struct sockbuf *sb; 1948 struct mchain cmc = MCHAIN_INITIALIZER(&cmc); 1949 struct mbuf *f; 1950 u_int cc, ctl, mbcnt; 1951 u_int dcc __diagused, dctl __diagused, dmbcnt __diagused; 1952 int error; 1953 1954 MPASS((uio != NULL && m == NULL) || (m != NULL && uio == NULL)); 1955 1956 error = 0; 1957 f = NULL; 1958 1959 if (__predict_false(flags & MSG_OOB)) { 1960 error = EOPNOTSUPP; 1961 goto out; 1962 } 1963 if (m == NULL) { 1964 if (__predict_false(uio->uio_resid > unpdg_maxdgram)) { 1965 error = EMSGSIZE; 1966 goto out; 1967 } 1968 m = m_uiotombuf(uio, M_WAITOK, 0, max_hdr, M_PKTHDR); 1969 if (__predict_false(m == NULL)) { 1970 error = EFAULT; 1971 goto out; 1972 } 1973 f = m_gethdr(M_WAITOK, MT_SONAME); 1974 cc = m->m_pkthdr.len; 1975 mbcnt = MSIZE + m->m_pkthdr.memlen; 1976 if (c != NULL && (error = unp_internalize(c, &cmc, td))) 1977 goto out; 1978 } else { 1979 struct mchain mc; 1980 1981 uipc_reset_kernel_mbuf(m, &mc); 1982 cc = mc.mc_len; 1983 mbcnt = mc.mc_mlen; 1984 if (__predict_false(m->m_pkthdr.len > unpdg_maxdgram)) { 1985 error = EMSGSIZE; 1986 goto out; 1987 } 1988 if ((f = m_gethdr(M_NOWAIT, MT_SONAME)) == NULL) { 1989 error = ENOBUFS; 1990 goto out; 1991 } 1992 } 1993 1994 unp = sotounpcb(so); 1995 MPASS(unp); 1996 1997 /* 1998 * XXXGL: would be cool to fully remove so_snd out of the equation 1999 * and avoid this lock, which is not only extraneous, but also being 2000 * released, thus still leaving possibility for a race. We can easily 2001 * handle SBS_CANTSENDMORE/SS_ISCONNECTED complement in unpcb, but it 2002 * is more difficult to invent something to handle so_error. 2003 */ 2004 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags)); 2005 if (error) 2006 goto out2; 2007 SOCK_SENDBUF_LOCK(so); 2008 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2009 SOCK_SENDBUF_UNLOCK(so); 2010 error = EPIPE; 2011 goto out3; 2012 } 2013 if (so->so_error != 0) { 2014 error = so->so_error; 2015 so->so_error = 0; 2016 SOCK_SENDBUF_UNLOCK(so); 2017 goto out3; 2018 } 2019 if (((so->so_state & SS_ISCONNECTED) == 0) && addr == NULL) { 2020 SOCK_SENDBUF_UNLOCK(so); 2021 error = EDESTADDRREQ; 2022 goto out3; 2023 } 2024 SOCK_SENDBUF_UNLOCK(so); 2025 2026 if (addr != NULL) { 2027 if ((error = unp_connectat(AT_FDCWD, so, addr, td, true))) 2028 goto out3; 2029 UNP_PCB_LOCK_ASSERT(unp); 2030 unp2 = unp->unp_conn; 2031 UNP_PCB_LOCK_ASSERT(unp2); 2032 } else { 2033 UNP_PCB_LOCK(unp); 2034 unp2 = unp_pcb_lock_peer(unp); 2035 if (unp2 == NULL) { 2036 UNP_PCB_UNLOCK(unp); 2037 error = ENOTCONN; 2038 goto out3; 2039 } 2040 } 2041 2042 if (unp2->unp_flags & UNP_WANTCRED_MASK) 2043 unp_addsockcred(td, &cmc, unp2->unp_flags); 2044 if (unp->unp_addr != NULL) 2045 from = (struct sockaddr *)unp->unp_addr; 2046 else 2047 from = &sun_noname; 2048 f->m_len = from->sa_len; 2049 MPASS(from->sa_len <= MLEN); 2050 bcopy(from, mtod(f, void *), from->sa_len); 2051 2052 /* 2053 * Concatenate mbufs: from -> control -> data. 2054 * Save overall cc and mbcnt in "from" mbuf. 2055 */ 2056 if (!STAILQ_EMPTY(&cmc.mc_q)) { 2057 f->m_next = mc_first(&cmc); 2058 mc_last(&cmc)->m_next = m; 2059 /* XXXGL: This is dirty as well as rollback after ENOBUFS. */ 2060 STAILQ_INIT(&cmc.mc_q); 2061 } else 2062 f->m_next = m; 2063 m = NULL; 2064 ctl = f->m_len + cmc.mc_len; 2065 mbcnt += cmc.mc_mlen; 2066 #ifdef INVARIANTS 2067 dcc = dctl = dmbcnt = 0; 2068 for (struct mbuf *mb = f; mb != NULL; mb = mb->m_next) { 2069 if (mb->m_type == MT_DATA) 2070 dcc += mb->m_len; 2071 else 2072 dctl += mb->m_len; 2073 dmbcnt += MSIZE; 2074 if (mb->m_flags & M_EXT) 2075 dmbcnt += mb->m_ext.ext_size; 2076 } 2077 MPASS(dcc == cc); 2078 MPASS(dctl == ctl); 2079 MPASS(dmbcnt == mbcnt); 2080 #endif 2081 f->m_pkthdr.len = cc + ctl; 2082 f->m_pkthdr.memlen = mbcnt; 2083 f->m_pkthdr.ctllen = ctl; 2084 2085 /* 2086 * Destination socket buffer selection. 2087 * 2088 * Unconnected sends, when !(so->so_state & SS_ISCONNECTED) and the 2089 * destination address is supplied, create a temporary connection for 2090 * the run time of the function (see call to unp_connectat() above and 2091 * to unp_disconnect() below). We distinguish them by condition of 2092 * (addr != NULL). We intentionally avoid adding 'bool connected' for 2093 * that condition, since, again, through the run time of this code we 2094 * are always connected. For such "unconnected" sends, the destination 2095 * buffer would be the receive buffer of destination socket so2. 2096 * 2097 * For connected sends, data lands on the send buffer of the sender's 2098 * socket "so". Then, if we just added the very first datagram 2099 * on this send buffer, we need to add the send buffer on to the 2100 * receiving socket's buffer list. We put ourselves on top of the 2101 * list. Such logic gives infrequent senders priority over frequent 2102 * senders. 2103 * 2104 * Note on byte count management. As long as event methods kevent(2), 2105 * select(2) are not protocol specific (yet), we need to maintain 2106 * meaningful values on the receive buffer. So, the receive buffer 2107 * would accumulate counters from all connected buffers potentially 2108 * having sb_ccc > sb_hiwat or sb_mbcnt > sb_mbmax. 2109 */ 2110 so2 = unp2->unp_socket; 2111 sb = (addr == NULL) ? &so->so_snd : &so2->so_rcv; 2112 SOCK_RECVBUF_LOCK(so2); 2113 if (uipc_dgram_sbspace(sb, cc + ctl, mbcnt)) { 2114 if (addr == NULL && STAILQ_EMPTY(&sb->uxdg_mb)) 2115 TAILQ_INSERT_HEAD(&so2->so_rcv.uxdg_conns, &so->so_snd, 2116 uxdg_clist); 2117 STAILQ_INSERT_TAIL(&sb->uxdg_mb, f, m_stailqpkt); 2118 sb->uxdg_cc += cc + ctl; 2119 sb->uxdg_ctl += ctl; 2120 sb->uxdg_mbcnt += mbcnt; 2121 so2->so_rcv.sb_acc += cc + ctl; 2122 so2->so_rcv.sb_ccc += cc + ctl; 2123 so2->so_rcv.sb_ctl += ctl; 2124 so2->so_rcv.sb_mbcnt += mbcnt; 2125 sorwakeup_locked(so2); 2126 f = NULL; 2127 } else { 2128 soroverflow_locked(so2); 2129 error = ENOBUFS; 2130 if (f->m_next->m_type == MT_CONTROL) { 2131 STAILQ_FIRST(&cmc.mc_q) = f->m_next; 2132 f->m_next = NULL; 2133 } 2134 } 2135 2136 if (addr != NULL) 2137 unp_disconnect(unp, unp2); 2138 else 2139 unp_pcb_unlock_pair(unp, unp2); 2140 2141 td->td_ru.ru_msgsnd++; 2142 2143 out3: 2144 SOCK_IO_SEND_UNLOCK(so); 2145 out2: 2146 if (!mc_empty(&cmc)) 2147 unp_scan(mc_first(&cmc), unp_freerights); 2148 out: 2149 if (f) 2150 m_freem(f); 2151 mc_freem(&cmc); 2152 if (m) 2153 m_freem(m); 2154 2155 return (error); 2156 } 2157 2158 /* 2159 * PF_UNIX/SOCK_DGRAM receive with MSG_PEEK. 2160 * The mbuf has already been unlinked from the uxdg_mb of socket buffer 2161 * and needs to be linked onto uxdg_peeked of receive socket buffer. 2162 */ 2163 static int 2164 uipc_peek_dgram(struct socket *so, struct mbuf *m, struct sockaddr **psa, 2165 struct uio *uio, struct mbuf **controlp, int *flagsp) 2166 { 2167 ssize_t len = 0; 2168 int error; 2169 2170 so->so_rcv.uxdg_peeked = m; 2171 so->so_rcv.uxdg_cc += m->m_pkthdr.len; 2172 so->so_rcv.uxdg_ctl += m->m_pkthdr.ctllen; 2173 so->so_rcv.uxdg_mbcnt += m->m_pkthdr.memlen; 2174 SOCK_RECVBUF_UNLOCK(so); 2175 2176 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2177 if (psa != NULL) 2178 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2179 2180 m = m->m_next; 2181 KASSERT(m, ("%s: no data or control after soname", __func__)); 2182 2183 /* 2184 * With MSG_PEEK the control isn't executed, just copied. 2185 */ 2186 while (m != NULL && m->m_type == MT_CONTROL) { 2187 if (controlp != NULL) { 2188 *controlp = m_copym(m, 0, m->m_len, M_WAITOK); 2189 controlp = &(*controlp)->m_next; 2190 } 2191 m = m->m_next; 2192 } 2193 KASSERT(m == NULL || m->m_type == MT_DATA, 2194 ("%s: not MT_DATA mbuf %p", __func__, m)); 2195 while (m != NULL && uio->uio_resid > 0) { 2196 len = uio->uio_resid; 2197 if (len > m->m_len) 2198 len = m->m_len; 2199 error = uiomove(mtod(m, char *), (int)len, uio); 2200 if (error) { 2201 SOCK_IO_RECV_UNLOCK(so); 2202 return (error); 2203 } 2204 if (len == m->m_len) 2205 m = m->m_next; 2206 } 2207 SOCK_IO_RECV_UNLOCK(so); 2208 2209 if (flagsp != NULL) { 2210 if (m != NULL) { 2211 if (*flagsp & MSG_TRUNC) { 2212 /* Report real length of the packet */ 2213 uio->uio_resid -= m_length(m, NULL) - len; 2214 } 2215 *flagsp |= MSG_TRUNC; 2216 } else 2217 *flagsp &= ~MSG_TRUNC; 2218 } 2219 2220 return (0); 2221 } 2222 2223 /* 2224 * PF_UNIX/SOCK_DGRAM receive 2225 */ 2226 static int 2227 uipc_soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2228 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2229 { 2230 struct sockbuf *sb = NULL; 2231 struct mbuf *m; 2232 int flags, error; 2233 ssize_t len = 0; 2234 bool nonblock; 2235 2236 MPASS(mp0 == NULL); 2237 2238 if (psa != NULL) 2239 *psa = NULL; 2240 if (controlp != NULL) 2241 *controlp = NULL; 2242 2243 flags = flagsp != NULL ? *flagsp : 0; 2244 nonblock = (so->so_state & SS_NBIO) || 2245 (flags & (MSG_DONTWAIT | MSG_NBIO)); 2246 2247 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags)); 2248 if (__predict_false(error)) 2249 return (error); 2250 2251 /* 2252 * Loop blocking while waiting for a datagram. Prioritize connected 2253 * peers over unconnected sends. Set sb to selected socket buffer 2254 * containing an mbuf on exit from the wait loop. A datagram that 2255 * had already been peeked at has top priority. 2256 */ 2257 SOCK_RECVBUF_LOCK(so); 2258 while ((m = so->so_rcv.uxdg_peeked) == NULL && 2259 (sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) == NULL && 2260 (m = STAILQ_FIRST(&so->so_rcv.uxdg_mb)) == NULL) { 2261 if (so->so_error) { 2262 error = so->so_error; 2263 if (!(flags & MSG_PEEK)) 2264 so->so_error = 0; 2265 SOCK_RECVBUF_UNLOCK(so); 2266 SOCK_IO_RECV_UNLOCK(so); 2267 return (error); 2268 } 2269 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2270 uio->uio_resid == 0) { 2271 SOCK_RECVBUF_UNLOCK(so); 2272 SOCK_IO_RECV_UNLOCK(so); 2273 return (0); 2274 } 2275 if (nonblock) { 2276 SOCK_RECVBUF_UNLOCK(so); 2277 SOCK_IO_RECV_UNLOCK(so); 2278 return (EWOULDBLOCK); 2279 } 2280 error = sbwait(so, SO_RCV); 2281 if (error) { 2282 SOCK_RECVBUF_UNLOCK(so); 2283 SOCK_IO_RECV_UNLOCK(so); 2284 return (error); 2285 } 2286 } 2287 2288 if (sb == NULL) 2289 sb = &so->so_rcv; 2290 else if (m == NULL) 2291 m = STAILQ_FIRST(&sb->uxdg_mb); 2292 else 2293 MPASS(m == so->so_rcv.uxdg_peeked); 2294 2295 MPASS(sb->uxdg_cc > 0); 2296 M_ASSERTPKTHDR(m); 2297 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); 2298 2299 if (uio->uio_td) 2300 uio->uio_td->td_ru.ru_msgrcv++; 2301 2302 if (__predict_true(m != so->so_rcv.uxdg_peeked)) { 2303 STAILQ_REMOVE_HEAD(&sb->uxdg_mb, m_stailqpkt); 2304 if (STAILQ_EMPTY(&sb->uxdg_mb) && sb != &so->so_rcv) 2305 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 2306 } else 2307 so->so_rcv.uxdg_peeked = NULL; 2308 2309 sb->uxdg_cc -= m->m_pkthdr.len; 2310 sb->uxdg_ctl -= m->m_pkthdr.ctllen; 2311 sb->uxdg_mbcnt -= m->m_pkthdr.memlen; 2312 2313 if (__predict_false(flags & MSG_PEEK)) 2314 return (uipc_peek_dgram(so, m, psa, uio, controlp, flagsp)); 2315 2316 so->so_rcv.sb_acc -= m->m_pkthdr.len; 2317 so->so_rcv.sb_ccc -= m->m_pkthdr.len; 2318 so->so_rcv.sb_ctl -= m->m_pkthdr.ctllen; 2319 so->so_rcv.sb_mbcnt -= m->m_pkthdr.memlen; 2320 SOCK_RECVBUF_UNLOCK(so); 2321 2322 if (psa != NULL) 2323 *psa = sodupsockaddr(mtod(m, struct sockaddr *), M_WAITOK); 2324 m = m_free(m); 2325 KASSERT(m, ("%s: no data or control after soname", __func__)); 2326 2327 /* 2328 * Packet to copyout() is now in 'm' and it is disconnected from the 2329 * queue. 2330 * 2331 * Process one or more MT_CONTROL mbufs present before any data mbufs 2332 * in the first mbuf chain on the socket buffer. We call into the 2333 * unp_externalize() to perform externalization (or freeing if 2334 * controlp == NULL). In some cases there can be only MT_CONTROL mbufs 2335 * without MT_DATA mbufs. 2336 */ 2337 while (m != NULL && m->m_type == MT_CONTROL) { 2338 error = unp_externalize(m, controlp, flags); 2339 m = m_free(m); 2340 if (error != 0) { 2341 SOCK_IO_RECV_UNLOCK(so); 2342 unp_scan(m, unp_freerights); 2343 m_freem(m); 2344 return (error); 2345 } 2346 if (controlp != NULL) { 2347 while (*controlp != NULL) 2348 controlp = &(*controlp)->m_next; 2349 } 2350 } 2351 KASSERT(m == NULL || m->m_type == MT_DATA, 2352 ("%s: not MT_DATA mbuf %p", __func__, m)); 2353 while (m != NULL && uio->uio_resid > 0) { 2354 len = uio->uio_resid; 2355 if (len > m->m_len) 2356 len = m->m_len; 2357 error = uiomove(mtod(m, char *), (int)len, uio); 2358 if (error) { 2359 SOCK_IO_RECV_UNLOCK(so); 2360 m_freem(m); 2361 return (error); 2362 } 2363 if (len == m->m_len) 2364 m = m_free(m); 2365 else { 2366 m->m_data += len; 2367 m->m_len -= len; 2368 } 2369 } 2370 SOCK_IO_RECV_UNLOCK(so); 2371 2372 if (m != NULL) { 2373 if (flagsp != NULL) { 2374 if (flags & MSG_TRUNC) { 2375 /* Report real length of the packet */ 2376 uio->uio_resid -= m_length(m, NULL); 2377 } 2378 *flagsp |= MSG_TRUNC; 2379 } 2380 m_freem(m); 2381 } else if (flagsp != NULL) 2382 *flagsp &= ~MSG_TRUNC; 2383 2384 return (0); 2385 } 2386 2387 static int 2388 uipc_sendfile_wait(struct socket *so, off_t need, int *space) 2389 { 2390 struct unpcb *unp2; 2391 struct socket *so2; 2392 struct sockbuf *sb; 2393 bool nonblock, sockref; 2394 int error; 2395 2396 MPASS(so->so_type == SOCK_STREAM); 2397 MPASS(need > 0); 2398 MPASS(space != NULL); 2399 2400 nonblock = so->so_state & SS_NBIO; 2401 sockref = false; 2402 2403 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) 2404 return (ENOTCONN); 2405 2406 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2407 return (error); 2408 2409 so2 = unp2->unp_socket; 2410 sb = &so2->so_rcv; 2411 SOCK_RECVBUF_LOCK(so2); 2412 UNP_PCB_UNLOCK(unp2); 2413 while ((*space = uipc_stream_sbspace(sb)) < need && 2414 (*space < so->so_snd.sb_hiwat / 2)) { 2415 UIPC_STREAM_SBCHECK(sb); 2416 if (nonblock) { 2417 SOCK_RECVBUF_UNLOCK(so2); 2418 return (EAGAIN); 2419 } 2420 if (!sockref) 2421 soref(so2); 2422 error = uipc_stream_sbwait(so2, so->so_snd.sb_timeo); 2423 if (error == 0 && 2424 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) 2425 error = EPIPE; 2426 if (error) { 2427 SOCK_RECVBUF_UNLOCK(so2); 2428 sorele(so2); 2429 return (error); 2430 } 2431 } 2432 UIPC_STREAM_SBCHECK(sb); 2433 SOCK_RECVBUF_UNLOCK(so2); 2434 if (sockref) 2435 sorele(so2); 2436 2437 return (0); 2438 } 2439 2440 /* 2441 * Although this is a pr_send method, for unix(4) it is called only via 2442 * sendfile(2) path. This means we can be sure that mbufs are clear of 2443 * any extra flags and don't require any conditioning. 2444 */ 2445 static int 2446 uipc_sendfile(struct socket *so, int flags, struct mbuf *m, 2447 struct sockaddr *from, struct mbuf *control, struct thread *td) 2448 { 2449 struct mchain mc; 2450 struct unpcb *unp2; 2451 struct socket *so2; 2452 struct sockbuf *sb; 2453 bool notready, wakeup; 2454 int error; 2455 2456 MPASS(so->so_type == SOCK_STREAM); 2457 MPASS(from == NULL && control == NULL); 2458 KASSERT(!(m->m_flags & M_EXTPG), 2459 ("unix(4): TLS sendfile(2) not supported")); 2460 2461 notready = flags & PRUS_NOTREADY; 2462 2463 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) { 2464 error = ENOTCONN; 2465 goto out; 2466 } 2467 2468 if (__predict_false((error = uipc_lock_peer(so, &unp2)) != 0)) 2469 goto out; 2470 2471 mc_init_m(&mc, m); 2472 2473 so2 = unp2->unp_socket; 2474 sb = &so2->so_rcv; 2475 SOCK_RECVBUF_LOCK(so2); 2476 UNP_PCB_UNLOCK(unp2); 2477 UIPC_STREAM_SBCHECK(sb); 2478 sb->sb_ccc += mc.mc_len; 2479 sb->sb_mbcnt += mc.mc_mlen; 2480 if (sb->uxst_fnrdy == NULL) { 2481 if (notready) { 2482 wakeup = false; 2483 STAILQ_FOREACH(m, &mc.mc_q, m_stailq) { 2484 if (m->m_flags & M_NOTREADY) { 2485 sb->uxst_fnrdy = m; 2486 break; 2487 } else { 2488 sb->sb_acc += m->m_len; 2489 wakeup = true; 2490 } 2491 } 2492 } else { 2493 wakeup = true; 2494 sb->sb_acc += mc.mc_len; 2495 } 2496 } else { 2497 wakeup = false; 2498 } 2499 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); 2500 UIPC_STREAM_SBCHECK(sb); 2501 if (wakeup) 2502 sorwakeup_locked(so2); 2503 else 2504 SOCK_RECVBUF_UNLOCK(so2); 2505 2506 return (0); 2507 out: 2508 /* 2509 * In case of not ready data, uipc_ready() is responsible 2510 * for freeing memory. 2511 */ 2512 if (m != NULL && !notready) 2513 m_freem(m); 2514 2515 return (error); 2516 } 2517 2518 static int 2519 uipc_sbready(struct sockbuf *sb, struct mbuf *m, int count) 2520 { 2521 bool blocker; 2522 2523 /* assert locked */ 2524 2525 blocker = (sb->uxst_fnrdy == m); 2526 STAILQ_FOREACH_FROM(m, &sb->uxst_mbq, m_stailq) { 2527 if (count > 0) { 2528 MPASS(m->m_flags & M_NOTREADY); 2529 m->m_flags &= ~M_NOTREADY; 2530 if (blocker) 2531 sb->sb_acc += m->m_len; 2532 count--; 2533 } else if (m->m_flags & M_NOTREADY) 2534 break; 2535 else if (blocker) 2536 sb->sb_acc += m->m_len; 2537 } 2538 if (blocker) { 2539 sb->uxst_fnrdy = m; 2540 return (0); 2541 } else 2542 return (EINPROGRESS); 2543 } 2544 2545 static bool 2546 uipc_ready_scan(struct socket *so, struct mbuf *m, int count, int *errorp) 2547 { 2548 struct mbuf *mb; 2549 struct sockbuf *sb; 2550 2551 SOCK_LOCK(so); 2552 if (SOLISTENING(so)) { 2553 SOCK_UNLOCK(so); 2554 return (false); 2555 } 2556 mb = NULL; 2557 sb = &so->so_rcv; 2558 SOCK_RECVBUF_LOCK(so); 2559 if (sb->uxst_fnrdy != NULL) { 2560 STAILQ_FOREACH(mb, &sb->uxst_mbq, m_stailq) { 2561 if (mb == m) { 2562 *errorp = uipc_sbready(sb, m, count); 2563 break; 2564 } 2565 } 2566 } 2567 SOCK_RECVBUF_UNLOCK(so); 2568 SOCK_UNLOCK(so); 2569 return (mb != NULL); 2570 } 2571 2572 static int 2573 uipc_ready(struct socket *so, struct mbuf *m, int count) 2574 { 2575 struct unpcb *unp, *unp2; 2576 int error; 2577 2578 MPASS(so->so_type == SOCK_STREAM); 2579 2580 if (__predict_true(uipc_lock_peer(so, &unp2) == 0)) { 2581 struct socket *so2; 2582 struct sockbuf *sb; 2583 2584 so2 = unp2->unp_socket; 2585 sb = &so2->so_rcv; 2586 SOCK_RECVBUF_LOCK(so2); 2587 UNP_PCB_UNLOCK(unp2); 2588 UIPC_STREAM_SBCHECK(sb); 2589 error = uipc_sbready(sb, m, count); 2590 UIPC_STREAM_SBCHECK(sb); 2591 if (error == 0) 2592 sorwakeup_locked(so2); 2593 else 2594 SOCK_RECVBUF_UNLOCK(so2); 2595 } else { 2596 /* 2597 * The receiving socket has been disconnected, but may still 2598 * be valid. In this case, the not-ready mbufs are still 2599 * present in its socket buffer, so perform an exhaustive 2600 * search before giving up and freeing the mbufs. 2601 */ 2602 UNP_LINK_RLOCK(); 2603 LIST_FOREACH(unp, &unp_shead, unp_link) { 2604 if (uipc_ready_scan(unp->unp_socket, m, count, &error)) 2605 break; 2606 } 2607 UNP_LINK_RUNLOCK(); 2608 2609 if (unp == NULL) { 2610 for (int i = 0; i < count; i++) 2611 m = m_free(m); 2612 return (ECONNRESET); 2613 } 2614 } 2615 return (error); 2616 } 2617 2618 static int 2619 uipc_sense(struct socket *so, struct stat *sb) 2620 { 2621 struct unpcb *unp; 2622 2623 unp = sotounpcb(so); 2624 KASSERT(unp != NULL, ("uipc_sense: unp == NULL")); 2625 2626 sb->st_blksize = so->so_snd.sb_hiwat; 2627 sb->st_dev = NODEV; 2628 sb->st_ino = unp->unp_ino; 2629 return (0); 2630 } 2631 2632 static int 2633 uipc_shutdown(struct socket *so, enum shutdown_how how) 2634 { 2635 struct unpcb *unp = sotounpcb(so); 2636 int error; 2637 2638 SOCK_LOCK(so); 2639 if (SOLISTENING(so)) { 2640 if (how != SHUT_WR) { 2641 so->so_error = ECONNABORTED; 2642 solisten_wakeup(so); /* unlocks so */ 2643 } else 2644 SOCK_UNLOCK(so); 2645 return (ENOTCONN); 2646 } else if ((so->so_state & 2647 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2648 /* 2649 * POSIX mandates us to just return ENOTCONN when shutdown(2) is 2650 * invoked on a datagram sockets, however historically we would 2651 * actually tear socket down. This is known to be leveraged by 2652 * some applications to unblock process waiting in recv(2) by 2653 * other process that it shares that socket with. Try to meet 2654 * both backward-compatibility and POSIX requirements by forcing 2655 * ENOTCONN but still flushing buffers and performing wakeup(9). 2656 * 2657 * XXXGL: it remains unknown what applications expect this 2658 * behavior and is this isolated to unix/dgram or inet/dgram or 2659 * both. See: D10351, D3039. 2660 */ 2661 error = ENOTCONN; 2662 if (so->so_type != SOCK_DGRAM) { 2663 SOCK_UNLOCK(so); 2664 return (error); 2665 } 2666 } else 2667 error = 0; 2668 SOCK_UNLOCK(so); 2669 2670 switch (how) { 2671 case SHUT_RD: 2672 if (so->so_type == SOCK_DGRAM) 2673 socantrcvmore(so); 2674 else 2675 uipc_cantrcvmore(so); 2676 unp_dispose(so); 2677 break; 2678 case SHUT_RDWR: 2679 if (so->so_type == SOCK_DGRAM) 2680 socantrcvmore(so); 2681 else 2682 uipc_cantrcvmore(so); 2683 unp_dispose(so); 2684 /* FALLTHROUGH */ 2685 case SHUT_WR: 2686 if (so->so_type == SOCK_DGRAM) { 2687 socantsendmore(so); 2688 } else { 2689 UNP_PCB_LOCK(unp); 2690 if (unp->unp_conn != NULL) 2691 uipc_cantrcvmore(unp->unp_conn->unp_socket); 2692 UNP_PCB_UNLOCK(unp); 2693 } 2694 } 2695 wakeup(&so->so_timeo); 2696 2697 return (error); 2698 } 2699 2700 static int 2701 uipc_sockaddr(struct socket *so, struct sockaddr *ret) 2702 { 2703 struct unpcb *unp; 2704 const struct sockaddr *sa; 2705 2706 unp = sotounpcb(so); 2707 KASSERT(unp != NULL, ("uipc_sockaddr: unp == NULL")); 2708 2709 UNP_PCB_LOCK(unp); 2710 if (unp->unp_addr != NULL) 2711 sa = (struct sockaddr *) unp->unp_addr; 2712 else 2713 sa = &sun_noname; 2714 bcopy(sa, ret, sa->sa_len); 2715 UNP_PCB_UNLOCK(unp); 2716 return (0); 2717 } 2718 2719 static int 2720 uipc_ctloutput(struct socket *so, struct sockopt *sopt) 2721 { 2722 struct unpcb *unp; 2723 struct xucred xu; 2724 int error, optval; 2725 2726 if (sopt->sopt_level != SOL_LOCAL) 2727 return (EINVAL); 2728 2729 unp = sotounpcb(so); 2730 KASSERT(unp != NULL, ("uipc_ctloutput: unp == NULL")); 2731 error = 0; 2732 switch (sopt->sopt_dir) { 2733 case SOPT_GET: 2734 switch (sopt->sopt_name) { 2735 case LOCAL_PEERCRED: 2736 UNP_PCB_LOCK(unp); 2737 if (unp->unp_flags & UNP_HAVEPC) 2738 xu = unp->unp_peercred; 2739 else { 2740 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 2741 error = ENOTCONN; 2742 else 2743 error = EINVAL; 2744 } 2745 UNP_PCB_UNLOCK(unp); 2746 if (error == 0) 2747 error = sooptcopyout(sopt, &xu, sizeof(xu)); 2748 break; 2749 2750 case LOCAL_CREDS: 2751 /* Unlocked read. */ 2752 optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0; 2753 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2754 break; 2755 2756 case LOCAL_CREDS_PERSISTENT: 2757 /* Unlocked read. */ 2758 optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0; 2759 error = sooptcopyout(sopt, &optval, sizeof(optval)); 2760 break; 2761 2762 default: 2763 error = EOPNOTSUPP; 2764 break; 2765 } 2766 break; 2767 2768 case SOPT_SET: 2769 switch (sopt->sopt_name) { 2770 case LOCAL_CREDS: 2771 case LOCAL_CREDS_PERSISTENT: 2772 error = sooptcopyin(sopt, &optval, sizeof(optval), 2773 sizeof(optval)); 2774 if (error) 2775 break; 2776 2777 #define OPTSET(bit, exclusive) do { \ 2778 UNP_PCB_LOCK(unp); \ 2779 if (optval) { \ 2780 if ((unp->unp_flags & (exclusive)) != 0) { \ 2781 UNP_PCB_UNLOCK(unp); \ 2782 error = EINVAL; \ 2783 break; \ 2784 } \ 2785 unp->unp_flags |= (bit); \ 2786 } else \ 2787 unp->unp_flags &= ~(bit); \ 2788 UNP_PCB_UNLOCK(unp); \ 2789 } while (0) 2790 2791 switch (sopt->sopt_name) { 2792 case LOCAL_CREDS: 2793 OPTSET(UNP_WANTCRED_ONESHOT, UNP_WANTCRED_ALWAYS); 2794 break; 2795 2796 case LOCAL_CREDS_PERSISTENT: 2797 OPTSET(UNP_WANTCRED_ALWAYS, UNP_WANTCRED_ONESHOT); 2798 break; 2799 2800 default: 2801 break; 2802 } 2803 break; 2804 #undef OPTSET 2805 default: 2806 error = ENOPROTOOPT; 2807 break; 2808 } 2809 break; 2810 2811 default: 2812 error = EOPNOTSUPP; 2813 break; 2814 } 2815 return (error); 2816 } 2817 2818 static int 2819 unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td) 2820 { 2821 2822 return (unp_connectat(AT_FDCWD, so, nam, td, false)); 2823 } 2824 2825 static int 2826 unp_connectat(int fd, struct socket *so, struct sockaddr *nam, 2827 struct thread *td, bool return_locked) 2828 { 2829 struct mtx *vplock; 2830 struct sockaddr_un *soun; 2831 struct vnode *vp; 2832 struct socket *so2; 2833 struct unpcb *unp, *unp2, *unp3; 2834 struct nameidata nd; 2835 char buf[SOCK_MAXADDRLEN]; 2836 struct sockaddr *sa; 2837 cap_rights_t rights; 2838 int error, len; 2839 bool connreq; 2840 2841 CURVNET_ASSERT_SET(); 2842 2843 if (nam->sa_family != AF_UNIX) 2844 return (EAFNOSUPPORT); 2845 if (nam->sa_len > sizeof(struct sockaddr_un)) 2846 return (EINVAL); 2847 len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); 2848 if (len <= 0) 2849 return (EINVAL); 2850 soun = (struct sockaddr_un *)nam; 2851 bcopy(soun->sun_path, buf, len); 2852 buf[len] = 0; 2853 2854 error = 0; 2855 unp = sotounpcb(so); 2856 UNP_PCB_LOCK(unp); 2857 for (;;) { 2858 /* 2859 * Wait for connection state to stabilize. If a connection 2860 * already exists, give up. For datagram sockets, which permit 2861 * multiple consecutive connect(2) calls, upper layers are 2862 * responsible for disconnecting in advance of a subsequent 2863 * connect(2), but this is not synchronized with PCB connection 2864 * state. 2865 * 2866 * Also make sure that no threads are currently attempting to 2867 * lock the peer socket, to ensure that unp_conn cannot 2868 * transition between two valid sockets while locks are dropped. 2869 */ 2870 if (SOLISTENING(so)) 2871 error = EOPNOTSUPP; 2872 else if (unp->unp_conn != NULL) 2873 error = EISCONN; 2874 else if ((unp->unp_flags & UNP_CONNECTING) != 0) { 2875 error = EALREADY; 2876 } 2877 if (error != 0) { 2878 UNP_PCB_UNLOCK(unp); 2879 return (error); 2880 } 2881 if (unp->unp_pairbusy > 0) { 2882 unp->unp_flags |= UNP_WAITING; 2883 mtx_sleep(unp, UNP_PCB_LOCKPTR(unp), 0, "unpeer", 0); 2884 continue; 2885 } 2886 break; 2887 } 2888 unp->unp_flags |= UNP_CONNECTING; 2889 UNP_PCB_UNLOCK(unp); 2890 2891 connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0; 2892 if (connreq) 2893 sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK); 2894 else 2895 sa = NULL; 2896 NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, 2897 UIO_SYSSPACE, buf, fd, cap_rights_init_one(&rights, CAP_CONNECTAT)); 2898 error = namei(&nd); 2899 if (error) 2900 vp = NULL; 2901 else 2902 vp = nd.ni_vp; 2903 ASSERT_VOP_LOCKED(vp, "unp_connect"); 2904 if (error) 2905 goto bad; 2906 NDFREE_PNBUF(&nd); 2907 2908 if (vp->v_type != VSOCK) { 2909 error = ENOTSOCK; 2910 goto bad; 2911 } 2912 #ifdef MAC 2913 error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD); 2914 if (error) 2915 goto bad; 2916 #endif 2917 error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td); 2918 if (error) 2919 goto bad; 2920 2921 unp = sotounpcb(so); 2922 KASSERT(unp != NULL, ("unp_connect: unp == NULL")); 2923 2924 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 2925 mtx_lock(vplock); 2926 VOP_UNP_CONNECT(vp, &unp2); 2927 if (unp2 == NULL) { 2928 error = ECONNREFUSED; 2929 goto bad2; 2930 } 2931 so2 = unp2->unp_socket; 2932 if (so->so_type != so2->so_type) { 2933 error = EPROTOTYPE; 2934 goto bad2; 2935 } 2936 if (connreq) { 2937 if (SOLISTENING(so2)) 2938 so2 = solisten_clone(so2); 2939 else 2940 so2 = NULL; 2941 if (so2 == NULL) { 2942 error = ECONNREFUSED; 2943 goto bad2; 2944 } 2945 if ((error = uipc_attach(so2, 0, NULL)) != 0) { 2946 sodealloc(so2); 2947 goto bad2; 2948 } 2949 unp3 = sotounpcb(so2); 2950 unp_pcb_lock_pair(unp2, unp3); 2951 if (unp2->unp_addr != NULL) { 2952 bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); 2953 unp3->unp_addr = (struct sockaddr_un *) sa; 2954 sa = NULL; 2955 } 2956 2957 unp_copy_peercred(td, unp3, unp, unp2); 2958 2959 UNP_PCB_UNLOCK(unp2); 2960 unp2 = unp3; 2961 2962 /* 2963 * It is safe to block on the PCB lock here since unp2 is 2964 * nascent and cannot be connected to any other sockets. 2965 */ 2966 UNP_PCB_LOCK(unp); 2967 #ifdef MAC 2968 mac_socketpeer_set_from_socket(so, so2); 2969 mac_socketpeer_set_from_socket(so2, so); 2970 #endif 2971 } else { 2972 unp_pcb_lock_pair(unp, unp2); 2973 } 2974 KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && 2975 sotounpcb(so2) == unp2, 2976 ("%s: unp2 %p so2 %p", __func__, unp2, so2)); 2977 unp_connect2(so, so2, connreq); 2978 if (connreq) 2979 (void)solisten_enqueue(so2, SS_ISCONNECTED); 2980 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 2981 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 2982 unp->unp_flags &= ~UNP_CONNECTING; 2983 if (!return_locked) 2984 unp_pcb_unlock_pair(unp, unp2); 2985 bad2: 2986 mtx_unlock(vplock); 2987 bad: 2988 if (vp != NULL) { 2989 /* 2990 * If we are returning locked (called via uipc_sosend_dgram()), 2991 * we need to be sure that vput() won't sleep. This is 2992 * guaranteed by VOP_UNP_CONNECT() call above and unp2 lock. 2993 * SOCK_STREAM/SEQPACKET can't request return_locked (yet). 2994 */ 2995 MPASS(!(return_locked && connreq)); 2996 vput(vp); 2997 } 2998 free(sa, M_SONAME); 2999 if (__predict_false(error)) { 3000 UNP_PCB_LOCK(unp); 3001 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, 3002 ("%s: unp %p has UNP_CONNECTING clear", __func__, unp)); 3003 unp->unp_flags &= ~UNP_CONNECTING; 3004 UNP_PCB_UNLOCK(unp); 3005 } 3006 return (error); 3007 } 3008 3009 /* 3010 * Set socket peer credentials at connection time. 3011 * 3012 * The client's PCB credentials are copied from its process structure. The 3013 * server's PCB credentials are copied from the socket on which it called 3014 * listen(2). uipc_listen cached that process's credentials at the time. 3015 */ 3016 void 3017 unp_copy_peercred(struct thread *td, struct unpcb *client_unp, 3018 struct unpcb *server_unp, struct unpcb *listen_unp) 3019 { 3020 cru2xt(td, &client_unp->unp_peercred); 3021 client_unp->unp_flags |= UNP_HAVEPC; 3022 3023 memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred, 3024 sizeof(server_unp->unp_peercred)); 3025 server_unp->unp_flags |= UNP_HAVEPC; 3026 client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK); 3027 } 3028 3029 /* 3030 * unix/stream & unix/seqpacket version of soisconnected(). 3031 * 3032 * The crucial thing we are doing here is setting up the uxst_peer linkage, 3033 * holding unp and receive buffer locks of the both sockets. The disconnect 3034 * procedure does the same. This gives as a safe way to access the peer in the 3035 * send(2) and recv(2) during the socket lifetime. 3036 * 3037 * The less important thing is event notification of the fact that a socket is 3038 * now connected. It is unusual for a software to put a socket into event 3039 * mechanism before connect(2), but is supposed to be supported. Note that 3040 * there can not be any sleeping I/O on the socket, yet, only presence in the 3041 * select/poll/kevent. 3042 * 3043 * This function can be called via two call paths: 3044 * 1) socketpair(2) - in this case socket has not been yet reported to userland 3045 * and just can't have any event notifications mechanisms set up. The 3046 * 'wakeup' boolean is always false. 3047 * 2) connect(2) of existing socket to a recent clone of a listener: 3048 * 2.1) Socket that connect(2)s will have 'wakeup' true. An application 3049 * could have already put it into event mechanism, is it shall be 3050 * reported as readable and as writable. 3051 * 2.2) Socket that was just cloned with solisten_clone(). Same as 1). 3052 */ 3053 static void 3054 unp_soisconnected(struct socket *so, bool wakeup) 3055 { 3056 struct socket *so2 = sotounpcb(so)->unp_conn->unp_socket; 3057 struct sockbuf *sb; 3058 3059 SOCK_LOCK_ASSERT(so); 3060 UNP_PCB_LOCK_ASSERT(sotounpcb(so)); 3061 UNP_PCB_LOCK_ASSERT(sotounpcb(so2)); 3062 SOCK_RECVBUF_LOCK_ASSERT(so); 3063 SOCK_RECVBUF_LOCK_ASSERT(so2); 3064 3065 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3066 MPASS((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 3067 SS_ISDISCONNECTING)) == 0); 3068 MPASS(so->so_qstate == SQ_NONE); 3069 3070 so->so_state &= ~SS_ISDISCONNECTED; 3071 so->so_state |= SS_ISCONNECTED; 3072 3073 sb = &so2->so_rcv; 3074 sb->uxst_peer = so; 3075 3076 if (wakeup) { 3077 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); 3078 sb = &so->so_rcv; 3079 selwakeuppri(sb->sb_sel, PSOCK); 3080 SOCK_SENDBUF_LOCK_ASSERT(so); 3081 sb = &so->so_snd; 3082 selwakeuppri(sb->sb_sel, PSOCK); 3083 SOCK_SENDBUF_UNLOCK(so); 3084 } 3085 } 3086 3087 static void 3088 unp_connect2(struct socket *so, struct socket *so2, bool wakeup) 3089 { 3090 struct unpcb *unp; 3091 struct unpcb *unp2; 3092 3093 MPASS(so2->so_type == so->so_type); 3094 unp = sotounpcb(so); 3095 KASSERT(unp != NULL, ("unp_connect2: unp == NULL")); 3096 unp2 = sotounpcb(so2); 3097 KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL")); 3098 3099 UNP_PCB_LOCK_ASSERT(unp); 3100 UNP_PCB_LOCK_ASSERT(unp2); 3101 KASSERT(unp->unp_conn == NULL, 3102 ("%s: socket %p is already connected", __func__, unp)); 3103 3104 unp->unp_conn = unp2; 3105 unp_pcb_hold(unp2); 3106 unp_pcb_hold(unp); 3107 switch (so->so_type) { 3108 case SOCK_DGRAM: 3109 UNP_REF_LIST_LOCK(); 3110 LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); 3111 UNP_REF_LIST_UNLOCK(); 3112 soisconnected(so); 3113 break; 3114 3115 case SOCK_STREAM: 3116 case SOCK_SEQPACKET: 3117 KASSERT(unp2->unp_conn == NULL, 3118 ("%s: socket %p is already connected", __func__, unp2)); 3119 unp2->unp_conn = unp; 3120 SOCK_LOCK(so); 3121 SOCK_LOCK(so2); 3122 if (wakeup) /* Avoid LOR with receive buffer lock. */ 3123 SOCK_SENDBUF_LOCK(so); 3124 SOCK_RECVBUF_LOCK(so); 3125 SOCK_RECVBUF_LOCK(so2); 3126 unp_soisconnected(so, wakeup); /* Will unlock send buffer. */ 3127 unp_soisconnected(so2, false); 3128 SOCK_RECVBUF_UNLOCK(so); 3129 SOCK_RECVBUF_UNLOCK(so2); 3130 SOCK_UNLOCK(so); 3131 SOCK_UNLOCK(so2); 3132 break; 3133 3134 default: 3135 panic("unp_connect2"); 3136 } 3137 } 3138 3139 static void 3140 unp_soisdisconnected(struct socket *so) 3141 { 3142 SOCK_LOCK_ASSERT(so); 3143 SOCK_RECVBUF_LOCK_ASSERT(so); 3144 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); 3145 MPASS(!SOLISTENING(so)); 3146 MPASS((so->so_state & (SS_ISCONNECTING | SS_ISDISCONNECTING | 3147 SS_ISDISCONNECTED)) == 0); 3148 MPASS(so->so_state & SS_ISCONNECTED); 3149 3150 so->so_state |= SS_ISDISCONNECTED; 3151 so->so_state &= ~SS_ISCONNECTED; 3152 so->so_rcv.uxst_peer = NULL; 3153 socantrcvmore_locked(so); 3154 } 3155 3156 static void 3157 unp_disconnect(struct unpcb *unp, struct unpcb *unp2) 3158 { 3159 struct socket *so, *so2; 3160 struct mbuf *m = NULL; 3161 #ifdef INVARIANTS 3162 struct unpcb *unptmp; 3163 #endif 3164 3165 UNP_PCB_LOCK_ASSERT(unp); 3166 UNP_PCB_LOCK_ASSERT(unp2); 3167 KASSERT(unp->unp_conn == unp2, 3168 ("%s: unpcb %p is not connected to %p", __func__, unp, unp2)); 3169 3170 unp->unp_conn = NULL; 3171 so = unp->unp_socket; 3172 so2 = unp2->unp_socket; 3173 switch (unp->unp_socket->so_type) { 3174 case SOCK_DGRAM: 3175 /* 3176 * Remove our send socket buffer from the peer's receive buffer. 3177 * Move the data to the receive buffer only if it is empty. 3178 * This is a protection against a scenario where a peer 3179 * connects, floods and disconnects, effectively blocking 3180 * sendto() from unconnected sockets. 3181 */ 3182 SOCK_RECVBUF_LOCK(so2); 3183 if (!STAILQ_EMPTY(&so->so_snd.uxdg_mb)) { 3184 TAILQ_REMOVE(&so2->so_rcv.uxdg_conns, &so->so_snd, 3185 uxdg_clist); 3186 if (__predict_true((so2->so_rcv.sb_state & 3187 SBS_CANTRCVMORE) == 0) && 3188 STAILQ_EMPTY(&so2->so_rcv.uxdg_mb)) { 3189 STAILQ_CONCAT(&so2->so_rcv.uxdg_mb, 3190 &so->so_snd.uxdg_mb); 3191 so2->so_rcv.uxdg_cc += so->so_snd.uxdg_cc; 3192 so2->so_rcv.uxdg_ctl += so->so_snd.uxdg_ctl; 3193 so2->so_rcv.uxdg_mbcnt += so->so_snd.uxdg_mbcnt; 3194 } else { 3195 m = STAILQ_FIRST(&so->so_snd.uxdg_mb); 3196 STAILQ_INIT(&so->so_snd.uxdg_mb); 3197 so2->so_rcv.sb_acc -= so->so_snd.uxdg_cc; 3198 so2->so_rcv.sb_ccc -= so->so_snd.uxdg_cc; 3199 so2->so_rcv.sb_ctl -= so->so_snd.uxdg_ctl; 3200 so2->so_rcv.sb_mbcnt -= so->so_snd.uxdg_mbcnt; 3201 } 3202 /* Note: so may reconnect. */ 3203 so->so_snd.uxdg_cc = 0; 3204 so->so_snd.uxdg_ctl = 0; 3205 so->so_snd.uxdg_mbcnt = 0; 3206 } 3207 SOCK_RECVBUF_UNLOCK(so2); 3208 UNP_REF_LIST_LOCK(); 3209 #ifdef INVARIANTS 3210 LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) { 3211 if (unptmp == unp) 3212 break; 3213 } 3214 KASSERT(unptmp != NULL, 3215 ("%s: %p not found in reflist of %p", __func__, unp, unp2)); 3216 #endif 3217 LIST_REMOVE(unp, unp_reflink); 3218 UNP_REF_LIST_UNLOCK(); 3219 if (so) { 3220 SOCK_LOCK(so); 3221 so->so_state &= ~SS_ISCONNECTED; 3222 SOCK_UNLOCK(so); 3223 } 3224 break; 3225 3226 case SOCK_STREAM: 3227 case SOCK_SEQPACKET: 3228 SOCK_LOCK(so); 3229 SOCK_LOCK(so2); 3230 SOCK_RECVBUF_LOCK(so); 3231 SOCK_RECVBUF_LOCK(so2); 3232 unp_soisdisconnected(so); 3233 MPASS(unp2->unp_conn == unp); 3234 unp2->unp_conn = NULL; 3235 unp_soisdisconnected(so2); 3236 SOCK_UNLOCK(so); 3237 SOCK_UNLOCK(so2); 3238 break; 3239 } 3240 3241 if (unp == unp2) { 3242 unp_pcb_rele_notlast(unp); 3243 if (!unp_pcb_rele(unp)) 3244 UNP_PCB_UNLOCK(unp); 3245 } else { 3246 if (!unp_pcb_rele(unp)) 3247 UNP_PCB_UNLOCK(unp); 3248 if (!unp_pcb_rele(unp2)) 3249 UNP_PCB_UNLOCK(unp2); 3250 } 3251 3252 if (m != NULL) { 3253 unp_scan(m, unp_freerights); 3254 m_freemp(m); 3255 } 3256 } 3257 3258 /* 3259 * unp_pcblist() walks the global list of struct unpcb's to generate a 3260 * pointer list, bumping the refcount on each unpcb. It then copies them out 3261 * sequentially, validating the generation number on each to see if it has 3262 * been detached. All of this is necessary because copyout() may sleep on 3263 * disk I/O. 3264 */ 3265 static int 3266 unp_pcblist(SYSCTL_HANDLER_ARGS) 3267 { 3268 struct unpcb *unp, **unp_list; 3269 unp_gen_t gencnt; 3270 struct xunpgen *xug; 3271 struct unp_head *head; 3272 struct xunpcb *xu; 3273 u_int i; 3274 int error, n; 3275 3276 switch ((intptr_t)arg1) { 3277 case SOCK_STREAM: 3278 head = &unp_shead; 3279 break; 3280 3281 case SOCK_DGRAM: 3282 head = &unp_dhead; 3283 break; 3284 3285 case SOCK_SEQPACKET: 3286 head = &unp_sphead; 3287 break; 3288 3289 default: 3290 panic("unp_pcblist: arg1 %d", (int)(intptr_t)arg1); 3291 } 3292 3293 /* 3294 * The process of preparing the PCB list is too time-consuming and 3295 * resource-intensive to repeat twice on every request. 3296 */ 3297 if (req->oldptr == NULL) { 3298 n = unp_count; 3299 req->oldidx = 2 * (sizeof *xug) 3300 + (n + n/8) * sizeof(struct xunpcb); 3301 return (0); 3302 } 3303 3304 if (req->newptr != NULL) 3305 return (EPERM); 3306 3307 /* 3308 * OK, now we're committed to doing something. 3309 */ 3310 xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK | M_ZERO); 3311 UNP_LINK_RLOCK(); 3312 gencnt = unp_gencnt; 3313 n = unp_count; 3314 UNP_LINK_RUNLOCK(); 3315 3316 xug->xug_len = sizeof *xug; 3317 xug->xug_count = n; 3318 xug->xug_gen = gencnt; 3319 xug->xug_sogen = so_gencnt; 3320 error = SYSCTL_OUT(req, xug, sizeof *xug); 3321 if (error) { 3322 free(xug, M_TEMP); 3323 return (error); 3324 } 3325 3326 unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK); 3327 3328 UNP_LINK_RLOCK(); 3329 for (unp = LIST_FIRST(head), i = 0; unp && i < n; 3330 unp = LIST_NEXT(unp, unp_link)) { 3331 UNP_PCB_LOCK(unp); 3332 if (unp->unp_gencnt <= gencnt) { 3333 if (cr_cansee(req->td->td_ucred, 3334 unp->unp_socket->so_cred)) { 3335 UNP_PCB_UNLOCK(unp); 3336 continue; 3337 } 3338 unp_list[i++] = unp; 3339 unp_pcb_hold(unp); 3340 } 3341 UNP_PCB_UNLOCK(unp); 3342 } 3343 UNP_LINK_RUNLOCK(); 3344 n = i; /* In case we lost some during malloc. */ 3345 3346 error = 0; 3347 xu = malloc(sizeof(*xu), M_TEMP, M_WAITOK | M_ZERO); 3348 for (i = 0; i < n; i++) { 3349 unp = unp_list[i]; 3350 UNP_PCB_LOCK(unp); 3351 if (unp_pcb_rele(unp)) 3352 continue; 3353 3354 if (unp->unp_gencnt <= gencnt) { 3355 xu->xu_len = sizeof *xu; 3356 xu->xu_unpp = (uintptr_t)unp; 3357 /* 3358 * XXX - need more locking here to protect against 3359 * connect/disconnect races for SMP. 3360 */ 3361 if (unp->unp_addr != NULL) 3362 bcopy(unp->unp_addr, &xu->xu_addr, 3363 unp->unp_addr->sun_len); 3364 else 3365 bzero(&xu->xu_addr, sizeof(xu->xu_addr)); 3366 if (unp->unp_conn != NULL && 3367 unp->unp_conn->unp_addr != NULL) 3368 bcopy(unp->unp_conn->unp_addr, 3369 &xu->xu_caddr, 3370 unp->unp_conn->unp_addr->sun_len); 3371 else 3372 bzero(&xu->xu_caddr, sizeof(xu->xu_caddr)); 3373 xu->unp_vnode = (uintptr_t)unp->unp_vnode; 3374 xu->unp_conn = (uintptr_t)unp->unp_conn; 3375 xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs); 3376 xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink); 3377 xu->unp_gencnt = unp->unp_gencnt; 3378 sotoxsocket(unp->unp_socket, &xu->xu_socket); 3379 UNP_PCB_UNLOCK(unp); 3380 error = SYSCTL_OUT(req, xu, sizeof *xu); 3381 } else { 3382 UNP_PCB_UNLOCK(unp); 3383 } 3384 } 3385 free(xu, M_TEMP); 3386 if (!error) { 3387 /* 3388 * Give the user an updated idea of our state. If the 3389 * generation differs from what we told her before, she knows 3390 * that something happened while we were processing this 3391 * request, and it might be necessary to retry. 3392 */ 3393 xug->xug_gen = unp_gencnt; 3394 xug->xug_sogen = so_gencnt; 3395 xug->xug_count = unp_count; 3396 error = SYSCTL_OUT(req, xug, sizeof *xug); 3397 } 3398 free(unp_list, M_TEMP); 3399 free(xug, M_TEMP); 3400 return (error); 3401 } 3402 3403 SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, 3404 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3405 (void *)(intptr_t)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb", 3406 "List of active local datagram sockets"); 3407 SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, 3408 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3409 (void *)(intptr_t)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb", 3410 "List of active local stream sockets"); 3411 SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist, 3412 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, 3413 (void *)(intptr_t)SOCK_SEQPACKET, 0, unp_pcblist, "S,xunpcb", 3414 "List of active local seqpacket sockets"); 3415 3416 static void 3417 unp_drop(struct unpcb *unp) 3418 { 3419 struct socket *so; 3420 struct unpcb *unp2; 3421 3422 /* 3423 * Regardless of whether the socket's peer dropped the connection 3424 * with this socket by aborting or disconnecting, POSIX requires 3425 * that ECONNRESET is returned on next connected send(2) in case of 3426 * a SOCK_DGRAM socket and EPIPE for SOCK_STREAM. 3427 */ 3428 UNP_PCB_LOCK(unp); 3429 if ((so = unp->unp_socket) != NULL) 3430 so->so_error = 3431 so->so_proto->pr_type == SOCK_DGRAM ? ECONNRESET : EPIPE; 3432 if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) { 3433 /* Last reference dropped in unp_disconnect(). */ 3434 unp_pcb_rele_notlast(unp); 3435 unp_disconnect(unp, unp2); 3436 } else if (!unp_pcb_rele(unp)) { 3437 UNP_PCB_UNLOCK(unp); 3438 } 3439 } 3440 3441 static void 3442 unp_freerights(struct filedescent **fdep, int fdcount) 3443 { 3444 struct file *fp; 3445 int i; 3446 3447 KASSERT(fdcount > 0, ("%s: fdcount %d", __func__, fdcount)); 3448 3449 for (i = 0; i < fdcount; i++) { 3450 fp = fdep[i]->fde_file; 3451 filecaps_free(&fdep[i]->fde_caps); 3452 unp_discard(fp); 3453 } 3454 free(fdep[0], M_FILECAPS); 3455 } 3456 3457 static bool 3458 restrict_rights(struct file *fp, struct thread *td) 3459 { 3460 struct prison *prison1, *prison2; 3461 3462 prison1 = fp->f_cred->cr_prison; 3463 prison2 = td->td_ucred->cr_prison; 3464 return (prison1 != prison2 && prison1->pr_root != prison2->pr_root && 3465 prison2 != &prison0); 3466 } 3467 3468 static int 3469 unp_externalize(struct mbuf *control, struct mbuf **controlp, int flags) 3470 { 3471 struct thread *td = curthread; /* XXX */ 3472 struct cmsghdr *cm = mtod(control, struct cmsghdr *); 3473 int *fdp; 3474 struct filedesc *fdesc = td->td_proc->p_fd; 3475 struct filedescent **fdep; 3476 void *data; 3477 socklen_t clen = control->m_len, datalen; 3478 int error, fdflags, newfds; 3479 u_int newlen; 3480 3481 UNP_LINK_UNLOCK_ASSERT(); 3482 3483 fdflags = ((flags & MSG_CMSG_CLOEXEC) ? O_CLOEXEC : 0) | 3484 ((flags & MSG_CMSG_CLOFORK) ? O_CLOFORK : 0); 3485 3486 error = 0; 3487 if (controlp != NULL) /* controlp == NULL => free control messages */ 3488 *controlp = NULL; 3489 while (cm != NULL) { 3490 MPASS(clen >= sizeof(*cm) && clen >= cm->cmsg_len); 3491 3492 data = CMSG_DATA(cm); 3493 datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; 3494 if (cm->cmsg_level == SOL_SOCKET 3495 && cm->cmsg_type == SCM_RIGHTS) { 3496 newfds = datalen / sizeof(*fdep); 3497 if (newfds == 0) 3498 goto next; 3499 fdep = data; 3500 3501 /* If we're not outputting the descriptors free them. */ 3502 if (error || controlp == NULL) { 3503 unp_freerights(fdep, newfds); 3504 goto next; 3505 } 3506 FILEDESC_XLOCK(fdesc); 3507 3508 /* 3509 * Now change each pointer to an fd in the global 3510 * table to an integer that is the index to the local 3511 * fd table entry that we set up to point to the 3512 * global one we are transferring. 3513 */ 3514 newlen = newfds * sizeof(int); 3515 *controlp = sbcreatecontrol(NULL, newlen, 3516 SCM_RIGHTS, SOL_SOCKET, M_WAITOK); 3517 3518 fdp = (int *) 3519 CMSG_DATA(mtod(*controlp, struct cmsghdr *)); 3520 if ((error = fdallocn(td, 0, fdp, newfds))) { 3521 FILEDESC_XUNLOCK(fdesc); 3522 unp_freerights(fdep, newfds); 3523 m_freem(*controlp); 3524 *controlp = NULL; 3525 goto next; 3526 } 3527 for (int i = 0; i < newfds; i++, fdp++) { 3528 struct file *fp; 3529 3530 fp = fdep[i]->fde_file; 3531 _finstall(fdesc, fp, *fdp, fdflags | 3532 (restrict_rights(fp, td) ? 3533 O_RESOLVE_BENEATH : 0), &fdep[i]->fde_caps); 3534 unp_externalize_fp(fp); 3535 } 3536 3537 /* 3538 * The new type indicates that the mbuf data refers to 3539 * kernel resources that may need to be released before 3540 * the mbuf is freed. 3541 */ 3542 m_chtype(*controlp, MT_EXTCONTROL); 3543 FILEDESC_XUNLOCK(fdesc); 3544 free(fdep[0], M_FILECAPS); 3545 } else { 3546 /* We can just copy anything else across. */ 3547 if (error || controlp == NULL) 3548 goto next; 3549 *controlp = sbcreatecontrol(NULL, datalen, 3550 cm->cmsg_type, cm->cmsg_level, M_WAITOK); 3551 bcopy(data, 3552 CMSG_DATA(mtod(*controlp, struct cmsghdr *)), 3553 datalen); 3554 } 3555 controlp = &(*controlp)->m_next; 3556 3557 next: 3558 if (CMSG_SPACE(datalen) < clen) { 3559 clen -= CMSG_SPACE(datalen); 3560 cm = (struct cmsghdr *) 3561 ((caddr_t)cm + CMSG_SPACE(datalen)); 3562 } else { 3563 clen = 0; 3564 cm = NULL; 3565 } 3566 } 3567 3568 return (error); 3569 } 3570 3571 static void 3572 unp_zone_change(void *tag) 3573 { 3574 3575 uma_zone_set_max(unp_zone, maxsockets); 3576 } 3577 3578 #ifdef INVARIANTS 3579 static void 3580 unp_zdtor(void *mem, int size __unused, void *arg __unused) 3581 { 3582 struct unpcb *unp; 3583 3584 unp = mem; 3585 3586 KASSERT(LIST_EMPTY(&unp->unp_refs), 3587 ("%s: unpcb %p has lingering refs", __func__, unp)); 3588 KASSERT(unp->unp_socket == NULL, 3589 ("%s: unpcb %p has socket backpointer", __func__, unp)); 3590 KASSERT(unp->unp_vnode == NULL, 3591 ("%s: unpcb %p has vnode references", __func__, unp)); 3592 KASSERT(unp->unp_conn == NULL, 3593 ("%s: unpcb %p is still connected", __func__, unp)); 3594 KASSERT(unp->unp_addr == NULL, 3595 ("%s: unpcb %p has leaked addr", __func__, unp)); 3596 } 3597 #endif 3598 3599 static void 3600 unp_init(void *arg __unused) 3601 { 3602 uma_dtor dtor; 3603 3604 #ifdef INVARIANTS 3605 dtor = unp_zdtor; 3606 #else 3607 dtor = NULL; 3608 #endif 3609 unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, dtor, 3610 NULL, NULL, UMA_ALIGN_CACHE, 0); 3611 uma_zone_set_max(unp_zone, maxsockets); 3612 uma_zone_set_warning(unp_zone, "kern.ipc.maxsockets limit reached"); 3613 EVENTHANDLER_REGISTER(maxsockets_change, unp_zone_change, 3614 NULL, EVENTHANDLER_PRI_ANY); 3615 LIST_INIT(&unp_dhead); 3616 LIST_INIT(&unp_shead); 3617 LIST_INIT(&unp_sphead); 3618 SLIST_INIT(&unp_defers); 3619 TIMEOUT_TASK_INIT(taskqueue_thread, &unp_gc_task, 0, unp_gc, NULL); 3620 TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL); 3621 UNP_LINK_LOCK_INIT(); 3622 UNP_DEFERRED_LOCK_INIT(); 3623 unp_vp_mtxpool = mtx_pool_create("unp vp mtxpool", 32, MTX_DEF); 3624 } 3625 SYSINIT(unp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_SECOND, unp_init, NULL); 3626 3627 static void 3628 unp_internalize_cleanup_rights(struct mbuf *control) 3629 { 3630 struct cmsghdr *cp; 3631 struct mbuf *m; 3632 void *data; 3633 socklen_t datalen; 3634 3635 for (m = control; m != NULL; m = m->m_next) { 3636 cp = mtod(m, struct cmsghdr *); 3637 if (cp->cmsg_level != SOL_SOCKET || 3638 cp->cmsg_type != SCM_RIGHTS) 3639 continue; 3640 data = CMSG_DATA(cp); 3641 datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data; 3642 unp_freerights(data, datalen / sizeof(struct filedesc *)); 3643 } 3644 } 3645 3646 static int 3647 unp_internalize(struct mbuf *control, struct mchain *mc, struct thread *td) 3648 { 3649 struct proc *p; 3650 struct filedesc *fdesc; 3651 struct bintime *bt; 3652 struct cmsghdr *cm; 3653 struct cmsgcred *cmcred; 3654 struct mbuf *m; 3655 struct filedescent *fde, **fdep, *fdev; 3656 struct file *fp; 3657 struct timeval *tv; 3658 struct timespec *ts; 3659 void *data; 3660 socklen_t clen, datalen; 3661 int i, j, error, *fdp, oldfds; 3662 u_int newlen; 3663 3664 MPASS(control->m_next == NULL); /* COMPAT_OLDSOCK may violate */ 3665 UNP_LINK_UNLOCK_ASSERT(); 3666 3667 p = td->td_proc; 3668 fdesc = p->p_fd; 3669 error = 0; 3670 *mc = MCHAIN_INITIALIZER(mc); 3671 for (clen = control->m_len, cm = mtod(control, struct cmsghdr *), 3672 data = CMSG_DATA(cm); 3673 3674 clen >= sizeof(*cm) && cm->cmsg_level == SOL_SOCKET && 3675 clen >= cm->cmsg_len && cm->cmsg_len >= sizeof(*cm) && 3676 (char *)cm + cm->cmsg_len >= (char *)data; 3677 3678 clen -= min(CMSG_SPACE(datalen), clen), 3679 cm = (struct cmsghdr *) ((char *)cm + CMSG_SPACE(datalen)), 3680 data = CMSG_DATA(cm)) { 3681 datalen = (char *)cm + cm->cmsg_len - (char *)data; 3682 switch (cm->cmsg_type) { 3683 case SCM_CREDS: 3684 m = sbcreatecontrol(NULL, sizeof(*cmcred), SCM_CREDS, 3685 SOL_SOCKET, M_WAITOK); 3686 cmcred = (struct cmsgcred *) 3687 CMSG_DATA(mtod(m, struct cmsghdr *)); 3688 cmcred->cmcred_pid = p->p_pid; 3689 cmcred->cmcred_uid = td->td_ucred->cr_ruid; 3690 cmcred->cmcred_gid = td->td_ucred->cr_rgid; 3691 cmcred->cmcred_euid = td->td_ucred->cr_uid; 3692 _Static_assert(CMGROUP_MAX >= 1, 3693 "Room needed for the effective GID."); 3694 cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups + 1, 3695 CMGROUP_MAX); 3696 cmcred->cmcred_groups[0] = td->td_ucred->cr_gid; 3697 for (i = 1; i < cmcred->cmcred_ngroups; i++) 3698 cmcred->cmcred_groups[i] = 3699 td->td_ucred->cr_groups[i - 1]; 3700 break; 3701 3702 case SCM_RIGHTS: 3703 oldfds = datalen / sizeof (int); 3704 if (oldfds == 0) 3705 continue; 3706 /* On some machines sizeof pointer is bigger than 3707 * sizeof int, so we need to check if data fits into 3708 * single mbuf. We could allocate several mbufs, and 3709 * unp_externalize() should even properly handle that. 3710 * But it is not worth to complicate the code for an 3711 * insane scenario of passing over 200 file descriptors 3712 * at once. 3713 */ 3714 newlen = oldfds * sizeof(fdep[0]); 3715 if (CMSG_SPACE(newlen) > MCLBYTES) { 3716 error = EMSGSIZE; 3717 goto out; 3718 } 3719 /* 3720 * Check that all the FDs passed in refer to legal 3721 * files. If not, reject the entire operation. 3722 */ 3723 fdp = data; 3724 FILEDESC_SLOCK(fdesc); 3725 for (i = 0; i < oldfds; i++, fdp++) { 3726 fp = fget_noref(fdesc, *fdp); 3727 if (fp == NULL) { 3728 FILEDESC_SUNLOCK(fdesc); 3729 error = EBADF; 3730 goto out; 3731 } 3732 if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) { 3733 FILEDESC_SUNLOCK(fdesc); 3734 error = EOPNOTSUPP; 3735 goto out; 3736 } 3737 } 3738 3739 /* 3740 * Now replace the integer FDs with pointers to the 3741 * file structure and capability rights. 3742 */ 3743 m = sbcreatecontrol(NULL, newlen, SCM_RIGHTS, 3744 SOL_SOCKET, M_WAITOK); 3745 fdp = data; 3746 for (i = 0; i < oldfds; i++, fdp++) { 3747 if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) { 3748 fdp = data; 3749 for (j = 0; j < i; j++, fdp++) { 3750 fdrop(fdesc->fd_ofiles[*fdp]. 3751 fde_file, td); 3752 } 3753 FILEDESC_SUNLOCK(fdesc); 3754 error = EBADF; 3755 goto out; 3756 } 3757 } 3758 fdp = data; 3759 fdep = (struct filedescent **) 3760 CMSG_DATA(mtod(m, struct cmsghdr *)); 3761 fdev = malloc(sizeof(*fdev) * oldfds, M_FILECAPS, 3762 M_WAITOK); 3763 for (i = 0; i < oldfds; i++, fdev++, fdp++) { 3764 fde = &fdesc->fd_ofiles[*fdp]; 3765 fdep[i] = fdev; 3766 fdep[i]->fde_file = fde->fde_file; 3767 filecaps_copy(&fde->fde_caps, 3768 &fdep[i]->fde_caps, true); 3769 unp_internalize_fp(fdep[i]->fde_file); 3770 } 3771 FILEDESC_SUNLOCK(fdesc); 3772 break; 3773 3774 case SCM_TIMESTAMP: 3775 m = sbcreatecontrol(NULL, sizeof(*tv), SCM_TIMESTAMP, 3776 SOL_SOCKET, M_WAITOK); 3777 tv = (struct timeval *) 3778 CMSG_DATA(mtod(m, struct cmsghdr *)); 3779 microtime(tv); 3780 break; 3781 3782 case SCM_BINTIME: 3783 m = sbcreatecontrol(NULL, sizeof(*bt), SCM_BINTIME, 3784 SOL_SOCKET, M_WAITOK); 3785 bt = (struct bintime *) 3786 CMSG_DATA(mtod(m, struct cmsghdr *)); 3787 bintime(bt); 3788 break; 3789 3790 case SCM_REALTIME: 3791 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_REALTIME, 3792 SOL_SOCKET, M_WAITOK); 3793 ts = (struct timespec *) 3794 CMSG_DATA(mtod(m, struct cmsghdr *)); 3795 nanotime(ts); 3796 break; 3797 3798 case SCM_MONOTONIC: 3799 m = sbcreatecontrol(NULL, sizeof(*ts), SCM_MONOTONIC, 3800 SOL_SOCKET, M_WAITOK); 3801 ts = (struct timespec *) 3802 CMSG_DATA(mtod(m, struct cmsghdr *)); 3803 nanouptime(ts); 3804 break; 3805 3806 default: 3807 error = EINVAL; 3808 goto out; 3809 } 3810 3811 mc_append(mc, m); 3812 } 3813 if (clen > 0) 3814 error = EINVAL; 3815 3816 out: 3817 if (error != 0) 3818 unp_internalize_cleanup_rights(mc_first(mc)); 3819 m_freem(control); 3820 return (error); 3821 } 3822 3823 static void 3824 unp_addsockcred(struct thread *td, struct mchain *mc, int mode) 3825 { 3826 struct mbuf *m, *n, *n_prev; 3827 const struct cmsghdr *cm; 3828 int ngroups, i, cmsgtype; 3829 size_t ctrlsz; 3830 3831 ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX); 3832 if (mode & UNP_WANTCRED_ALWAYS) { 3833 ctrlsz = SOCKCRED2SIZE(ngroups); 3834 cmsgtype = SCM_CREDS2; 3835 } else { 3836 ctrlsz = SOCKCREDSIZE(ngroups); 3837 cmsgtype = SCM_CREDS; 3838 } 3839 3840 /* XXXGL: uipc_sosend_*() need to be improved so that we can M_WAITOK */ 3841 m = sbcreatecontrol(NULL, ctrlsz, cmsgtype, SOL_SOCKET, M_NOWAIT); 3842 if (m == NULL) 3843 return; 3844 MPASS((m->m_flags & M_EXT) == 0 && m->m_next == NULL); 3845 3846 if (mode & UNP_WANTCRED_ALWAYS) { 3847 struct sockcred2 *sc; 3848 3849 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3850 sc->sc_version = 0; 3851 sc->sc_pid = td->td_proc->p_pid; 3852 sc->sc_uid = td->td_ucred->cr_ruid; 3853 sc->sc_euid = td->td_ucred->cr_uid; 3854 sc->sc_gid = td->td_ucred->cr_rgid; 3855 sc->sc_egid = td->td_ucred->cr_gid; 3856 sc->sc_ngroups = ngroups; 3857 for (i = 0; i < sc->sc_ngroups; i++) 3858 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3859 } else { 3860 struct sockcred *sc; 3861 3862 sc = (void *)CMSG_DATA(mtod(m, struct cmsghdr *)); 3863 sc->sc_uid = td->td_ucred->cr_ruid; 3864 sc->sc_euid = td->td_ucred->cr_uid; 3865 sc->sc_gid = td->td_ucred->cr_rgid; 3866 sc->sc_egid = td->td_ucred->cr_gid; 3867 sc->sc_ngroups = ngroups; 3868 for (i = 0; i < sc->sc_ngroups; i++) 3869 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; 3870 } 3871 3872 /* 3873 * Unlink SCM_CREDS control messages (struct cmsgcred), since just 3874 * created SCM_CREDS control message (struct sockcred) has another 3875 * format. 3876 */ 3877 if (!STAILQ_EMPTY(&mc->mc_q) && cmsgtype == SCM_CREDS) 3878 STAILQ_FOREACH_SAFE(n, &mc->mc_q, m_stailq, n_prev) { 3879 cm = mtod(n, struct cmsghdr *); 3880 if (cm->cmsg_level == SOL_SOCKET && 3881 cm->cmsg_type == SCM_CREDS) { 3882 mc_remove(mc, n); 3883 m_free(n); 3884 } 3885 } 3886 3887 /* Prepend it to the head. */ 3888 mc_prepend(mc, m); 3889 } 3890 3891 static struct unpcb * 3892 fptounp(struct file *fp) 3893 { 3894 struct socket *so; 3895 3896 if (fp->f_type != DTYPE_SOCKET) 3897 return (NULL); 3898 if ((so = fp->f_data) == NULL) 3899 return (NULL); 3900 if (so->so_proto->pr_domain != &localdomain) 3901 return (NULL); 3902 return sotounpcb(so); 3903 } 3904 3905 static void 3906 unp_discard(struct file *fp) 3907 { 3908 struct unp_defer *dr; 3909 3910 if (unp_externalize_fp(fp)) { 3911 dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK); 3912 dr->ud_fp = fp; 3913 UNP_DEFERRED_LOCK(); 3914 SLIST_INSERT_HEAD(&unp_defers, dr, ud_link); 3915 UNP_DEFERRED_UNLOCK(); 3916 atomic_add_int(&unp_defers_count, 1); 3917 taskqueue_enqueue(taskqueue_thread, &unp_defer_task); 3918 } else 3919 closef_nothread(fp); 3920 } 3921 3922 static void 3923 unp_process_defers(void *arg __unused, int pending) 3924 { 3925 struct unp_defer *dr; 3926 SLIST_HEAD(, unp_defer) drl; 3927 int count; 3928 3929 SLIST_INIT(&drl); 3930 for (;;) { 3931 UNP_DEFERRED_LOCK(); 3932 if (SLIST_FIRST(&unp_defers) == NULL) { 3933 UNP_DEFERRED_UNLOCK(); 3934 break; 3935 } 3936 SLIST_SWAP(&unp_defers, &drl, unp_defer); 3937 UNP_DEFERRED_UNLOCK(); 3938 count = 0; 3939 while ((dr = SLIST_FIRST(&drl)) != NULL) { 3940 SLIST_REMOVE_HEAD(&drl, ud_link); 3941 closef_nothread(dr->ud_fp); 3942 free(dr, M_TEMP); 3943 count++; 3944 } 3945 atomic_add_int(&unp_defers_count, -count); 3946 } 3947 } 3948 3949 static void 3950 unp_internalize_fp(struct file *fp) 3951 { 3952 struct unpcb *unp; 3953 3954 UNP_LINK_WLOCK(); 3955 if ((unp = fptounp(fp)) != NULL) { 3956 unp->unp_file = fp; 3957 unp->unp_msgcount++; 3958 } 3959 unp_rights++; 3960 UNP_LINK_WUNLOCK(); 3961 } 3962 3963 static int 3964 unp_externalize_fp(struct file *fp) 3965 { 3966 struct unpcb *unp; 3967 int ret; 3968 3969 UNP_LINK_WLOCK(); 3970 if ((unp = fptounp(fp)) != NULL) { 3971 unp->unp_msgcount--; 3972 ret = 1; 3973 } else 3974 ret = 0; 3975 unp_rights--; 3976 UNP_LINK_WUNLOCK(); 3977 return (ret); 3978 } 3979 3980 /* 3981 * unp_defer indicates whether additional work has been defered for a future 3982 * pass through unp_gc(). It is thread local and does not require explicit 3983 * synchronization. 3984 */ 3985 static int unp_marked; 3986 3987 static void 3988 unp_remove_dead_ref(struct filedescent **fdep, int fdcount) 3989 { 3990 struct unpcb *unp; 3991 struct file *fp; 3992 int i; 3993 3994 /* 3995 * This function can only be called from the gc task. 3996 */ 3997 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 3998 ("%s: not on gc callout", __func__)); 3999 UNP_LINK_LOCK_ASSERT(); 4000 4001 for (i = 0; i < fdcount; i++) { 4002 fp = fdep[i]->fde_file; 4003 if ((unp = fptounp(fp)) == NULL) 4004 continue; 4005 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4006 continue; 4007 unp->unp_gcrefs--; 4008 } 4009 } 4010 4011 static void 4012 unp_restore_undead_ref(struct filedescent **fdep, int fdcount) 4013 { 4014 struct unpcb *unp; 4015 struct file *fp; 4016 int i; 4017 4018 /* 4019 * This function can only be called from the gc task. 4020 */ 4021 KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0, 4022 ("%s: not on gc callout", __func__)); 4023 UNP_LINK_LOCK_ASSERT(); 4024 4025 for (i = 0; i < fdcount; i++) { 4026 fp = fdep[i]->fde_file; 4027 if ((unp = fptounp(fp)) == NULL) 4028 continue; 4029 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) 4030 continue; 4031 unp->unp_gcrefs++; 4032 unp_marked++; 4033 } 4034 } 4035 4036 static void 4037 unp_scan_socket(struct socket *so, void (*op)(struct filedescent **, int)) 4038 { 4039 struct sockbuf *sb; 4040 4041 SOCK_LOCK_ASSERT(so); 4042 4043 if (sotounpcb(so)->unp_gcflag & UNPGC_IGNORE_RIGHTS) 4044 return; 4045 4046 SOCK_RECVBUF_LOCK(so); 4047 switch (so->so_type) { 4048 case SOCK_DGRAM: 4049 unp_scan(STAILQ_FIRST(&so->so_rcv.uxdg_mb), op); 4050 unp_scan(so->so_rcv.uxdg_peeked, op); 4051 TAILQ_FOREACH(sb, &so->so_rcv.uxdg_conns, uxdg_clist) 4052 unp_scan(STAILQ_FIRST(&sb->uxdg_mb), op); 4053 break; 4054 case SOCK_STREAM: 4055 case SOCK_SEQPACKET: 4056 unp_scan(STAILQ_FIRST(&so->so_rcv.uxst_mbq), op); 4057 break; 4058 } 4059 SOCK_RECVBUF_UNLOCK(so); 4060 } 4061 4062 static void 4063 unp_gc_scan(struct unpcb *unp, void (*op)(struct filedescent **, int)) 4064 { 4065 struct socket *so, *soa; 4066 4067 so = unp->unp_socket; 4068 SOCK_LOCK(so); 4069 if (SOLISTENING(so)) { 4070 /* 4071 * Mark all sockets in our accept queue. 4072 */ 4073 TAILQ_FOREACH(soa, &so->sol_comp, so_list) 4074 unp_scan_socket(soa, op); 4075 } else { 4076 /* 4077 * Mark all sockets we reference with RIGHTS. 4078 */ 4079 unp_scan_socket(so, op); 4080 } 4081 SOCK_UNLOCK(so); 4082 } 4083 4084 static int unp_recycled; 4085 SYSCTL_INT(_net_local, OID_AUTO, recycled, CTLFLAG_RD, &unp_recycled, 0, 4086 "Number of unreachable sockets claimed by the garbage collector."); 4087 4088 static int unp_taskcount; 4089 SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0, 4090 "Number of times the garbage collector has run."); 4091 4092 SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0, 4093 "Number of active local sockets."); 4094 4095 static void 4096 unp_gc(__unused void *arg, int pending) 4097 { 4098 struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead, 4099 NULL }; 4100 struct unp_head **head; 4101 struct unp_head unp_deadhead; /* List of potentially-dead sockets. */ 4102 struct file *f, **unref; 4103 struct unpcb *unp, *unptmp; 4104 int i, total, unp_unreachable; 4105 4106 LIST_INIT(&unp_deadhead); 4107 unp_taskcount++; 4108 UNP_LINK_RLOCK(); 4109 /* 4110 * First determine which sockets may be in cycles. 4111 */ 4112 unp_unreachable = 0; 4113 4114 for (head = heads; *head != NULL; head++) 4115 LIST_FOREACH(unp, *head, unp_link) { 4116 KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0, 4117 ("%s: unp %p has unexpected gc flags 0x%x", 4118 __func__, unp, (unsigned int)unp->unp_gcflag)); 4119 4120 f = unp->unp_file; 4121 4122 /* 4123 * Check for an unreachable socket potentially in a 4124 * cycle. It must be in a queue as indicated by 4125 * msgcount, and this must equal the file reference 4126 * count. Note that when msgcount is 0 the file is 4127 * NULL. 4128 */ 4129 if (f != NULL && unp->unp_msgcount != 0 && 4130 refcount_load(&f->f_count) == unp->unp_msgcount) { 4131 LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead); 4132 unp->unp_gcflag |= UNPGC_DEAD; 4133 unp->unp_gcrefs = unp->unp_msgcount; 4134 unp_unreachable++; 4135 } 4136 } 4137 4138 /* 4139 * Scan all sockets previously marked as potentially being in a cycle 4140 * and remove the references each socket holds on any UNPGC_DEAD 4141 * sockets in its queue. After this step, all remaining references on 4142 * sockets marked UNPGC_DEAD should not be part of any cycle. 4143 */ 4144 LIST_FOREACH(unp, &unp_deadhead, unp_dead) 4145 unp_gc_scan(unp, unp_remove_dead_ref); 4146 4147 /* 4148 * If a socket still has a non-negative refcount, it cannot be in a 4149 * cycle. In this case increment refcount of all children iteratively. 4150 * Stop the scan once we do a complete loop without discovering 4151 * a new reachable socket. 4152 */ 4153 do { 4154 unp_marked = 0; 4155 LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp) 4156 if (unp->unp_gcrefs > 0) { 4157 unp->unp_gcflag &= ~UNPGC_DEAD; 4158 LIST_REMOVE(unp, unp_dead); 4159 KASSERT(unp_unreachable > 0, 4160 ("%s: unp_unreachable underflow.", 4161 __func__)); 4162 unp_unreachable--; 4163 unp_gc_scan(unp, unp_restore_undead_ref); 4164 } 4165 } while (unp_marked); 4166 4167 UNP_LINK_RUNLOCK(); 4168 4169 if (unp_unreachable == 0) 4170 return; 4171 4172 /* 4173 * Allocate space for a local array of dead unpcbs. 4174 * TODO: can this path be simplified by instead using the local 4175 * dead list at unp_deadhead, after taking out references 4176 * on the file object and/or unpcb and dropping the link lock? 4177 */ 4178 unref = malloc(unp_unreachable * sizeof(struct file *), 4179 M_TEMP, M_WAITOK); 4180 4181 /* 4182 * Iterate looking for sockets which have been specifically marked 4183 * as unreachable and store them locally. 4184 */ 4185 UNP_LINK_RLOCK(); 4186 total = 0; 4187 LIST_FOREACH(unp, &unp_deadhead, unp_dead) { 4188 KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0, 4189 ("%s: unp %p not marked UNPGC_DEAD", __func__, unp)); 4190 unp->unp_gcflag &= ~UNPGC_DEAD; 4191 f = unp->unp_file; 4192 if (unp->unp_msgcount == 0 || f == NULL || 4193 refcount_load(&f->f_count) != unp->unp_msgcount || 4194 !fhold(f)) 4195 continue; 4196 unref[total++] = f; 4197 KASSERT(total <= unp_unreachable, 4198 ("%s: incorrect unreachable count.", __func__)); 4199 } 4200 UNP_LINK_RUNLOCK(); 4201 4202 /* 4203 * Now flush all sockets, free'ing rights. This will free the 4204 * struct files associated with these sockets but leave each socket 4205 * with one remaining ref. 4206 */ 4207 for (i = 0; i < total; i++) { 4208 struct socket *so; 4209 4210 so = unref[i]->f_data; 4211 CURVNET_SET(so->so_vnet); 4212 socantrcvmore(so); 4213 unp_dispose(so); 4214 CURVNET_RESTORE(); 4215 } 4216 4217 /* 4218 * And finally release the sockets so they can be reclaimed. 4219 */ 4220 for (i = 0; i < total; i++) 4221 fdrop(unref[i], NULL); 4222 unp_recycled += total; 4223 free(unref, M_TEMP); 4224 } 4225 4226 /* 4227 * Synchronize against unp_gc, which can trip over data as we are freeing it. 4228 */ 4229 static void 4230 unp_dispose(struct socket *so) 4231 { 4232 struct sockbuf *sb; 4233 struct unpcb *unp; 4234 struct mbuf *m; 4235 int error __diagused; 4236 4237 MPASS(!SOLISTENING(so)); 4238 4239 unp = sotounpcb(so); 4240 UNP_LINK_WLOCK(); 4241 unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS; 4242 UNP_LINK_WUNLOCK(); 4243 4244 /* 4245 * Grab our special mbufs before calling sbrelease(). 4246 */ 4247 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR); 4248 MPASS(!error); 4249 SOCK_RECVBUF_LOCK(so); 4250 switch (so->so_type) { 4251 case SOCK_DGRAM: 4252 while ((sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) != NULL) { 4253 STAILQ_CONCAT(&so->so_rcv.uxdg_mb, &sb->uxdg_mb); 4254 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); 4255 /* Note: socket of sb may reconnect. */ 4256 sb->uxdg_cc = sb->uxdg_ctl = sb->uxdg_mbcnt = 0; 4257 } 4258 sb = &so->so_rcv; 4259 if (sb->uxdg_peeked != NULL) { 4260 STAILQ_INSERT_HEAD(&sb->uxdg_mb, sb->uxdg_peeked, 4261 m_stailqpkt); 4262 sb->uxdg_peeked = NULL; 4263 } 4264 m = STAILQ_FIRST(&sb->uxdg_mb); 4265 STAILQ_INIT(&sb->uxdg_mb); 4266 break; 4267 case SOCK_STREAM: 4268 case SOCK_SEQPACKET: 4269 sb = &so->so_rcv; 4270 m = STAILQ_FIRST(&sb->uxst_mbq); 4271 STAILQ_INIT(&sb->uxst_mbq); 4272 sb->sb_acc = sb->sb_ccc = sb->sb_ctl = sb->sb_mbcnt = 0; 4273 /* 4274 * Trim M_NOTREADY buffers from the free list. They are 4275 * referenced by the I/O thread. 4276 */ 4277 if (sb->uxst_fnrdy != NULL) { 4278 struct mbuf *n, *prev; 4279 4280 while (m != NULL && m->m_flags & M_NOTREADY) 4281 m = m->m_next; 4282 for (prev = n = m; n != NULL; n = n->m_next) { 4283 if (n->m_flags & M_NOTREADY) 4284 prev->m_next = n->m_next; 4285 else 4286 prev = n; 4287 } 4288 sb->uxst_fnrdy = NULL; 4289 } 4290 break; 4291 } 4292 /* 4293 * Mark sb with SBS_CANTRCVMORE. This is needed to prevent 4294 * uipc_sosend_*() or unp_disconnect() adding more data to the socket. 4295 * We came here either through shutdown(2) or from the final sofree(). 4296 * The sofree() case is simple as it guarantees that no more sends will 4297 * happen, however we can race with unp_disconnect() from our peer. 4298 * The shutdown(2) case is more exotic. It would call into 4299 * unp_dispose() only if socket is SS_ISCONNECTED. This is possible if 4300 * we did connect(2) on this socket and we also had it bound with 4301 * bind(2) and receive connections from other sockets. Because 4302 * uipc_shutdown() violates POSIX (see comment there) this applies to 4303 * SOCK_DGRAM as well. For SOCK_DGRAM this SBS_CANTRCVMORE will have 4304 * affect not only on the peer we connect(2)ed to, but also on all of 4305 * the peers who had connect(2)ed to us. Their sends would end up 4306 * with ENOBUFS. 4307 */ 4308 sb->sb_state |= SBS_CANTRCVMORE; 4309 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 4310 RLIM_INFINITY); 4311 SOCK_RECVBUF_UNLOCK(so); 4312 SOCK_IO_RECV_UNLOCK(so); 4313 4314 if (m != NULL) { 4315 unp_scan(m, unp_freerights); 4316 m_freemp(m); 4317 } 4318 } 4319 4320 static void 4321 unp_scan(struct mbuf *m0, void (*op)(struct filedescent **, int)) 4322 { 4323 struct mbuf *m; 4324 struct cmsghdr *cm; 4325 void *data; 4326 socklen_t clen, datalen; 4327 4328 while (m0 != NULL) { 4329 for (m = m0; m; m = m->m_next) { 4330 if (m->m_type != MT_CONTROL) 4331 continue; 4332 4333 cm = mtod(m, struct cmsghdr *); 4334 clen = m->m_len; 4335 4336 while (cm != NULL) { 4337 if (sizeof(*cm) > clen || cm->cmsg_len > clen) 4338 break; 4339 4340 data = CMSG_DATA(cm); 4341 datalen = (caddr_t)cm + cm->cmsg_len 4342 - (caddr_t)data; 4343 4344 if (cm->cmsg_level == SOL_SOCKET && 4345 cm->cmsg_type == SCM_RIGHTS) { 4346 (*op)(data, datalen / 4347 sizeof(struct filedescent *)); 4348 } 4349 4350 if (CMSG_SPACE(datalen) < clen) { 4351 clen -= CMSG_SPACE(datalen); 4352 cm = (struct cmsghdr *) 4353 ((caddr_t)cm + CMSG_SPACE(datalen)); 4354 } else { 4355 clen = 0; 4356 cm = NULL; 4357 } 4358 } 4359 } 4360 m0 = m0->m_nextpkt; 4361 } 4362 } 4363 4364 /* 4365 * Definitions of protocols supported in the LOCAL domain. 4366 */ 4367 static struct protosw streamproto = { 4368 .pr_type = SOCK_STREAM, 4369 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4370 .pr_ctloutput = &uipc_ctloutput, 4371 .pr_abort = uipc_abort, 4372 .pr_accept = uipc_peeraddr, 4373 .pr_attach = uipc_attach, 4374 .pr_bind = uipc_bind, 4375 .pr_bindat = uipc_bindat, 4376 .pr_connect = uipc_connect, 4377 .pr_connectat = uipc_connectat, 4378 .pr_connect2 = uipc_connect2, 4379 .pr_detach = uipc_detach, 4380 .pr_disconnect = uipc_disconnect, 4381 .pr_listen = uipc_listen, 4382 .pr_peeraddr = uipc_peeraddr, 4383 .pr_send = uipc_sendfile, 4384 .pr_sendfile_wait = uipc_sendfile_wait, 4385 .pr_ready = uipc_ready, 4386 .pr_sense = uipc_sense, 4387 .pr_shutdown = uipc_shutdown, 4388 .pr_sockaddr = uipc_sockaddr, 4389 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4390 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4391 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4392 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4393 .pr_close = uipc_close, 4394 .pr_chmod = uipc_chmod, 4395 }; 4396 4397 static struct protosw dgramproto = { 4398 .pr_type = SOCK_DGRAM, 4399 .pr_flags = PR_ATOMIC | PR_ADDR | PR_CAPATTACH | PR_SOCKBUF, 4400 .pr_ctloutput = &uipc_ctloutput, 4401 .pr_abort = uipc_abort, 4402 .pr_accept = uipc_peeraddr, 4403 .pr_attach = uipc_attach, 4404 .pr_bind = uipc_bind, 4405 .pr_bindat = uipc_bindat, 4406 .pr_connect = uipc_connect, 4407 .pr_connectat = uipc_connectat, 4408 .pr_connect2 = uipc_connect2, 4409 .pr_detach = uipc_detach, 4410 .pr_disconnect = uipc_disconnect, 4411 .pr_peeraddr = uipc_peeraddr, 4412 .pr_sosend = uipc_sosend_dgram, 4413 .pr_sense = uipc_sense, 4414 .pr_shutdown = uipc_shutdown, 4415 .pr_sockaddr = uipc_sockaddr, 4416 .pr_soreceive = uipc_soreceive_dgram, 4417 .pr_close = uipc_close, 4418 .pr_chmod = uipc_chmod, 4419 }; 4420 4421 static struct protosw seqpacketproto = { 4422 .pr_type = SOCK_SEQPACKET, 4423 .pr_flags = PR_CONNREQUIRED | PR_CAPATTACH | PR_SOCKBUF, 4424 .pr_ctloutput = &uipc_ctloutput, 4425 .pr_abort = uipc_abort, 4426 .pr_accept = uipc_peeraddr, 4427 .pr_attach = uipc_attach, 4428 .pr_bind = uipc_bind, 4429 .pr_bindat = uipc_bindat, 4430 .pr_connect = uipc_connect, 4431 .pr_connectat = uipc_connectat, 4432 .pr_connect2 = uipc_connect2, 4433 .pr_detach = uipc_detach, 4434 .pr_disconnect = uipc_disconnect, 4435 .pr_listen = uipc_listen, 4436 .pr_peeraddr = uipc_peeraddr, 4437 .pr_sense = uipc_sense, 4438 .pr_shutdown = uipc_shutdown, 4439 .pr_sockaddr = uipc_sockaddr, 4440 .pr_sosend = uipc_sosend_stream_or_seqpacket, 4441 .pr_soreceive = uipc_soreceive_stream_or_seqpacket, 4442 .pr_sopoll = uipc_sopoll_stream_or_seqpacket, 4443 .pr_kqfilter = uipc_kqfilter_stream_or_seqpacket, 4444 .pr_close = uipc_close, 4445 .pr_chmod = uipc_chmod, 4446 }; 4447 4448 static struct domain localdomain = { 4449 .dom_family = AF_LOCAL, 4450 .dom_name = "local", 4451 .dom_nprotosw = 3, 4452 .dom_protosw = { 4453 &streamproto, 4454 &dgramproto, 4455 &seqpacketproto, 4456 } 4457 }; 4458 DOMAIN_SET(local); 4459 4460 /* 4461 * A helper function called by VFS before socket-type vnode reclamation. 4462 * For an active vnode it clears unp_vnode pointer and decrements unp_vnode 4463 * use count. 4464 */ 4465 void 4466 vfs_unp_reclaim(struct vnode *vp) 4467 { 4468 struct unpcb *unp; 4469 int active; 4470 struct mtx *vplock; 4471 4472 ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim"); 4473 KASSERT(vp->v_type == VSOCK, 4474 ("vfs_unp_reclaim: vp->v_type != VSOCK")); 4475 4476 active = 0; 4477 vplock = mtx_pool_find(unp_vp_mtxpool, vp); 4478 mtx_lock(vplock); 4479 VOP_UNP_CONNECT(vp, &unp); 4480 if (unp == NULL) 4481 goto done; 4482 UNP_PCB_LOCK(unp); 4483 if (unp->unp_vnode == vp) { 4484 VOP_UNP_DETACH(vp); 4485 unp->unp_vnode = NULL; 4486 active = 1; 4487 } 4488 UNP_PCB_UNLOCK(unp); 4489 done: 4490 mtx_unlock(vplock); 4491 if (active) 4492 vunref(vp); 4493 } 4494 4495 #ifdef DDB 4496 static void 4497 db_print_indent(int indent) 4498 { 4499 int i; 4500 4501 for (i = 0; i < indent; i++) 4502 db_printf(" "); 4503 } 4504 4505 static void 4506 db_print_unpflags(int unp_flags) 4507 { 4508 int comma; 4509 4510 comma = 0; 4511 if (unp_flags & UNP_HAVEPC) { 4512 db_printf("%sUNP_HAVEPC", comma ? ", " : ""); 4513 comma = 1; 4514 } 4515 if (unp_flags & UNP_WANTCRED_ALWAYS) { 4516 db_printf("%sUNP_WANTCRED_ALWAYS", comma ? ", " : ""); 4517 comma = 1; 4518 } 4519 if (unp_flags & UNP_WANTCRED_ONESHOT) { 4520 db_printf("%sUNP_WANTCRED_ONESHOT", comma ? ", " : ""); 4521 comma = 1; 4522 } 4523 if (unp_flags & UNP_CONNECTING) { 4524 db_printf("%sUNP_CONNECTING", comma ? ", " : ""); 4525 comma = 1; 4526 } 4527 if (unp_flags & UNP_BINDING) { 4528 db_printf("%sUNP_BINDING", comma ? ", " : ""); 4529 comma = 1; 4530 } 4531 } 4532 4533 static void 4534 db_print_xucred(int indent, struct xucred *xu) 4535 { 4536 int comma, i; 4537 4538 db_print_indent(indent); 4539 db_printf("cr_version: %u cr_uid: %u cr_pid: %d cr_ngroups: %d\n", 4540 xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups); 4541 db_print_indent(indent); 4542 db_printf("cr_groups: "); 4543 comma = 0; 4544 for (i = 0; i < xu->cr_ngroups; i++) { 4545 db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]); 4546 comma = 1; 4547 } 4548 db_printf("\n"); 4549 } 4550 4551 static void 4552 db_print_unprefs(int indent, struct unp_head *uh) 4553 { 4554 struct unpcb *unp; 4555 int counter; 4556 4557 counter = 0; 4558 LIST_FOREACH(unp, uh, unp_reflink) { 4559 if (counter % 4 == 0) 4560 db_print_indent(indent); 4561 db_printf("%p ", unp); 4562 if (counter % 4 == 3) 4563 db_printf("\n"); 4564 counter++; 4565 } 4566 if (counter != 0 && counter % 4 != 0) 4567 db_printf("\n"); 4568 } 4569 4570 DB_SHOW_COMMAND(unpcb, db_show_unpcb) 4571 { 4572 struct unpcb *unp; 4573 4574 if (!have_addr) { 4575 db_printf("usage: show unpcb <addr>\n"); 4576 return; 4577 } 4578 unp = (struct unpcb *)addr; 4579 4580 db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket, 4581 unp->unp_vnode); 4582 4583 db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino, 4584 unp->unp_conn); 4585 4586 db_printf("unp_refs:\n"); 4587 db_print_unprefs(2, &unp->unp_refs); 4588 4589 /* XXXRW: Would be nice to print the full address, if any. */ 4590 db_printf("unp_addr: %p\n", unp->unp_addr); 4591 4592 db_printf("unp_gencnt: %llu\n", 4593 (unsigned long long)unp->unp_gencnt); 4594 4595 db_printf("unp_flags: %x (", unp->unp_flags); 4596 db_print_unpflags(unp->unp_flags); 4597 db_printf(")\n"); 4598 4599 db_printf("unp_peercred:\n"); 4600 db_print_xucred(2, &unp->unp_peercred); 4601 4602 db_printf("unp_refcount: %u\n", unp->unp_refcount); 4603 } 4604 #endif 4605