xref: /freebsd/sys/kern/uipc_usrreq.c (revision aebc96831f152a9da5f2c919103d571e80d3fade)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1982, 1986, 1989, 1991, 1993
5  *	The Regents of the University of California. All Rights Reserved.
6  * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved.
7  * Copyright (c) 2018 Matthew Macy
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  *
33  *	From: @(#)uipc_usrreq.c	8.3 (Berkeley) 1/4/94
34  */
35 
36 /*
37  * UNIX Domain (Local) Sockets
38  *
39  * This is an implementation of UNIX (local) domain sockets.  Each socket has
40  * an associated struct unpcb (UNIX protocol control block).  Stream sockets
41  * may be connected to 0 or 1 other socket.  Datagram sockets may be
42  * connected to 0, 1, or many other sockets.  Sockets may be created and
43  * connected in pairs (socketpair(2)), or bound/connected to using the file
44  * system name space.  For most purposes, only the receive socket buffer is
45  * used, as sending on one socket delivers directly to the receive socket
46  * buffer of a second socket.
47  *
48  * The implementation is substantially complicated by the fact that
49  * "ancillary data", such as file descriptors or credentials, may be passed
50  * across UNIX domain sockets.  The potential for passing UNIX domain sockets
51  * over other UNIX domain sockets requires the implementation of a simple
52  * garbage collector to find and tear down cycles of disconnected sockets.
53  *
54  * TODO:
55  *	RDM
56  *	rethink name space problems
57  *	need a proper out-of-band
58  */
59 
60 #include <sys/cdefs.h>
61 __FBSDID("$FreeBSD$");
62 
63 #include "opt_ddb.h"
64 
65 #include <sys/param.h>
66 #include <sys/capsicum.h>
67 #include <sys/domain.h>
68 #include <sys/eventhandler.h>
69 #include <sys/fcntl.h>
70 #include <sys/file.h>
71 #include <sys/filedesc.h>
72 #include <sys/kernel.h>
73 #include <sys/lock.h>
74 #include <sys/malloc.h>
75 #include <sys/mbuf.h>
76 #include <sys/mount.h>
77 #include <sys/mutex.h>
78 #include <sys/namei.h>
79 #include <sys/proc.h>
80 #include <sys/protosw.h>
81 #include <sys/queue.h>
82 #include <sys/resourcevar.h>
83 #include <sys/rwlock.h>
84 #include <sys/socket.h>
85 #include <sys/socketvar.h>
86 #include <sys/signalvar.h>
87 #include <sys/stat.h>
88 #include <sys/sx.h>
89 #include <sys/sysctl.h>
90 #include <sys/systm.h>
91 #include <sys/taskqueue.h>
92 #include <sys/un.h>
93 #include <sys/unpcb.h>
94 #include <sys/vnode.h>
95 
96 #include <net/vnet.h>
97 
98 #ifdef DDB
99 #include <ddb/ddb.h>
100 #endif
101 
102 #include <security/mac/mac_framework.h>
103 
104 #include <vm/uma.h>
105 
106 MALLOC_DECLARE(M_FILECAPS);
107 
108 /*
109  * See unpcb.h for the locking key.
110  */
111 
112 static uma_zone_t	unp_zone;
113 static unp_gen_t	unp_gencnt;	/* (l) */
114 static u_int		unp_count;	/* (l) Count of local sockets. */
115 static ino_t		unp_ino;	/* Prototype for fake inode numbers. */
116 static int		unp_rights;	/* (g) File descriptors in flight. */
117 static struct unp_head	unp_shead;	/* (l) List of stream sockets. */
118 static struct unp_head	unp_dhead;	/* (l) List of datagram sockets. */
119 static struct unp_head	unp_sphead;	/* (l) List of seqpacket sockets. */
120 
121 struct unp_defer {
122 	SLIST_ENTRY(unp_defer) ud_link;
123 	struct file *ud_fp;
124 };
125 static SLIST_HEAD(, unp_defer) unp_defers;
126 static int unp_defers_count;
127 
128 static const struct sockaddr	sun_noname = { sizeof(sun_noname), AF_LOCAL };
129 
130 /*
131  * Garbage collection of cyclic file descriptor/socket references occurs
132  * asynchronously in a taskqueue context in order to avoid recursion and
133  * reentrance in the UNIX domain socket, file descriptor, and socket layer
134  * code.  See unp_gc() for a full description.
135  */
136 static struct timeout_task unp_gc_task;
137 
138 /*
139  * The close of unix domain sockets attached as SCM_RIGHTS is
140  * postponed to the taskqueue, to avoid arbitrary recursion depth.
141  * The attached sockets might have another sockets attached.
142  */
143 static struct task	unp_defer_task;
144 
145 /*
146  * Both send and receive buffers are allocated PIPSIZ bytes of buffering for
147  * stream sockets, although the total for sender and receiver is actually
148  * only PIPSIZ.
149  *
150  * Datagram sockets really use the sendspace as the maximum datagram size,
151  * and don't really want to reserve the sendspace.  Their recvspace should be
152  * large enough for at least one max-size datagram plus address.
153  */
154 #ifndef PIPSIZ
155 #define	PIPSIZ	8192
156 #endif
157 static u_long	unpst_sendspace = PIPSIZ;
158 static u_long	unpst_recvspace = PIPSIZ;
159 static u_long	unpdg_sendspace = 2*1024;	/* really max datagram size */
160 static u_long	unpdg_recvspace = 4*1024;
161 static u_long	unpsp_sendspace = PIPSIZ;	/* really max datagram size */
162 static u_long	unpsp_recvspace = PIPSIZ;
163 
164 static SYSCTL_NODE(_net, PF_LOCAL, local, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
165     "Local domain");
166 static SYSCTL_NODE(_net_local, SOCK_STREAM, stream,
167     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
168     "SOCK_STREAM");
169 static SYSCTL_NODE(_net_local, SOCK_DGRAM, dgram,
170     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
171     "SOCK_DGRAM");
172 static SYSCTL_NODE(_net_local, SOCK_SEQPACKET, seqpacket,
173     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
174     "SOCK_SEQPACKET");
175 
176 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW,
177 	   &unpst_sendspace, 0, "Default stream send space.");
178 SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW,
179 	   &unpst_recvspace, 0, "Default stream receive space.");
180 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW,
181 	   &unpdg_sendspace, 0, "Default datagram send space.");
182 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW,
183 	   &unpdg_recvspace, 0, "Default datagram receive space.");
184 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW,
185 	   &unpsp_sendspace, 0, "Default seqpacket send space.");
186 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW,
187 	   &unpsp_recvspace, 0, "Default seqpacket receive space.");
188 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0,
189     "File descriptors in flight.");
190 SYSCTL_INT(_net_local, OID_AUTO, deferred, CTLFLAG_RD,
191     &unp_defers_count, 0,
192     "File descriptors deferred to taskqueue for close.");
193 
194 /*
195  * Locking and synchronization:
196  *
197  * Several types of locks exist in the local domain socket implementation:
198  * - a global linkage lock
199  * - a global connection list lock
200  * - the mtxpool lock
201  * - per-unpcb mutexes
202  *
203  * The linkage lock protects the global socket lists, the generation number
204  * counter and garbage collector state.
205  *
206  * The connection list lock protects the list of referring sockets in a datagram
207  * socket PCB.  This lock is also overloaded to protect a global list of
208  * sockets whose buffers contain socket references in the form of SCM_RIGHTS
209  * messages.  To avoid recursion, such references are released by a dedicated
210  * thread.
211  *
212  * The mtxpool lock protects the vnode from being modified while referenced.
213  * Lock ordering rules require that it be acquired before any PCB locks.
214  *
215  * The unpcb lock (unp_mtx) protects the most commonly referenced fields in the
216  * unpcb.  This includes the unp_conn field, which either links two connected
217  * PCBs together (for connected socket types) or points at the destination
218  * socket (for connectionless socket types).  The operations of creating or
219  * destroying a connection therefore involve locking multiple PCBs.  To avoid
220  * lock order reversals, in some cases this involves dropping a PCB lock and
221  * using a reference counter to maintain liveness.
222  *
223  * UNIX domain sockets each have an unpcb hung off of their so_pcb pointer,
224  * allocated in pru_attach() and freed in pru_detach().  The validity of that
225  * pointer is an invariant, so no lock is required to dereference the so_pcb
226  * pointer if a valid socket reference is held by the caller.  In practice,
227  * this is always true during operations performed on a socket.  Each unpcb
228  * has a back-pointer to its socket, unp_socket, which will be stable under
229  * the same circumstances.
230  *
231  * This pointer may only be safely dereferenced as long as a valid reference
232  * to the unpcb is held.  Typically, this reference will be from the socket,
233  * or from another unpcb when the referring unpcb's lock is held (in order
234  * that the reference not be invalidated during use).  For example, to follow
235  * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee
236  * that detach is not run clearing unp_socket.
237  *
238  * Blocking with UNIX domain sockets is a tricky issue: unlike most network
239  * protocols, bind() is a non-atomic operation, and connect() requires
240  * potential sleeping in the protocol, due to potentially waiting on local or
241  * distributed file systems.  We try to separate "lookup" operations, which
242  * may sleep, and the IPC operations themselves, which typically can occur
243  * with relative atomicity as locks can be held over the entire operation.
244  *
245  * Another tricky issue is simultaneous multi-threaded or multi-process
246  * access to a single UNIX domain socket.  These are handled by the flags
247  * UNP_CONNECTING and UNP_BINDING, which prevent concurrent connecting or
248  * binding, both of which involve dropping UNIX domain socket locks in order
249  * to perform namei() and other file system operations.
250  */
251 static struct rwlock	unp_link_rwlock;
252 static struct mtx	unp_defers_lock;
253 
254 #define	UNP_LINK_LOCK_INIT()		rw_init(&unp_link_rwlock,	\
255 					    "unp_link_rwlock")
256 
257 #define	UNP_LINK_LOCK_ASSERT()		rw_assert(&unp_link_rwlock,	\
258 					    RA_LOCKED)
259 #define	UNP_LINK_UNLOCK_ASSERT()	rw_assert(&unp_link_rwlock,	\
260 					    RA_UNLOCKED)
261 
262 #define	UNP_LINK_RLOCK()		rw_rlock(&unp_link_rwlock)
263 #define	UNP_LINK_RUNLOCK()		rw_runlock(&unp_link_rwlock)
264 #define	UNP_LINK_WLOCK()		rw_wlock(&unp_link_rwlock)
265 #define	UNP_LINK_WUNLOCK()		rw_wunlock(&unp_link_rwlock)
266 #define	UNP_LINK_WLOCK_ASSERT()		rw_assert(&unp_link_rwlock,	\
267 					    RA_WLOCKED)
268 #define	UNP_LINK_WOWNED()		rw_wowned(&unp_link_rwlock)
269 
270 #define	UNP_DEFERRED_LOCK_INIT()	mtx_init(&unp_defers_lock, \
271 					    "unp_defer", NULL, MTX_DEF)
272 #define	UNP_DEFERRED_LOCK()		mtx_lock(&unp_defers_lock)
273 #define	UNP_DEFERRED_UNLOCK()		mtx_unlock(&unp_defers_lock)
274 
275 #define UNP_REF_LIST_LOCK()		UNP_DEFERRED_LOCK();
276 #define UNP_REF_LIST_UNLOCK()		UNP_DEFERRED_UNLOCK();
277 
278 #define UNP_PCB_LOCK_INIT(unp)		mtx_init(&(unp)->unp_mtx,	\
279 					    "unp", "unp",	\
280 					    MTX_DUPOK|MTX_DEF)
281 #define	UNP_PCB_LOCK_DESTROY(unp)	mtx_destroy(&(unp)->unp_mtx)
282 #define	UNP_PCB_LOCKPTR(unp)		(&(unp)->unp_mtx)
283 #define	UNP_PCB_LOCK(unp)		mtx_lock(&(unp)->unp_mtx)
284 #define	UNP_PCB_TRYLOCK(unp)		mtx_trylock(&(unp)->unp_mtx)
285 #define	UNP_PCB_UNLOCK(unp)		mtx_unlock(&(unp)->unp_mtx)
286 #define	UNP_PCB_OWNED(unp)		mtx_owned(&(unp)->unp_mtx)
287 #define	UNP_PCB_LOCK_ASSERT(unp)	mtx_assert(&(unp)->unp_mtx, MA_OWNED)
288 #define	UNP_PCB_UNLOCK_ASSERT(unp)	mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED)
289 
290 static int	uipc_connect2(struct socket *, struct socket *);
291 static int	uipc_ctloutput(struct socket *, struct sockopt *);
292 static int	unp_connect(struct socket *, struct sockaddr *,
293 		    struct thread *);
294 static int	unp_connectat(int, struct socket *, struct sockaddr *,
295 		    struct thread *);
296 static int	unp_connect2(struct socket *so, struct socket *so2, int);
297 static void	unp_disconnect(struct unpcb *unp, struct unpcb *unp2);
298 static void	unp_dispose(struct socket *so);
299 static void	unp_dispose_mbuf(struct mbuf *);
300 static void	unp_shutdown(struct unpcb *);
301 static void	unp_drop(struct unpcb *);
302 static void	unp_gc(__unused void *, int);
303 static void	unp_scan(struct mbuf *, void (*)(struct filedescent **, int));
304 static void	unp_discard(struct file *);
305 static void	unp_freerights(struct filedescent **, int);
306 static void	unp_init(void);
307 static int	unp_internalize(struct mbuf **, struct thread *);
308 static void	unp_internalize_fp(struct file *);
309 static int	unp_externalize(struct mbuf *, struct mbuf **, int);
310 static int	unp_externalize_fp(struct file *);
311 static struct mbuf	*unp_addsockcred(struct thread *, struct mbuf *);
312 static void	unp_process_defers(void * __unused, int);
313 
314 static void
315 unp_pcb_hold(struct unpcb *unp)
316 {
317 	u_int old __unused;
318 
319 	old = refcount_acquire(&unp->unp_refcount);
320 	KASSERT(old > 0, ("%s: unpcb %p has no references", __func__, unp));
321 }
322 
323 static __result_use_check bool
324 unp_pcb_rele(struct unpcb *unp)
325 {
326 	bool ret;
327 
328 	UNP_PCB_LOCK_ASSERT(unp);
329 
330 	if ((ret = refcount_release(&unp->unp_refcount))) {
331 		UNP_PCB_UNLOCK(unp);
332 		UNP_PCB_LOCK_DESTROY(unp);
333 		uma_zfree(unp_zone, unp);
334 	}
335 	return (ret);
336 }
337 
338 static void
339 unp_pcb_rele_notlast(struct unpcb *unp)
340 {
341 	bool ret __unused;
342 
343 	ret = refcount_release(&unp->unp_refcount);
344 	KASSERT(!ret, ("%s: unpcb %p has no references", __func__, unp));
345 }
346 
347 static void
348 unp_pcb_lock_pair(struct unpcb *unp, struct unpcb *unp2)
349 {
350 	UNP_PCB_UNLOCK_ASSERT(unp);
351 	UNP_PCB_UNLOCK_ASSERT(unp2);
352 
353 	if (unp == unp2) {
354 		UNP_PCB_LOCK(unp);
355 	} else if ((uintptr_t)unp2 > (uintptr_t)unp) {
356 		UNP_PCB_LOCK(unp);
357 		UNP_PCB_LOCK(unp2);
358 	} else {
359 		UNP_PCB_LOCK(unp2);
360 		UNP_PCB_LOCK(unp);
361 	}
362 }
363 
364 static void
365 unp_pcb_unlock_pair(struct unpcb *unp, struct unpcb *unp2)
366 {
367 	UNP_PCB_UNLOCK(unp);
368 	if (unp != unp2)
369 		UNP_PCB_UNLOCK(unp2);
370 }
371 
372 /*
373  * Try to lock the connected peer of an already locked socket.  In some cases
374  * this requires that we unlock the current socket.  The pairbusy counter is
375  * used to block concurrent connection attempts while the lock is dropped.  The
376  * caller must be careful to revalidate PCB state.
377  */
378 static struct unpcb *
379 unp_pcb_lock_peer(struct unpcb *unp)
380 {
381 	struct unpcb *unp2;
382 
383 	UNP_PCB_LOCK_ASSERT(unp);
384 	unp2 = unp->unp_conn;
385 	if (__predict_false(unp2 == NULL))
386 		return (NULL);
387 	if (__predict_false(unp == unp2))
388 		return (unp);
389 
390 	UNP_PCB_UNLOCK_ASSERT(unp2);
391 
392 	if (__predict_true(UNP_PCB_TRYLOCK(unp2)))
393 		return (unp2);
394 	if ((uintptr_t)unp2 > (uintptr_t)unp) {
395 		UNP_PCB_LOCK(unp2);
396 		return (unp2);
397 	}
398 	unp->unp_pairbusy++;
399 	unp_pcb_hold(unp2);
400 	UNP_PCB_UNLOCK(unp);
401 
402 	UNP_PCB_LOCK(unp2);
403 	UNP_PCB_LOCK(unp);
404 	KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL,
405 	    ("%s: socket %p was reconnected", __func__, unp));
406 	if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) {
407 		unp->unp_flags &= ~UNP_WAITING;
408 		wakeup(unp);
409 	}
410 	if (unp_pcb_rele(unp2)) {
411 		/* unp2 is unlocked. */
412 		return (NULL);
413 	}
414 	if (unp->unp_conn == NULL) {
415 		UNP_PCB_UNLOCK(unp2);
416 		return (NULL);
417 	}
418 	return (unp2);
419 }
420 
421 /*
422  * Definitions of protocols supported in the LOCAL domain.
423  */
424 static struct domain localdomain;
425 static struct pr_usrreqs uipc_usrreqs_dgram, uipc_usrreqs_stream;
426 static struct pr_usrreqs uipc_usrreqs_seqpacket;
427 static struct protosw localsw[] = {
428 {
429 	.pr_type =		SOCK_STREAM,
430 	.pr_domain =		&localdomain,
431 	.pr_flags =		PR_CONNREQUIRED|PR_WANTRCVD|PR_RIGHTS,
432 	.pr_ctloutput =		&uipc_ctloutput,
433 	.pr_usrreqs =		&uipc_usrreqs_stream
434 },
435 {
436 	.pr_type =		SOCK_DGRAM,
437 	.pr_domain =		&localdomain,
438 	.pr_flags =		PR_ATOMIC|PR_ADDR|PR_RIGHTS,
439 	.pr_ctloutput =		&uipc_ctloutput,
440 	.pr_usrreqs =		&uipc_usrreqs_dgram
441 },
442 {
443 	.pr_type =		SOCK_SEQPACKET,
444 	.pr_domain =		&localdomain,
445 
446 	/*
447 	 * XXXRW: For now, PR_ADDR because soreceive will bump into them
448 	 * due to our use of sbappendaddr.  A new sbappend variants is needed
449 	 * that supports both atomic record writes and control data.
450 	 */
451 	.pr_flags =		PR_ADDR|PR_ATOMIC|PR_CONNREQUIRED|PR_WANTRCVD|
452 				    PR_RIGHTS,
453 	.pr_ctloutput =		&uipc_ctloutput,
454 	.pr_usrreqs =		&uipc_usrreqs_seqpacket,
455 },
456 };
457 
458 static struct domain localdomain = {
459 	.dom_family =		AF_LOCAL,
460 	.dom_name =		"local",
461 	.dom_init =		unp_init,
462 	.dom_externalize =	unp_externalize,
463 	.dom_dispose =		unp_dispose,
464 	.dom_protosw =		localsw,
465 	.dom_protoswNPROTOSW =	&localsw[nitems(localsw)]
466 };
467 DOMAIN_SET(local);
468 
469 static void
470 uipc_abort(struct socket *so)
471 {
472 	struct unpcb *unp, *unp2;
473 
474 	unp = sotounpcb(so);
475 	KASSERT(unp != NULL, ("uipc_abort: unp == NULL"));
476 	UNP_PCB_UNLOCK_ASSERT(unp);
477 
478 	UNP_PCB_LOCK(unp);
479 	unp2 = unp->unp_conn;
480 	if (unp2 != NULL) {
481 		unp_pcb_hold(unp2);
482 		UNP_PCB_UNLOCK(unp);
483 		unp_drop(unp2);
484 	} else
485 		UNP_PCB_UNLOCK(unp);
486 }
487 
488 static int
489 uipc_accept(struct socket *so, struct sockaddr **nam)
490 {
491 	struct unpcb *unp, *unp2;
492 	const struct sockaddr *sa;
493 
494 	/*
495 	 * Pass back name of connected socket, if it was bound and we are
496 	 * still connected (our peer may have closed already!).
497 	 */
498 	unp = sotounpcb(so);
499 	KASSERT(unp != NULL, ("uipc_accept: unp == NULL"));
500 
501 	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
502 	UNP_PCB_LOCK(unp);
503 	unp2 = unp_pcb_lock_peer(unp);
504 	if (unp2 != NULL && unp2->unp_addr != NULL)
505 		sa = (struct sockaddr *)unp2->unp_addr;
506 	else
507 		sa = &sun_noname;
508 	bcopy(sa, *nam, sa->sa_len);
509 	if (unp2 != NULL)
510 		unp_pcb_unlock_pair(unp, unp2);
511 	else
512 		UNP_PCB_UNLOCK(unp);
513 	return (0);
514 }
515 
516 static int
517 uipc_attach(struct socket *so, int proto, struct thread *td)
518 {
519 	u_long sendspace, recvspace;
520 	struct unpcb *unp;
521 	int error;
522 	bool locked;
523 
524 	KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL"));
525 	if (so->so_snd.sb_hiwat == 0 || so->so_rcv.sb_hiwat == 0) {
526 		switch (so->so_type) {
527 		case SOCK_STREAM:
528 			sendspace = unpst_sendspace;
529 			recvspace = unpst_recvspace;
530 			break;
531 
532 		case SOCK_DGRAM:
533 			sendspace = unpdg_sendspace;
534 			recvspace = unpdg_recvspace;
535 			break;
536 
537 		case SOCK_SEQPACKET:
538 			sendspace = unpsp_sendspace;
539 			recvspace = unpsp_recvspace;
540 			break;
541 
542 		default:
543 			panic("uipc_attach");
544 		}
545 		error = soreserve(so, sendspace, recvspace);
546 		if (error)
547 			return (error);
548 	}
549 	unp = uma_zalloc(unp_zone, M_NOWAIT | M_ZERO);
550 	if (unp == NULL)
551 		return (ENOBUFS);
552 	LIST_INIT(&unp->unp_refs);
553 	UNP_PCB_LOCK_INIT(unp);
554 	unp->unp_socket = so;
555 	so->so_pcb = unp;
556 	refcount_init(&unp->unp_refcount, 1);
557 
558 	if ((locked = UNP_LINK_WOWNED()) == false)
559 		UNP_LINK_WLOCK();
560 
561 	unp->unp_gencnt = ++unp_gencnt;
562 	unp->unp_ino = ++unp_ino;
563 	unp_count++;
564 	switch (so->so_type) {
565 	case SOCK_STREAM:
566 		LIST_INSERT_HEAD(&unp_shead, unp, unp_link);
567 		break;
568 
569 	case SOCK_DGRAM:
570 		LIST_INSERT_HEAD(&unp_dhead, unp, unp_link);
571 		break;
572 
573 	case SOCK_SEQPACKET:
574 		LIST_INSERT_HEAD(&unp_sphead, unp, unp_link);
575 		break;
576 
577 	default:
578 		panic("uipc_attach");
579 	}
580 
581 	if (locked == false)
582 		UNP_LINK_WUNLOCK();
583 
584 	return (0);
585 }
586 
587 static int
588 uipc_bindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td)
589 {
590 	struct sockaddr_un *soun = (struct sockaddr_un *)nam;
591 	struct vattr vattr;
592 	int error, namelen;
593 	struct nameidata nd;
594 	struct unpcb *unp;
595 	struct vnode *vp;
596 	struct mount *mp;
597 	cap_rights_t rights;
598 	char *buf;
599 
600 	if (nam->sa_family != AF_UNIX)
601 		return (EAFNOSUPPORT);
602 
603 	unp = sotounpcb(so);
604 	KASSERT(unp != NULL, ("uipc_bind: unp == NULL"));
605 
606 	if (soun->sun_len > sizeof(struct sockaddr_un))
607 		return (EINVAL);
608 	namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path);
609 	if (namelen <= 0)
610 		return (EINVAL);
611 
612 	/*
613 	 * We don't allow simultaneous bind() calls on a single UNIX domain
614 	 * socket, so flag in-progress operations, and return an error if an
615 	 * operation is already in progress.
616 	 *
617 	 * Historically, we have not allowed a socket to be rebound, so this
618 	 * also returns an error.  Not allowing re-binding simplifies the
619 	 * implementation and avoids a great many possible failure modes.
620 	 */
621 	UNP_PCB_LOCK(unp);
622 	if (unp->unp_vnode != NULL) {
623 		UNP_PCB_UNLOCK(unp);
624 		return (EINVAL);
625 	}
626 	if (unp->unp_flags & UNP_BINDING) {
627 		UNP_PCB_UNLOCK(unp);
628 		return (EALREADY);
629 	}
630 	unp->unp_flags |= UNP_BINDING;
631 	UNP_PCB_UNLOCK(unp);
632 
633 	buf = malloc(namelen + 1, M_TEMP, M_WAITOK);
634 	bcopy(soun->sun_path, buf, namelen);
635 	buf[namelen] = 0;
636 
637 restart:
638 	NDINIT_ATRIGHTS(&nd, CREATE, NOFOLLOW | LOCKPARENT | SAVENAME | NOCACHE,
639 	    UIO_SYSSPACE, buf, fd, cap_rights_init(&rights, CAP_BINDAT), td);
640 /* SHOULD BE ABLE TO ADOPT EXISTING AND wakeup() ALA FIFO's */
641 	error = namei(&nd);
642 	if (error)
643 		goto error;
644 	vp = nd.ni_vp;
645 	if (vp != NULL || vn_start_write(nd.ni_dvp, &mp, V_NOWAIT) != 0) {
646 		NDFREE(&nd, NDF_ONLY_PNBUF);
647 		if (nd.ni_dvp == vp)
648 			vrele(nd.ni_dvp);
649 		else
650 			vput(nd.ni_dvp);
651 		if (vp != NULL) {
652 			vrele(vp);
653 			error = EADDRINUSE;
654 			goto error;
655 		}
656 		error = vn_start_write(NULL, &mp, V_XSLEEP | PCATCH);
657 		if (error)
658 			goto error;
659 		goto restart;
660 	}
661 	VATTR_NULL(&vattr);
662 	vattr.va_type = VSOCK;
663 	vattr.va_mode = (ACCESSPERMS & ~td->td_proc->p_fd->fd_cmask);
664 #ifdef MAC
665 	error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd,
666 	    &vattr);
667 #endif
668 	if (error == 0)
669 		error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vattr);
670 	NDFREE(&nd, NDF_ONLY_PNBUF);
671 	vput(nd.ni_dvp);
672 	if (error) {
673 		vn_finished_write(mp);
674 		goto error;
675 	}
676 	vp = nd.ni_vp;
677 	ASSERT_VOP_ELOCKED(vp, "uipc_bind");
678 	soun = (struct sockaddr_un *)sodupsockaddr(nam, M_WAITOK);
679 
680 	UNP_PCB_LOCK(unp);
681 	VOP_UNP_BIND(vp, unp);
682 	unp->unp_vnode = vp;
683 	unp->unp_addr = soun;
684 	unp->unp_flags &= ~UNP_BINDING;
685 	UNP_PCB_UNLOCK(unp);
686 	VOP_UNLOCK(vp);
687 	vn_finished_write(mp);
688 	free(buf, M_TEMP);
689 	return (0);
690 
691 error:
692 	UNP_PCB_LOCK(unp);
693 	unp->unp_flags &= ~UNP_BINDING;
694 	UNP_PCB_UNLOCK(unp);
695 	free(buf, M_TEMP);
696 	return (error);
697 }
698 
699 static int
700 uipc_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
701 {
702 
703 	return (uipc_bindat(AT_FDCWD, so, nam, td));
704 }
705 
706 static int
707 uipc_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
708 {
709 	int error;
710 
711 	KASSERT(td == curthread, ("uipc_connect: td != curthread"));
712 	error = unp_connect(so, nam, td);
713 	return (error);
714 }
715 
716 static int
717 uipc_connectat(int fd, struct socket *so, struct sockaddr *nam,
718     struct thread *td)
719 {
720 	int error;
721 
722 	KASSERT(td == curthread, ("uipc_connectat: td != curthread"));
723 	error = unp_connectat(fd, so, nam, td);
724 	return (error);
725 }
726 
727 static void
728 uipc_close(struct socket *so)
729 {
730 	struct unpcb *unp, *unp2;
731 	struct vnode *vp = NULL;
732 	struct mtx *vplock;
733 
734 	unp = sotounpcb(so);
735 	KASSERT(unp != NULL, ("uipc_close: unp == NULL"));
736 
737 	vplock = NULL;
738 	if ((vp = unp->unp_vnode) != NULL) {
739 		vplock = mtx_pool_find(mtxpool_sleep, vp);
740 		mtx_lock(vplock);
741 	}
742 	UNP_PCB_LOCK(unp);
743 	if (vp && unp->unp_vnode == NULL) {
744 		mtx_unlock(vplock);
745 		vp = NULL;
746 	}
747 	if (vp != NULL) {
748 		VOP_UNP_DETACH(vp);
749 		unp->unp_vnode = NULL;
750 	}
751 	if ((unp2 = unp_pcb_lock_peer(unp)) != NULL)
752 		unp_disconnect(unp, unp2);
753 	else
754 		UNP_PCB_UNLOCK(unp);
755 	if (vp) {
756 		mtx_unlock(vplock);
757 		vrele(vp);
758 	}
759 }
760 
761 static int
762 uipc_connect2(struct socket *so1, struct socket *so2)
763 {
764 	struct unpcb *unp, *unp2;
765 	int error;
766 
767 	unp = so1->so_pcb;
768 	KASSERT(unp != NULL, ("uipc_connect2: unp == NULL"));
769 	unp2 = so2->so_pcb;
770 	KASSERT(unp2 != NULL, ("uipc_connect2: unp2 == NULL"));
771 	unp_pcb_lock_pair(unp, unp2);
772 	error = unp_connect2(so1, so2, PRU_CONNECT2);
773 	unp_pcb_unlock_pair(unp, unp2);
774 	return (error);
775 }
776 
777 static void
778 uipc_detach(struct socket *so)
779 {
780 	struct unpcb *unp, *unp2;
781 	struct mtx *vplock;
782 	struct vnode *vp;
783 	int local_unp_rights;
784 
785 	unp = sotounpcb(so);
786 	KASSERT(unp != NULL, ("uipc_detach: unp == NULL"));
787 
788 	vp = NULL;
789 	vplock = NULL;
790 
791 	SOCK_LOCK(so);
792 	if (!SOLISTENING(so)) {
793 		/*
794 		 * Once the socket is removed from the global lists,
795 		 * uipc_ready() will not be able to locate its socket buffer, so
796 		 * clear the buffer now.  At this point internalized rights have
797 		 * already been disposed of.
798 		 */
799 		sbrelease(&so->so_rcv, so);
800 	}
801 	SOCK_UNLOCK(so);
802 
803 	UNP_LINK_WLOCK();
804 	LIST_REMOVE(unp, unp_link);
805 	if (unp->unp_gcflag & UNPGC_DEAD)
806 		LIST_REMOVE(unp, unp_dead);
807 	unp->unp_gencnt = ++unp_gencnt;
808 	--unp_count;
809 	UNP_LINK_WUNLOCK();
810 
811 	UNP_PCB_UNLOCK_ASSERT(unp);
812  restart:
813 	if ((vp = unp->unp_vnode) != NULL) {
814 		vplock = mtx_pool_find(mtxpool_sleep, vp);
815 		mtx_lock(vplock);
816 	}
817 	UNP_PCB_LOCK(unp);
818 	if (unp->unp_vnode != vp && unp->unp_vnode != NULL) {
819 		if (vplock)
820 			mtx_unlock(vplock);
821 		UNP_PCB_UNLOCK(unp);
822 		goto restart;
823 	}
824 	if ((vp = unp->unp_vnode) != NULL) {
825 		VOP_UNP_DETACH(vp);
826 		unp->unp_vnode = NULL;
827 	}
828 	if ((unp2 = unp_pcb_lock_peer(unp)) != NULL)
829 		unp_disconnect(unp, unp2);
830 	else
831 		UNP_PCB_UNLOCK(unp);
832 
833 	UNP_REF_LIST_LOCK();
834 	while (!LIST_EMPTY(&unp->unp_refs)) {
835 		struct unpcb *ref = LIST_FIRST(&unp->unp_refs);
836 
837 		unp_pcb_hold(ref);
838 		UNP_REF_LIST_UNLOCK();
839 
840 		MPASS(ref != unp);
841 		UNP_PCB_UNLOCK_ASSERT(ref);
842 		unp_drop(ref);
843 		UNP_REF_LIST_LOCK();
844 	}
845 	UNP_REF_LIST_UNLOCK();
846 
847 	UNP_PCB_LOCK(unp);
848 	local_unp_rights = unp_rights;
849 	unp->unp_socket->so_pcb = NULL;
850 	unp->unp_socket = NULL;
851 	free(unp->unp_addr, M_SONAME);
852 	unp->unp_addr = NULL;
853 	if (!unp_pcb_rele(unp))
854 		UNP_PCB_UNLOCK(unp);
855 	if (vp) {
856 		mtx_unlock(vplock);
857 		vrele(vp);
858 	}
859 	if (local_unp_rights)
860 		taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1);
861 }
862 
863 static int
864 uipc_disconnect(struct socket *so)
865 {
866 	struct unpcb *unp, *unp2;
867 
868 	unp = sotounpcb(so);
869 	KASSERT(unp != NULL, ("uipc_disconnect: unp == NULL"));
870 
871 	UNP_PCB_LOCK(unp);
872 	if ((unp2 = unp_pcb_lock_peer(unp)) != NULL)
873 		unp_disconnect(unp, unp2);
874 	else
875 		UNP_PCB_UNLOCK(unp);
876 	return (0);
877 }
878 
879 static int
880 uipc_listen(struct socket *so, int backlog, struct thread *td)
881 {
882 	struct unpcb *unp;
883 	int error;
884 
885 	if (so->so_type != SOCK_STREAM && so->so_type != SOCK_SEQPACKET)
886 		return (EOPNOTSUPP);
887 
888 	unp = sotounpcb(so);
889 	KASSERT(unp != NULL, ("uipc_listen: unp == NULL"));
890 
891 	UNP_PCB_LOCK(unp);
892 	if (unp->unp_vnode == NULL) {
893 		/* Already connected or not bound to an address. */
894 		error = unp->unp_conn != NULL ? EINVAL : EDESTADDRREQ;
895 		UNP_PCB_UNLOCK(unp);
896 		return (error);
897 	}
898 
899 	SOCK_LOCK(so);
900 	error = solisten_proto_check(so);
901 	if (error == 0) {
902 		cru2xt(td, &unp->unp_peercred);
903 		solisten_proto(so, backlog);
904 	}
905 	SOCK_UNLOCK(so);
906 	UNP_PCB_UNLOCK(unp);
907 	return (error);
908 }
909 
910 static int
911 uipc_peeraddr(struct socket *so, struct sockaddr **nam)
912 {
913 	struct unpcb *unp, *unp2;
914 	const struct sockaddr *sa;
915 
916 	unp = sotounpcb(so);
917 	KASSERT(unp != NULL, ("uipc_peeraddr: unp == NULL"));
918 
919 	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
920 	UNP_LINK_RLOCK();
921 	/*
922 	 * XXX: It seems that this test always fails even when connection is
923 	 * established.  So, this else clause is added as workaround to
924 	 * return PF_LOCAL sockaddr.
925 	 */
926 	unp2 = unp->unp_conn;
927 	if (unp2 != NULL) {
928 		UNP_PCB_LOCK(unp2);
929 		if (unp2->unp_addr != NULL)
930 			sa = (struct sockaddr *) unp2->unp_addr;
931 		else
932 			sa = &sun_noname;
933 		bcopy(sa, *nam, sa->sa_len);
934 		UNP_PCB_UNLOCK(unp2);
935 	} else {
936 		sa = &sun_noname;
937 		bcopy(sa, *nam, sa->sa_len);
938 	}
939 	UNP_LINK_RUNLOCK();
940 	return (0);
941 }
942 
943 static int
944 uipc_rcvd(struct socket *so, int flags)
945 {
946 	struct unpcb *unp, *unp2;
947 	struct socket *so2;
948 	u_int mbcnt, sbcc;
949 
950 	unp = sotounpcb(so);
951 	KASSERT(unp != NULL, ("%s: unp == NULL", __func__));
952 	KASSERT(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET,
953 	    ("%s: socktype %d", __func__, so->so_type));
954 
955 	/*
956 	 * Adjust backpressure on sender and wakeup any waiting to write.
957 	 *
958 	 * The unp lock is acquired to maintain the validity of the unp_conn
959 	 * pointer; no lock on unp2 is required as unp2->unp_socket will be
960 	 * static as long as we don't permit unp2 to disconnect from unp,
961 	 * which is prevented by the lock on unp.  We cache values from
962 	 * so_rcv to avoid holding the so_rcv lock over the entire
963 	 * transaction on the remote so_snd.
964 	 */
965 	SOCKBUF_LOCK(&so->so_rcv);
966 	mbcnt = so->so_rcv.sb_mbcnt;
967 	sbcc = sbavail(&so->so_rcv);
968 	SOCKBUF_UNLOCK(&so->so_rcv);
969 	/*
970 	 * There is a benign race condition at this point.  If we're planning to
971 	 * clear SB_STOP, but uipc_send is called on the connected socket at
972 	 * this instant, it might add data to the sockbuf and set SB_STOP.  Then
973 	 * we would erroneously clear SB_STOP below, even though the sockbuf is
974 	 * full.  The race is benign because the only ill effect is to allow the
975 	 * sockbuf to exceed its size limit, and the size limits are not
976 	 * strictly guaranteed anyway.
977 	 */
978 	UNP_PCB_LOCK(unp);
979 	unp2 = unp->unp_conn;
980 	if (unp2 == NULL) {
981 		UNP_PCB_UNLOCK(unp);
982 		return (0);
983 	}
984 	so2 = unp2->unp_socket;
985 	SOCKBUF_LOCK(&so2->so_snd);
986 	if (sbcc < so2->so_snd.sb_hiwat && mbcnt < so2->so_snd.sb_mbmax)
987 		so2->so_snd.sb_flags &= ~SB_STOP;
988 	sowwakeup_locked(so2);
989 	UNP_PCB_UNLOCK(unp);
990 	return (0);
991 }
992 
993 static int
994 uipc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *nam,
995     struct mbuf *control, struct thread *td)
996 {
997 	struct unpcb *unp, *unp2;
998 	struct socket *so2;
999 	u_int mbcnt, sbcc;
1000 	int freed, error;
1001 
1002 	unp = sotounpcb(so);
1003 	KASSERT(unp != NULL, ("%s: unp == NULL", __func__));
1004 	KASSERT(so->so_type == SOCK_STREAM || so->so_type == SOCK_DGRAM ||
1005 	    so->so_type == SOCK_SEQPACKET,
1006 	    ("%s: socktype %d", __func__, so->so_type));
1007 
1008 	freed = error = 0;
1009 	if (flags & PRUS_OOB) {
1010 		error = EOPNOTSUPP;
1011 		goto release;
1012 	}
1013 	if (control != NULL && (error = unp_internalize(&control, td)))
1014 		goto release;
1015 
1016 	unp2 = NULL;
1017 	switch (so->so_type) {
1018 	case SOCK_DGRAM:
1019 	{
1020 		const struct sockaddr *from;
1021 
1022 		if (nam != NULL) {
1023 			error = unp_connect(so, nam, td);
1024 			if (error != 0)
1025 				break;
1026 		}
1027 		UNP_PCB_LOCK(unp);
1028 
1029 		/*
1030 		 * Because connect() and send() are non-atomic in a sendto()
1031 		 * with a target address, it's possible that the socket will
1032 		 * have disconnected before the send() can run.  In that case
1033 		 * return the slightly counter-intuitive but otherwise
1034 		 * correct error that the socket is not connected.
1035 		 */
1036 		unp2 = unp_pcb_lock_peer(unp);
1037 		if (unp2 == NULL) {
1038 			UNP_PCB_UNLOCK(unp);
1039 			error = ENOTCONN;
1040 			break;
1041 		}
1042 
1043 		if (unp2->unp_flags & UNP_WANTCRED_MASK)
1044 			control = unp_addsockcred(td, control);
1045 		if (unp->unp_addr != NULL)
1046 			from = (struct sockaddr *)unp->unp_addr;
1047 		else
1048 			from = &sun_noname;
1049 		so2 = unp2->unp_socket;
1050 		SOCKBUF_LOCK(&so2->so_rcv);
1051 		if (sbappendaddr_locked(&so2->so_rcv, from, m,
1052 		    control)) {
1053 			sorwakeup_locked(so2);
1054 			m = NULL;
1055 			control = NULL;
1056 		} else {
1057 			SOCKBUF_UNLOCK(&so2->so_rcv);
1058 			error = ENOBUFS;
1059 		}
1060 		if (nam != NULL)
1061 			unp_disconnect(unp, unp2);
1062 		else
1063 			unp_pcb_unlock_pair(unp, unp2);
1064 		break;
1065 	}
1066 
1067 	case SOCK_SEQPACKET:
1068 	case SOCK_STREAM:
1069 		if ((so->so_state & SS_ISCONNECTED) == 0) {
1070 			if (nam != NULL) {
1071 				error = unp_connect(so, nam, td);
1072 				if (error != 0)
1073 					break;
1074 			} else {
1075 				error = ENOTCONN;
1076 				break;
1077 			}
1078 		}
1079 
1080 		UNP_PCB_LOCK(unp);
1081 		if ((unp2 = unp_pcb_lock_peer(unp)) == NULL) {
1082 			UNP_PCB_UNLOCK(unp);
1083 			error = ENOTCONN;
1084 			break;
1085 		} else if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
1086 			unp_pcb_unlock_pair(unp, unp2);
1087 			error = EPIPE;
1088 			break;
1089 		}
1090 		UNP_PCB_UNLOCK(unp);
1091 		if ((so2 = unp2->unp_socket) == NULL) {
1092 			UNP_PCB_UNLOCK(unp2);
1093 			error = ENOTCONN;
1094 			break;
1095 		}
1096 		SOCKBUF_LOCK(&so2->so_rcv);
1097 		if (unp2->unp_flags & UNP_WANTCRED_MASK) {
1098 			/*
1099 			 * Credentials are passed only once on SOCK_STREAM and
1100 			 * SOCK_SEQPACKET (LOCAL_CREDS => WANTCRED_ONESHOT), or
1101 			 * forever (LOCAL_CREDS_PERSISTENT => WANTCRED_ALWAYS).
1102 			 */
1103 			unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT;
1104 			control = unp_addsockcred(td, control);
1105 		}
1106 
1107 		/*
1108 		 * Send to paired receive port and wake up readers.  Don't
1109 		 * check for space available in the receive buffer if we're
1110 		 * attaching ancillary data; Unix domain sockets only check
1111 		 * for space in the sending sockbuf, and that check is
1112 		 * performed one level up the stack.  At that level we cannot
1113 		 * precisely account for the amount of buffer space used
1114 		 * (e.g., because control messages are not yet internalized).
1115 		 */
1116 		switch (so->so_type) {
1117 		case SOCK_STREAM:
1118 			if (control != NULL) {
1119 				sbappendcontrol_locked(&so2->so_rcv, m,
1120 				    control, flags);
1121 				control = NULL;
1122 			} else
1123 				sbappend_locked(&so2->so_rcv, m, flags);
1124 			break;
1125 
1126 		case SOCK_SEQPACKET:
1127 			if (sbappendaddr_nospacecheck_locked(&so2->so_rcv,
1128 			    &sun_noname, m, control))
1129 				control = NULL;
1130 			break;
1131 		}
1132 
1133 		mbcnt = so2->so_rcv.sb_mbcnt;
1134 		sbcc = sbavail(&so2->so_rcv);
1135 		if (sbcc)
1136 			sorwakeup_locked(so2);
1137 		else
1138 			SOCKBUF_UNLOCK(&so2->so_rcv);
1139 
1140 		/*
1141 		 * The PCB lock on unp2 protects the SB_STOP flag.  Without it,
1142 		 * it would be possible for uipc_rcvd to be called at this
1143 		 * point, drain the receiving sockbuf, clear SB_STOP, and then
1144 		 * we would set SB_STOP below.  That could lead to an empty
1145 		 * sockbuf having SB_STOP set
1146 		 */
1147 		SOCKBUF_LOCK(&so->so_snd);
1148 		if (sbcc >= so->so_snd.sb_hiwat || mbcnt >= so->so_snd.sb_mbmax)
1149 			so->so_snd.sb_flags |= SB_STOP;
1150 		SOCKBUF_UNLOCK(&so->so_snd);
1151 		UNP_PCB_UNLOCK(unp2);
1152 		m = NULL;
1153 		break;
1154 	}
1155 
1156 	/*
1157 	 * PRUS_EOF is equivalent to pru_send followed by pru_shutdown.
1158 	 */
1159 	if (flags & PRUS_EOF) {
1160 		UNP_PCB_LOCK(unp);
1161 		socantsendmore(so);
1162 		unp_shutdown(unp);
1163 		UNP_PCB_UNLOCK(unp);
1164 	}
1165 	if (control != NULL && error != 0)
1166 		unp_dispose_mbuf(control);
1167 
1168 release:
1169 	if (control != NULL)
1170 		m_freem(control);
1171 	/*
1172 	 * In case of PRUS_NOTREADY, uipc_ready() is responsible
1173 	 * for freeing memory.
1174 	 */
1175 	if (m != NULL && (flags & PRUS_NOTREADY) == 0)
1176 		m_freem(m);
1177 	return (error);
1178 }
1179 
1180 static bool
1181 uipc_ready_scan(struct socket *so, struct mbuf *m, int count, int *errorp)
1182 {
1183 	struct mbuf *mb, *n;
1184 	struct sockbuf *sb;
1185 
1186 	SOCK_LOCK(so);
1187 	if (SOLISTENING(so)) {
1188 		SOCK_UNLOCK(so);
1189 		return (false);
1190 	}
1191 	mb = NULL;
1192 	sb = &so->so_rcv;
1193 	SOCKBUF_LOCK(sb);
1194 	if (sb->sb_fnrdy != NULL) {
1195 		for (mb = sb->sb_mb, n = mb->m_nextpkt; mb != NULL;) {
1196 			if (mb == m) {
1197 				*errorp = sbready(sb, m, count);
1198 				break;
1199 			}
1200 			mb = mb->m_next;
1201 			if (mb == NULL) {
1202 				mb = n;
1203 				if (mb != NULL)
1204 					n = mb->m_nextpkt;
1205 			}
1206 		}
1207 	}
1208 	SOCKBUF_UNLOCK(sb);
1209 	SOCK_UNLOCK(so);
1210 	return (mb != NULL);
1211 }
1212 
1213 static int
1214 uipc_ready(struct socket *so, struct mbuf *m, int count)
1215 {
1216 	struct unpcb *unp, *unp2;
1217 	struct socket *so2;
1218 	int error, i;
1219 
1220 	unp = sotounpcb(so);
1221 
1222 	KASSERT(so->so_type == SOCK_STREAM,
1223 	    ("%s: unexpected socket type for %p", __func__, so));
1224 
1225 	UNP_PCB_LOCK(unp);
1226 	if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) {
1227 		UNP_PCB_UNLOCK(unp);
1228 		so2 = unp2->unp_socket;
1229 		SOCKBUF_LOCK(&so2->so_rcv);
1230 		if ((error = sbready(&so2->so_rcv, m, count)) == 0)
1231 			sorwakeup_locked(so2);
1232 		else
1233 			SOCKBUF_UNLOCK(&so2->so_rcv);
1234 		UNP_PCB_UNLOCK(unp2);
1235 		return (error);
1236 	}
1237 	UNP_PCB_UNLOCK(unp);
1238 
1239 	/*
1240 	 * The receiving socket has been disconnected, but may still be valid.
1241 	 * In this case, the now-ready mbufs are still present in its socket
1242 	 * buffer, so perform an exhaustive search before giving up and freeing
1243 	 * the mbufs.
1244 	 */
1245 	UNP_LINK_RLOCK();
1246 	LIST_FOREACH(unp, &unp_shead, unp_link) {
1247 		if (uipc_ready_scan(unp->unp_socket, m, count, &error))
1248 			break;
1249 	}
1250 	UNP_LINK_RUNLOCK();
1251 
1252 	if (unp == NULL) {
1253 		for (i = 0; i < count; i++)
1254 			m = m_free(m);
1255 		error = ECONNRESET;
1256 	}
1257 	return (error);
1258 }
1259 
1260 static int
1261 uipc_sense(struct socket *so, struct stat *sb)
1262 {
1263 	struct unpcb *unp;
1264 
1265 	unp = sotounpcb(so);
1266 	KASSERT(unp != NULL, ("uipc_sense: unp == NULL"));
1267 
1268 	sb->st_blksize = so->so_snd.sb_hiwat;
1269 	sb->st_dev = NODEV;
1270 	sb->st_ino = unp->unp_ino;
1271 	return (0);
1272 }
1273 
1274 static int
1275 uipc_shutdown(struct socket *so)
1276 {
1277 	struct unpcb *unp;
1278 
1279 	unp = sotounpcb(so);
1280 	KASSERT(unp != NULL, ("uipc_shutdown: unp == NULL"));
1281 
1282 	UNP_PCB_LOCK(unp);
1283 	socantsendmore(so);
1284 	unp_shutdown(unp);
1285 	UNP_PCB_UNLOCK(unp);
1286 	return (0);
1287 }
1288 
1289 static int
1290 uipc_sockaddr(struct socket *so, struct sockaddr **nam)
1291 {
1292 	struct unpcb *unp;
1293 	const struct sockaddr *sa;
1294 
1295 	unp = sotounpcb(so);
1296 	KASSERT(unp != NULL, ("uipc_sockaddr: unp == NULL"));
1297 
1298 	*nam = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
1299 	UNP_PCB_LOCK(unp);
1300 	if (unp->unp_addr != NULL)
1301 		sa = (struct sockaddr *) unp->unp_addr;
1302 	else
1303 		sa = &sun_noname;
1304 	bcopy(sa, *nam, sa->sa_len);
1305 	UNP_PCB_UNLOCK(unp);
1306 	return (0);
1307 }
1308 
1309 static struct pr_usrreqs uipc_usrreqs_dgram = {
1310 	.pru_abort = 		uipc_abort,
1311 	.pru_accept =		uipc_accept,
1312 	.pru_attach =		uipc_attach,
1313 	.pru_bind =		uipc_bind,
1314 	.pru_bindat =		uipc_bindat,
1315 	.pru_connect =		uipc_connect,
1316 	.pru_connectat =	uipc_connectat,
1317 	.pru_connect2 =		uipc_connect2,
1318 	.pru_detach =		uipc_detach,
1319 	.pru_disconnect =	uipc_disconnect,
1320 	.pru_listen =		uipc_listen,
1321 	.pru_peeraddr =		uipc_peeraddr,
1322 	.pru_rcvd =		uipc_rcvd,
1323 	.pru_send =		uipc_send,
1324 	.pru_sense =		uipc_sense,
1325 	.pru_shutdown =		uipc_shutdown,
1326 	.pru_sockaddr =		uipc_sockaddr,
1327 	.pru_soreceive =	soreceive_dgram,
1328 	.pru_close =		uipc_close,
1329 };
1330 
1331 static struct pr_usrreqs uipc_usrreqs_seqpacket = {
1332 	.pru_abort =		uipc_abort,
1333 	.pru_accept =		uipc_accept,
1334 	.pru_attach =		uipc_attach,
1335 	.pru_bind =		uipc_bind,
1336 	.pru_bindat =		uipc_bindat,
1337 	.pru_connect =		uipc_connect,
1338 	.pru_connectat =	uipc_connectat,
1339 	.pru_connect2 =		uipc_connect2,
1340 	.pru_detach =		uipc_detach,
1341 	.pru_disconnect =	uipc_disconnect,
1342 	.pru_listen =		uipc_listen,
1343 	.pru_peeraddr =		uipc_peeraddr,
1344 	.pru_rcvd =		uipc_rcvd,
1345 	.pru_send =		uipc_send,
1346 	.pru_sense =		uipc_sense,
1347 	.pru_shutdown =		uipc_shutdown,
1348 	.pru_sockaddr =		uipc_sockaddr,
1349 	.pru_soreceive =	soreceive_generic,	/* XXX: or...? */
1350 	.pru_close =		uipc_close,
1351 };
1352 
1353 static struct pr_usrreqs uipc_usrreqs_stream = {
1354 	.pru_abort = 		uipc_abort,
1355 	.pru_accept =		uipc_accept,
1356 	.pru_attach =		uipc_attach,
1357 	.pru_bind =		uipc_bind,
1358 	.pru_bindat =		uipc_bindat,
1359 	.pru_connect =		uipc_connect,
1360 	.pru_connectat =	uipc_connectat,
1361 	.pru_connect2 =		uipc_connect2,
1362 	.pru_detach =		uipc_detach,
1363 	.pru_disconnect =	uipc_disconnect,
1364 	.pru_listen =		uipc_listen,
1365 	.pru_peeraddr =		uipc_peeraddr,
1366 	.pru_rcvd =		uipc_rcvd,
1367 	.pru_send =		uipc_send,
1368 	.pru_ready =		uipc_ready,
1369 	.pru_sense =		uipc_sense,
1370 	.pru_shutdown =		uipc_shutdown,
1371 	.pru_sockaddr =		uipc_sockaddr,
1372 	.pru_soreceive =	soreceive_generic,
1373 	.pru_close =		uipc_close,
1374 };
1375 
1376 static int
1377 uipc_ctloutput(struct socket *so, struct sockopt *sopt)
1378 {
1379 	struct unpcb *unp;
1380 	struct xucred xu;
1381 	int error, optval;
1382 
1383 	if (sopt->sopt_level != SOL_LOCAL)
1384 		return (EINVAL);
1385 
1386 	unp = sotounpcb(so);
1387 	KASSERT(unp != NULL, ("uipc_ctloutput: unp == NULL"));
1388 	error = 0;
1389 	switch (sopt->sopt_dir) {
1390 	case SOPT_GET:
1391 		switch (sopt->sopt_name) {
1392 		case LOCAL_PEERCRED:
1393 			UNP_PCB_LOCK(unp);
1394 			if (unp->unp_flags & UNP_HAVEPC)
1395 				xu = unp->unp_peercred;
1396 			else {
1397 				if (so->so_type == SOCK_STREAM)
1398 					error = ENOTCONN;
1399 				else
1400 					error = EINVAL;
1401 			}
1402 			UNP_PCB_UNLOCK(unp);
1403 			if (error == 0)
1404 				error = sooptcopyout(sopt, &xu, sizeof(xu));
1405 			break;
1406 
1407 		case LOCAL_CREDS:
1408 			/* Unlocked read. */
1409 			optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0;
1410 			error = sooptcopyout(sopt, &optval, sizeof(optval));
1411 			break;
1412 
1413 		case LOCAL_CREDS_PERSISTENT:
1414 			/* Unlocked read. */
1415 			optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0;
1416 			error = sooptcopyout(sopt, &optval, sizeof(optval));
1417 			break;
1418 
1419 		case LOCAL_CONNWAIT:
1420 			/* Unlocked read. */
1421 			optval = unp->unp_flags & UNP_CONNWAIT ? 1 : 0;
1422 			error = sooptcopyout(sopt, &optval, sizeof(optval));
1423 			break;
1424 
1425 		default:
1426 			error = EOPNOTSUPP;
1427 			break;
1428 		}
1429 		break;
1430 
1431 	case SOPT_SET:
1432 		switch (sopt->sopt_name) {
1433 		case LOCAL_CREDS:
1434 		case LOCAL_CREDS_PERSISTENT:
1435 		case LOCAL_CONNWAIT:
1436 			error = sooptcopyin(sopt, &optval, sizeof(optval),
1437 					    sizeof(optval));
1438 			if (error)
1439 				break;
1440 
1441 #define	OPTSET(bit, exclusive) do {					\
1442 	UNP_PCB_LOCK(unp);						\
1443 	if (optval) {							\
1444 		if ((unp->unp_flags & (exclusive)) != 0) {		\
1445 			UNP_PCB_UNLOCK(unp);				\
1446 			error = EINVAL;					\
1447 			break;						\
1448 		}							\
1449 		unp->unp_flags |= (bit);				\
1450 	} else								\
1451 		unp->unp_flags &= ~(bit);				\
1452 	UNP_PCB_UNLOCK(unp);						\
1453 } while (0)
1454 
1455 			switch (sopt->sopt_name) {
1456 			case LOCAL_CREDS:
1457 				OPTSET(UNP_WANTCRED_ONESHOT, UNP_WANTCRED_ALWAYS);
1458 				break;
1459 
1460 			case LOCAL_CREDS_PERSISTENT:
1461 				OPTSET(UNP_WANTCRED_ALWAYS, UNP_WANTCRED_ONESHOT);
1462 				break;
1463 
1464 			case LOCAL_CONNWAIT:
1465 				OPTSET(UNP_CONNWAIT, 0);
1466 				break;
1467 
1468 			default:
1469 				break;
1470 			}
1471 			break;
1472 #undef	OPTSET
1473 		default:
1474 			error = ENOPROTOOPT;
1475 			break;
1476 		}
1477 		break;
1478 
1479 	default:
1480 		error = EOPNOTSUPP;
1481 		break;
1482 	}
1483 	return (error);
1484 }
1485 
1486 static int
1487 unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
1488 {
1489 
1490 	return (unp_connectat(AT_FDCWD, so, nam, td));
1491 }
1492 
1493 static int
1494 unp_connectat(int fd, struct socket *so, struct sockaddr *nam,
1495     struct thread *td)
1496 {
1497 	struct mtx *vplock;
1498 	struct sockaddr_un *soun;
1499 	struct vnode *vp;
1500 	struct socket *so2;
1501 	struct unpcb *unp, *unp2, *unp3;
1502 	struct nameidata nd;
1503 	char buf[SOCK_MAXADDRLEN];
1504 	struct sockaddr *sa;
1505 	cap_rights_t rights;
1506 	int error, len;
1507 	bool connreq;
1508 
1509 	if (nam->sa_family != AF_UNIX)
1510 		return (EAFNOSUPPORT);
1511 	if (nam->sa_len > sizeof(struct sockaddr_un))
1512 		return (EINVAL);
1513 	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
1514 	if (len <= 0)
1515 		return (EINVAL);
1516 	soun = (struct sockaddr_un *)nam;
1517 	bcopy(soun->sun_path, buf, len);
1518 	buf[len] = 0;
1519 
1520 	unp = sotounpcb(so);
1521 	UNP_PCB_LOCK(unp);
1522 	for (;;) {
1523 		/*
1524 		 * Wait for connection state to stabilize.  If a connection
1525 		 * already exists, give up.  For datagram sockets, which permit
1526 		 * multiple consecutive connect(2) calls, upper layers are
1527 		 * responsible for disconnecting in advance of a subsequent
1528 		 * connect(2), but this is not synchronized with PCB connection
1529 		 * state.
1530 		 *
1531 		 * Also make sure that no threads are currently attempting to
1532 		 * lock the peer socket, to ensure that unp_conn cannot
1533 		 * transition between two valid sockets while locks are dropped.
1534 		 */
1535 		if (unp->unp_conn != NULL) {
1536 			UNP_PCB_UNLOCK(unp);
1537 			return (EISCONN);
1538 		}
1539 		if ((unp->unp_flags & UNP_CONNECTING) != 0) {
1540 			UNP_PCB_UNLOCK(unp);
1541 			return (EALREADY);
1542 		}
1543 		if (unp->unp_pairbusy > 0) {
1544 			unp->unp_flags |= UNP_WAITING;
1545 			mtx_sleep(unp, UNP_PCB_LOCKPTR(unp), 0, "unpeer", 0);
1546 			continue;
1547 		}
1548 		break;
1549 	}
1550 	unp->unp_flags |= UNP_CONNECTING;
1551 	UNP_PCB_UNLOCK(unp);
1552 
1553 	connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0;
1554 	if (connreq)
1555 		sa = malloc(sizeof(struct sockaddr_un), M_SONAME, M_WAITOK);
1556 	else
1557 		sa = NULL;
1558 	NDINIT_ATRIGHTS(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF,
1559 	    UIO_SYSSPACE, buf, fd, cap_rights_init(&rights, CAP_CONNECTAT), td);
1560 	error = namei(&nd);
1561 	if (error)
1562 		vp = NULL;
1563 	else
1564 		vp = nd.ni_vp;
1565 	ASSERT_VOP_LOCKED(vp, "unp_connect");
1566 	NDFREE(&nd, NDF_ONLY_PNBUF);
1567 	if (error)
1568 		goto bad;
1569 
1570 	if (vp->v_type != VSOCK) {
1571 		error = ENOTSOCK;
1572 		goto bad;
1573 	}
1574 #ifdef MAC
1575 	error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD);
1576 	if (error)
1577 		goto bad;
1578 #endif
1579 	error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td);
1580 	if (error)
1581 		goto bad;
1582 
1583 	unp = sotounpcb(so);
1584 	KASSERT(unp != NULL, ("unp_connect: unp == NULL"));
1585 
1586 	vplock = mtx_pool_find(mtxpool_sleep, vp);
1587 	mtx_lock(vplock);
1588 	VOP_UNP_CONNECT(vp, &unp2);
1589 	if (unp2 == NULL) {
1590 		error = ECONNREFUSED;
1591 		goto bad2;
1592 	}
1593 	so2 = unp2->unp_socket;
1594 	if (so->so_type != so2->so_type) {
1595 		error = EPROTOTYPE;
1596 		goto bad2;
1597 	}
1598 	if (connreq) {
1599 		if (so2->so_options & SO_ACCEPTCONN) {
1600 			CURVNET_SET(so2->so_vnet);
1601 			so2 = sonewconn(so2, 0);
1602 			CURVNET_RESTORE();
1603 		} else
1604 			so2 = NULL;
1605 		if (so2 == NULL) {
1606 			error = ECONNREFUSED;
1607 			goto bad2;
1608 		}
1609 		unp3 = sotounpcb(so2);
1610 		unp_pcb_lock_pair(unp2, unp3);
1611 		if (unp2->unp_addr != NULL) {
1612 			bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len);
1613 			unp3->unp_addr = (struct sockaddr_un *) sa;
1614 			sa = NULL;
1615 		}
1616 
1617 		unp_copy_peercred(td, unp3, unp, unp2);
1618 
1619 		UNP_PCB_UNLOCK(unp2);
1620 		unp2 = unp3;
1621 
1622 		/*
1623 		 * It is safe to block on the PCB lock here since unp2 is
1624 		 * nascent and cannot be connected to any other sockets.
1625 		 */
1626 		UNP_PCB_LOCK(unp);
1627 #ifdef MAC
1628 		mac_socketpeer_set_from_socket(so, so2);
1629 		mac_socketpeer_set_from_socket(so2, so);
1630 #endif
1631 	} else {
1632 		unp_pcb_lock_pair(unp, unp2);
1633 	}
1634 	KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 &&
1635 	    sotounpcb(so2) == unp2,
1636 	    ("%s: unp2 %p so2 %p", __func__, unp2, so2));
1637 	error = unp_connect2(so, so2, PRU_CONNECT);
1638 	unp_pcb_unlock_pair(unp, unp2);
1639 bad2:
1640 	mtx_unlock(vplock);
1641 bad:
1642 	if (vp != NULL) {
1643 		vput(vp);
1644 	}
1645 	free(sa, M_SONAME);
1646 	UNP_PCB_LOCK(unp);
1647 	KASSERT((unp->unp_flags & UNP_CONNECTING) != 0,
1648 	    ("%s: unp %p has UNP_CONNECTING clear", __func__, unp));
1649 	unp->unp_flags &= ~UNP_CONNECTING;
1650 	UNP_PCB_UNLOCK(unp);
1651 	return (error);
1652 }
1653 
1654 /*
1655  * Set socket peer credentials at connection time.
1656  *
1657  * The client's PCB credentials are copied from its process structure.  The
1658  * server's PCB credentials are copied from the socket on which it called
1659  * listen(2).  uipc_listen cached that process's credentials at the time.
1660  */
1661 void
1662 unp_copy_peercred(struct thread *td, struct unpcb *client_unp,
1663     struct unpcb *server_unp, struct unpcb *listen_unp)
1664 {
1665 	cru2xt(td, &client_unp->unp_peercred);
1666 	client_unp->unp_flags |= UNP_HAVEPC;
1667 
1668 	memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred,
1669 	    sizeof(server_unp->unp_peercred));
1670 	server_unp->unp_flags |= UNP_HAVEPC;
1671 	client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK);
1672 }
1673 
1674 static int
1675 unp_connect2(struct socket *so, struct socket *so2, int req)
1676 {
1677 	struct unpcb *unp;
1678 	struct unpcb *unp2;
1679 
1680 	unp = sotounpcb(so);
1681 	KASSERT(unp != NULL, ("unp_connect2: unp == NULL"));
1682 	unp2 = sotounpcb(so2);
1683 	KASSERT(unp2 != NULL, ("unp_connect2: unp2 == NULL"));
1684 
1685 	UNP_PCB_LOCK_ASSERT(unp);
1686 	UNP_PCB_LOCK_ASSERT(unp2);
1687 	KASSERT(unp->unp_conn == NULL,
1688 	    ("%s: socket %p is already connected", __func__, unp));
1689 
1690 	if (so2->so_type != so->so_type)
1691 		return (EPROTOTYPE);
1692 	unp->unp_conn = unp2;
1693 	unp_pcb_hold(unp2);
1694 	unp_pcb_hold(unp);
1695 	switch (so->so_type) {
1696 	case SOCK_DGRAM:
1697 		UNP_REF_LIST_LOCK();
1698 		LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink);
1699 		UNP_REF_LIST_UNLOCK();
1700 		soisconnected(so);
1701 		break;
1702 
1703 	case SOCK_STREAM:
1704 	case SOCK_SEQPACKET:
1705 		KASSERT(unp2->unp_conn == NULL,
1706 		    ("%s: socket %p is already connected", __func__, unp2));
1707 		unp2->unp_conn = unp;
1708 		if (req == PRU_CONNECT &&
1709 		    ((unp->unp_flags | unp2->unp_flags) & UNP_CONNWAIT))
1710 			soisconnecting(so);
1711 		else
1712 			soisconnected(so);
1713 		soisconnected(so2);
1714 		break;
1715 
1716 	default:
1717 		panic("unp_connect2");
1718 	}
1719 	return (0);
1720 }
1721 
1722 static void
1723 unp_disconnect(struct unpcb *unp, struct unpcb *unp2)
1724 {
1725 	struct socket *so, *so2;
1726 #ifdef INVARIANTS
1727 	struct unpcb *unptmp;
1728 #endif
1729 
1730 	UNP_PCB_LOCK_ASSERT(unp);
1731 	UNP_PCB_LOCK_ASSERT(unp2);
1732 	KASSERT(unp->unp_conn == unp2,
1733 	    ("%s: unpcb %p is not connected to %p", __func__, unp, unp2));
1734 
1735 	unp->unp_conn = NULL;
1736 	so = unp->unp_socket;
1737 	so2 = unp2->unp_socket;
1738 	switch (unp->unp_socket->so_type) {
1739 	case SOCK_DGRAM:
1740 		UNP_REF_LIST_LOCK();
1741 #ifdef INVARIANTS
1742 		LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) {
1743 			if (unptmp == unp)
1744 				break;
1745 		}
1746 		KASSERT(unptmp != NULL,
1747 		    ("%s: %p not found in reflist of %p", __func__, unp, unp2));
1748 #endif
1749 		LIST_REMOVE(unp, unp_reflink);
1750 		UNP_REF_LIST_UNLOCK();
1751 		if (so) {
1752 			SOCK_LOCK(so);
1753 			so->so_state &= ~SS_ISCONNECTED;
1754 			SOCK_UNLOCK(so);
1755 		}
1756 		break;
1757 
1758 	case SOCK_STREAM:
1759 	case SOCK_SEQPACKET:
1760 		if (so)
1761 			soisdisconnected(so);
1762 		MPASS(unp2->unp_conn == unp);
1763 		unp2->unp_conn = NULL;
1764 		if (so2)
1765 			soisdisconnected(so2);
1766 		break;
1767 	}
1768 
1769 	if (unp == unp2) {
1770 		unp_pcb_rele_notlast(unp);
1771 		if (!unp_pcb_rele(unp))
1772 			UNP_PCB_UNLOCK(unp);
1773 	} else {
1774 		if (!unp_pcb_rele(unp))
1775 			UNP_PCB_UNLOCK(unp);
1776 		if (!unp_pcb_rele(unp2))
1777 			UNP_PCB_UNLOCK(unp2);
1778 	}
1779 }
1780 
1781 /*
1782  * unp_pcblist() walks the global list of struct unpcb's to generate a
1783  * pointer list, bumping the refcount on each unpcb.  It then copies them out
1784  * sequentially, validating the generation number on each to see if it has
1785  * been detached.  All of this is necessary because copyout() may sleep on
1786  * disk I/O.
1787  */
1788 static int
1789 unp_pcblist(SYSCTL_HANDLER_ARGS)
1790 {
1791 	struct unpcb *unp, **unp_list;
1792 	unp_gen_t gencnt;
1793 	struct xunpgen *xug;
1794 	struct unp_head *head;
1795 	struct xunpcb *xu;
1796 	u_int i;
1797 	int error, n;
1798 
1799 	switch ((intptr_t)arg1) {
1800 	case SOCK_STREAM:
1801 		head = &unp_shead;
1802 		break;
1803 
1804 	case SOCK_DGRAM:
1805 		head = &unp_dhead;
1806 		break;
1807 
1808 	case SOCK_SEQPACKET:
1809 		head = &unp_sphead;
1810 		break;
1811 
1812 	default:
1813 		panic("unp_pcblist: arg1 %d", (int)(intptr_t)arg1);
1814 	}
1815 
1816 	/*
1817 	 * The process of preparing the PCB list is too time-consuming and
1818 	 * resource-intensive to repeat twice on every request.
1819 	 */
1820 	if (req->oldptr == NULL) {
1821 		n = unp_count;
1822 		req->oldidx = 2 * (sizeof *xug)
1823 			+ (n + n/8) * sizeof(struct xunpcb);
1824 		return (0);
1825 	}
1826 
1827 	if (req->newptr != NULL)
1828 		return (EPERM);
1829 
1830 	/*
1831 	 * OK, now we're committed to doing something.
1832 	 */
1833 	xug = malloc(sizeof(*xug), M_TEMP, M_WAITOK | M_ZERO);
1834 	UNP_LINK_RLOCK();
1835 	gencnt = unp_gencnt;
1836 	n = unp_count;
1837 	UNP_LINK_RUNLOCK();
1838 
1839 	xug->xug_len = sizeof *xug;
1840 	xug->xug_count = n;
1841 	xug->xug_gen = gencnt;
1842 	xug->xug_sogen = so_gencnt;
1843 	error = SYSCTL_OUT(req, xug, sizeof *xug);
1844 	if (error) {
1845 		free(xug, M_TEMP);
1846 		return (error);
1847 	}
1848 
1849 	unp_list = malloc(n * sizeof *unp_list, M_TEMP, M_WAITOK);
1850 
1851 	UNP_LINK_RLOCK();
1852 	for (unp = LIST_FIRST(head), i = 0; unp && i < n;
1853 	     unp = LIST_NEXT(unp, unp_link)) {
1854 		UNP_PCB_LOCK(unp);
1855 		if (unp->unp_gencnt <= gencnt) {
1856 			if (cr_cansee(req->td->td_ucred,
1857 			    unp->unp_socket->so_cred)) {
1858 				UNP_PCB_UNLOCK(unp);
1859 				continue;
1860 			}
1861 			unp_list[i++] = unp;
1862 			unp_pcb_hold(unp);
1863 		}
1864 		UNP_PCB_UNLOCK(unp);
1865 	}
1866 	UNP_LINK_RUNLOCK();
1867 	n = i;			/* In case we lost some during malloc. */
1868 
1869 	error = 0;
1870 	xu = malloc(sizeof(*xu), M_TEMP, M_WAITOK | M_ZERO);
1871 	for (i = 0; i < n; i++) {
1872 		unp = unp_list[i];
1873 		UNP_PCB_LOCK(unp);
1874 		if (unp_pcb_rele(unp))
1875 			continue;
1876 
1877 		if (unp->unp_gencnt <= gencnt) {
1878 			xu->xu_len = sizeof *xu;
1879 			xu->xu_unpp = (uintptr_t)unp;
1880 			/*
1881 			 * XXX - need more locking here to protect against
1882 			 * connect/disconnect races for SMP.
1883 			 */
1884 			if (unp->unp_addr != NULL)
1885 				bcopy(unp->unp_addr, &xu->xu_addr,
1886 				      unp->unp_addr->sun_len);
1887 			else
1888 				bzero(&xu->xu_addr, sizeof(xu->xu_addr));
1889 			if (unp->unp_conn != NULL &&
1890 			    unp->unp_conn->unp_addr != NULL)
1891 				bcopy(unp->unp_conn->unp_addr,
1892 				      &xu->xu_caddr,
1893 				      unp->unp_conn->unp_addr->sun_len);
1894 			else
1895 				bzero(&xu->xu_caddr, sizeof(xu->xu_caddr));
1896 			xu->unp_vnode = (uintptr_t)unp->unp_vnode;
1897 			xu->unp_conn = (uintptr_t)unp->unp_conn;
1898 			xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs);
1899 			xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink);
1900 			xu->unp_gencnt = unp->unp_gencnt;
1901 			sotoxsocket(unp->unp_socket, &xu->xu_socket);
1902 			UNP_PCB_UNLOCK(unp);
1903 			error = SYSCTL_OUT(req, xu, sizeof *xu);
1904 		} else {
1905 			UNP_PCB_UNLOCK(unp);
1906 		}
1907 	}
1908 	free(xu, M_TEMP);
1909 	if (!error) {
1910 		/*
1911 		 * Give the user an updated idea of our state.  If the
1912 		 * generation differs from what we told her before, she knows
1913 		 * that something happened while we were processing this
1914 		 * request, and it might be necessary to retry.
1915 		 */
1916 		xug->xug_gen = unp_gencnt;
1917 		xug->xug_sogen = so_gencnt;
1918 		xug->xug_count = unp_count;
1919 		error = SYSCTL_OUT(req, xug, sizeof *xug);
1920 	}
1921 	free(unp_list, M_TEMP);
1922 	free(xug, M_TEMP);
1923 	return (error);
1924 }
1925 
1926 SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist,
1927     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE,
1928     (void *)(intptr_t)SOCK_DGRAM, 0, unp_pcblist, "S,xunpcb",
1929     "List of active local datagram sockets");
1930 SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist,
1931     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE,
1932     (void *)(intptr_t)SOCK_STREAM, 0, unp_pcblist, "S,xunpcb",
1933     "List of active local stream sockets");
1934 SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist,
1935     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE,
1936     (void *)(intptr_t)SOCK_SEQPACKET, 0, unp_pcblist, "S,xunpcb",
1937     "List of active local seqpacket sockets");
1938 
1939 static void
1940 unp_shutdown(struct unpcb *unp)
1941 {
1942 	struct unpcb *unp2;
1943 	struct socket *so;
1944 
1945 	UNP_PCB_LOCK_ASSERT(unp);
1946 
1947 	unp2 = unp->unp_conn;
1948 	if ((unp->unp_socket->so_type == SOCK_STREAM ||
1949 	    (unp->unp_socket->so_type == SOCK_SEQPACKET)) && unp2 != NULL) {
1950 		so = unp2->unp_socket;
1951 		if (so != NULL)
1952 			socantrcvmore(so);
1953 	}
1954 }
1955 
1956 static void
1957 unp_drop(struct unpcb *unp)
1958 {
1959 	struct socket *so = unp->unp_socket;
1960 	struct unpcb *unp2;
1961 
1962 	/*
1963 	 * Regardless of whether the socket's peer dropped the connection
1964 	 * with this socket by aborting or disconnecting, POSIX requires
1965 	 * that ECONNRESET is returned.
1966 	 */
1967 
1968 	UNP_PCB_LOCK(unp);
1969 	if (so)
1970 		so->so_error = ECONNRESET;
1971 	if ((unp2 = unp_pcb_lock_peer(unp)) != NULL) {
1972 		/* Last reference dropped in unp_disconnect(). */
1973 		unp_pcb_rele_notlast(unp);
1974 		unp_disconnect(unp, unp2);
1975 	} else if (!unp_pcb_rele(unp)) {
1976 		UNP_PCB_UNLOCK(unp);
1977 	}
1978 }
1979 
1980 static void
1981 unp_freerights(struct filedescent **fdep, int fdcount)
1982 {
1983 	struct file *fp;
1984 	int i;
1985 
1986 	KASSERT(fdcount > 0, ("%s: fdcount %d", __func__, fdcount));
1987 
1988 	for (i = 0; i < fdcount; i++) {
1989 		fp = fdep[i]->fde_file;
1990 		filecaps_free(&fdep[i]->fde_caps);
1991 		unp_discard(fp);
1992 	}
1993 	free(fdep[0], M_FILECAPS);
1994 }
1995 
1996 static int
1997 unp_externalize(struct mbuf *control, struct mbuf **controlp, int flags)
1998 {
1999 	struct thread *td = curthread;		/* XXX */
2000 	struct cmsghdr *cm = mtod(control, struct cmsghdr *);
2001 	int i;
2002 	int *fdp;
2003 	struct filedesc *fdesc = td->td_proc->p_fd;
2004 	struct filedescent **fdep;
2005 	void *data;
2006 	socklen_t clen = control->m_len, datalen;
2007 	int error, newfds;
2008 	u_int newlen;
2009 
2010 	UNP_LINK_UNLOCK_ASSERT();
2011 
2012 	error = 0;
2013 	if (controlp != NULL) /* controlp == NULL => free control messages */
2014 		*controlp = NULL;
2015 	while (cm != NULL) {
2016 		if (sizeof(*cm) > clen || cm->cmsg_len > clen) {
2017 			error = EINVAL;
2018 			break;
2019 		}
2020 		data = CMSG_DATA(cm);
2021 		datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data;
2022 		if (cm->cmsg_level == SOL_SOCKET
2023 		    && cm->cmsg_type == SCM_RIGHTS) {
2024 			newfds = datalen / sizeof(*fdep);
2025 			if (newfds == 0)
2026 				goto next;
2027 			fdep = data;
2028 
2029 			/* If we're not outputting the descriptors free them. */
2030 			if (error || controlp == NULL) {
2031 				unp_freerights(fdep, newfds);
2032 				goto next;
2033 			}
2034 			FILEDESC_XLOCK(fdesc);
2035 
2036 			/*
2037 			 * Now change each pointer to an fd in the global
2038 			 * table to an integer that is the index to the local
2039 			 * fd table entry that we set up to point to the
2040 			 * global one we are transferring.
2041 			 */
2042 			newlen = newfds * sizeof(int);
2043 			*controlp = sbcreatecontrol(NULL, newlen,
2044 			    SCM_RIGHTS, SOL_SOCKET);
2045 			if (*controlp == NULL) {
2046 				FILEDESC_XUNLOCK(fdesc);
2047 				error = E2BIG;
2048 				unp_freerights(fdep, newfds);
2049 				goto next;
2050 			}
2051 
2052 			fdp = (int *)
2053 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2054 			if (fdallocn(td, 0, fdp, newfds) != 0) {
2055 				FILEDESC_XUNLOCK(fdesc);
2056 				error = EMSGSIZE;
2057 				unp_freerights(fdep, newfds);
2058 				m_freem(*controlp);
2059 				*controlp = NULL;
2060 				goto next;
2061 			}
2062 			for (i = 0; i < newfds; i++, fdp++) {
2063 				_finstall(fdesc, fdep[i]->fde_file, *fdp,
2064 				    (flags & MSG_CMSG_CLOEXEC) != 0 ? UF_EXCLOSE : 0,
2065 				    &fdep[i]->fde_caps);
2066 				unp_externalize_fp(fdep[i]->fde_file);
2067 			}
2068 
2069 			/*
2070 			 * The new type indicates that the mbuf data refers to
2071 			 * kernel resources that may need to be released before
2072 			 * the mbuf is freed.
2073 			 */
2074 			m_chtype(*controlp, MT_EXTCONTROL);
2075 			FILEDESC_XUNLOCK(fdesc);
2076 			free(fdep[0], M_FILECAPS);
2077 		} else {
2078 			/* We can just copy anything else across. */
2079 			if (error || controlp == NULL)
2080 				goto next;
2081 			*controlp = sbcreatecontrol(NULL, datalen,
2082 			    cm->cmsg_type, cm->cmsg_level);
2083 			if (*controlp == NULL) {
2084 				error = ENOBUFS;
2085 				goto next;
2086 			}
2087 			bcopy(data,
2088 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *)),
2089 			    datalen);
2090 		}
2091 		controlp = &(*controlp)->m_next;
2092 
2093 next:
2094 		if (CMSG_SPACE(datalen) < clen) {
2095 			clen -= CMSG_SPACE(datalen);
2096 			cm = (struct cmsghdr *)
2097 			    ((caddr_t)cm + CMSG_SPACE(datalen));
2098 		} else {
2099 			clen = 0;
2100 			cm = NULL;
2101 		}
2102 	}
2103 
2104 	m_freem(control);
2105 	return (error);
2106 }
2107 
2108 static void
2109 unp_zone_change(void *tag)
2110 {
2111 
2112 	uma_zone_set_max(unp_zone, maxsockets);
2113 }
2114 
2115 #ifdef INVARIANTS
2116 static void
2117 unp_zdtor(void *mem, int size __unused, void *arg __unused)
2118 {
2119 	struct unpcb *unp;
2120 
2121 	unp = mem;
2122 
2123 	KASSERT(LIST_EMPTY(&unp->unp_refs),
2124 	    ("%s: unpcb %p has lingering refs", __func__, unp));
2125 	KASSERT(unp->unp_socket == NULL,
2126 	    ("%s: unpcb %p has socket backpointer", __func__, unp));
2127 	KASSERT(unp->unp_vnode == NULL,
2128 	    ("%s: unpcb %p has vnode references", __func__, unp));
2129 	KASSERT(unp->unp_conn == NULL,
2130 	    ("%s: unpcb %p is still connected", __func__, unp));
2131 	KASSERT(unp->unp_addr == NULL,
2132 	    ("%s: unpcb %p has leaked addr", __func__, unp));
2133 }
2134 #endif
2135 
2136 static void
2137 unp_init(void)
2138 {
2139 	uma_dtor dtor;
2140 
2141 #ifdef VIMAGE
2142 	if (!IS_DEFAULT_VNET(curvnet))
2143 		return;
2144 #endif
2145 
2146 #ifdef INVARIANTS
2147 	dtor = unp_zdtor;
2148 #else
2149 	dtor = NULL;
2150 #endif
2151 	unp_zone = uma_zcreate("unpcb", sizeof(struct unpcb), NULL, dtor,
2152 	    NULL, NULL, UMA_ALIGN_CACHE, 0);
2153 	uma_zone_set_max(unp_zone, maxsockets);
2154 	uma_zone_set_warning(unp_zone, "kern.ipc.maxsockets limit reached");
2155 	EVENTHANDLER_REGISTER(maxsockets_change, unp_zone_change,
2156 	    NULL, EVENTHANDLER_PRI_ANY);
2157 	LIST_INIT(&unp_dhead);
2158 	LIST_INIT(&unp_shead);
2159 	LIST_INIT(&unp_sphead);
2160 	SLIST_INIT(&unp_defers);
2161 	TIMEOUT_TASK_INIT(taskqueue_thread, &unp_gc_task, 0, unp_gc, NULL);
2162 	TASK_INIT(&unp_defer_task, 0, unp_process_defers, NULL);
2163 	UNP_LINK_LOCK_INIT();
2164 	UNP_DEFERRED_LOCK_INIT();
2165 }
2166 
2167 static void
2168 unp_internalize_cleanup_rights(struct mbuf *control)
2169 {
2170 	struct cmsghdr *cp;
2171 	struct mbuf *m;
2172 	void *data;
2173 	socklen_t datalen;
2174 
2175 	for (m = control; m != NULL; m = m->m_next) {
2176 		cp = mtod(m, struct cmsghdr *);
2177 		if (cp->cmsg_level != SOL_SOCKET ||
2178 		    cp->cmsg_type != SCM_RIGHTS)
2179 			continue;
2180 		data = CMSG_DATA(cp);
2181 		datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data;
2182 		unp_freerights(data, datalen / sizeof(struct filedesc *));
2183 	}
2184 }
2185 
2186 static int
2187 unp_internalize(struct mbuf **controlp, struct thread *td)
2188 {
2189 	struct mbuf *control, **initial_controlp;
2190 	struct proc *p;
2191 	struct filedesc *fdesc;
2192 	struct bintime *bt;
2193 	struct cmsghdr *cm;
2194 	struct cmsgcred *cmcred;
2195 	struct filedescent *fde, **fdep, *fdev;
2196 	struct file *fp;
2197 	struct timeval *tv;
2198 	struct timespec *ts;
2199 	void *data;
2200 	socklen_t clen, datalen;
2201 	int i, j, error, *fdp, oldfds;
2202 	u_int newlen;
2203 
2204 	UNP_LINK_UNLOCK_ASSERT();
2205 
2206 	p = td->td_proc;
2207 	fdesc = p->p_fd;
2208 	error = 0;
2209 	control = *controlp;
2210 	clen = control->m_len;
2211 	*controlp = NULL;
2212 	initial_controlp = controlp;
2213 	for (cm = mtod(control, struct cmsghdr *); cm != NULL;) {
2214 		if (sizeof(*cm) > clen || cm->cmsg_level != SOL_SOCKET
2215 		    || cm->cmsg_len > clen || cm->cmsg_len < sizeof(*cm)) {
2216 			error = EINVAL;
2217 			goto out;
2218 		}
2219 		data = CMSG_DATA(cm);
2220 		datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data;
2221 
2222 		switch (cm->cmsg_type) {
2223 		/*
2224 		 * Fill in credential information.
2225 		 */
2226 		case SCM_CREDS:
2227 			*controlp = sbcreatecontrol(NULL, sizeof(*cmcred),
2228 			    SCM_CREDS, SOL_SOCKET);
2229 			if (*controlp == NULL) {
2230 				error = ENOBUFS;
2231 				goto out;
2232 			}
2233 			cmcred = (struct cmsgcred *)
2234 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2235 			cmcred->cmcred_pid = p->p_pid;
2236 			cmcred->cmcred_uid = td->td_ucred->cr_ruid;
2237 			cmcred->cmcred_gid = td->td_ucred->cr_rgid;
2238 			cmcred->cmcred_euid = td->td_ucred->cr_uid;
2239 			cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups,
2240 			    CMGROUP_MAX);
2241 			for (i = 0; i < cmcred->cmcred_ngroups; i++)
2242 				cmcred->cmcred_groups[i] =
2243 				    td->td_ucred->cr_groups[i];
2244 			break;
2245 
2246 		case SCM_RIGHTS:
2247 			oldfds = datalen / sizeof (int);
2248 			if (oldfds == 0)
2249 				break;
2250 			/*
2251 			 * Check that all the FDs passed in refer to legal
2252 			 * files.  If not, reject the entire operation.
2253 			 */
2254 			fdp = data;
2255 			FILEDESC_SLOCK(fdesc);
2256 			for (i = 0; i < oldfds; i++, fdp++) {
2257 				fp = fget_locked(fdesc, *fdp);
2258 				if (fp == NULL) {
2259 					FILEDESC_SUNLOCK(fdesc);
2260 					error = EBADF;
2261 					goto out;
2262 				}
2263 				if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) {
2264 					FILEDESC_SUNLOCK(fdesc);
2265 					error = EOPNOTSUPP;
2266 					goto out;
2267 				}
2268 			}
2269 
2270 			/*
2271 			 * Now replace the integer FDs with pointers to the
2272 			 * file structure and capability rights.
2273 			 */
2274 			newlen = oldfds * sizeof(fdep[0]);
2275 			*controlp = sbcreatecontrol(NULL, newlen,
2276 			    SCM_RIGHTS, SOL_SOCKET);
2277 			if (*controlp == NULL) {
2278 				FILEDESC_SUNLOCK(fdesc);
2279 				error = E2BIG;
2280 				goto out;
2281 			}
2282 			fdp = data;
2283 			for (i = 0; i < oldfds; i++, fdp++) {
2284 				if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) {
2285 					fdp = data;
2286 					for (j = 0; j < i; j++, fdp++) {
2287 						fdrop(fdesc->fd_ofiles[*fdp].
2288 						    fde_file, td);
2289 					}
2290 					FILEDESC_SUNLOCK(fdesc);
2291 					error = EBADF;
2292 					goto out;
2293 				}
2294 			}
2295 			fdp = data;
2296 			fdep = (struct filedescent **)
2297 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2298 			fdev = malloc(sizeof(*fdev) * oldfds, M_FILECAPS,
2299 			    M_WAITOK);
2300 			for (i = 0; i < oldfds; i++, fdev++, fdp++) {
2301 				fde = &fdesc->fd_ofiles[*fdp];
2302 				fdep[i] = fdev;
2303 				fdep[i]->fde_file = fde->fde_file;
2304 				filecaps_copy(&fde->fde_caps,
2305 				    &fdep[i]->fde_caps, true);
2306 				unp_internalize_fp(fdep[i]->fde_file);
2307 			}
2308 			FILEDESC_SUNLOCK(fdesc);
2309 			break;
2310 
2311 		case SCM_TIMESTAMP:
2312 			*controlp = sbcreatecontrol(NULL, sizeof(*tv),
2313 			    SCM_TIMESTAMP, SOL_SOCKET);
2314 			if (*controlp == NULL) {
2315 				error = ENOBUFS;
2316 				goto out;
2317 			}
2318 			tv = (struct timeval *)
2319 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2320 			microtime(tv);
2321 			break;
2322 
2323 		case SCM_BINTIME:
2324 			*controlp = sbcreatecontrol(NULL, sizeof(*bt),
2325 			    SCM_BINTIME, SOL_SOCKET);
2326 			if (*controlp == NULL) {
2327 				error = ENOBUFS;
2328 				goto out;
2329 			}
2330 			bt = (struct bintime *)
2331 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2332 			bintime(bt);
2333 			break;
2334 
2335 		case SCM_REALTIME:
2336 			*controlp = sbcreatecontrol(NULL, sizeof(*ts),
2337 			    SCM_REALTIME, SOL_SOCKET);
2338 			if (*controlp == NULL) {
2339 				error = ENOBUFS;
2340 				goto out;
2341 			}
2342 			ts = (struct timespec *)
2343 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2344 			nanotime(ts);
2345 			break;
2346 
2347 		case SCM_MONOTONIC:
2348 			*controlp = sbcreatecontrol(NULL, sizeof(*ts),
2349 			    SCM_MONOTONIC, SOL_SOCKET);
2350 			if (*controlp == NULL) {
2351 				error = ENOBUFS;
2352 				goto out;
2353 			}
2354 			ts = (struct timespec *)
2355 			    CMSG_DATA(mtod(*controlp, struct cmsghdr *));
2356 			nanouptime(ts);
2357 			break;
2358 
2359 		default:
2360 			error = EINVAL;
2361 			goto out;
2362 		}
2363 
2364 		if (*controlp != NULL)
2365 			controlp = &(*controlp)->m_next;
2366 		if (CMSG_SPACE(datalen) < clen) {
2367 			clen -= CMSG_SPACE(datalen);
2368 			cm = (struct cmsghdr *)
2369 			    ((caddr_t)cm + CMSG_SPACE(datalen));
2370 		} else {
2371 			clen = 0;
2372 			cm = NULL;
2373 		}
2374 	}
2375 
2376 out:
2377 	if (error != 0 && initial_controlp != NULL)
2378 		unp_internalize_cleanup_rights(*initial_controlp);
2379 	m_freem(control);
2380 	return (error);
2381 }
2382 
2383 static struct mbuf *
2384 unp_addsockcred(struct thread *td, struct mbuf *control)
2385 {
2386 	struct mbuf *m, *n, *n_prev;
2387 	struct sockcred *sc;
2388 	const struct cmsghdr *cm;
2389 	int ngroups;
2390 	int i;
2391 
2392 	ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX);
2393 	m = sbcreatecontrol(NULL, SOCKCREDSIZE(ngroups), SCM_CREDS, SOL_SOCKET);
2394 	if (m == NULL)
2395 		return (control);
2396 
2397 	sc = (struct sockcred *) CMSG_DATA(mtod(m, struct cmsghdr *));
2398 	sc->sc_uid = td->td_ucred->cr_ruid;
2399 	sc->sc_euid = td->td_ucred->cr_uid;
2400 	sc->sc_gid = td->td_ucred->cr_rgid;
2401 	sc->sc_egid = td->td_ucred->cr_gid;
2402 	sc->sc_ngroups = ngroups;
2403 	for (i = 0; i < sc->sc_ngroups; i++)
2404 		sc->sc_groups[i] = td->td_ucred->cr_groups[i];
2405 
2406 	/*
2407 	 * Unlink SCM_CREDS control messages (struct cmsgcred), since just
2408 	 * created SCM_CREDS control message (struct sockcred) has another
2409 	 * format.
2410 	 */
2411 	if (control != NULL)
2412 		for (n = control, n_prev = NULL; n != NULL;) {
2413 			cm = mtod(n, struct cmsghdr *);
2414     			if (cm->cmsg_level == SOL_SOCKET &&
2415 			    cm->cmsg_type == SCM_CREDS) {
2416     				if (n_prev == NULL)
2417 					control = n->m_next;
2418 				else
2419 					n_prev->m_next = n->m_next;
2420 				n = m_free(n);
2421 			} else {
2422 				n_prev = n;
2423 				n = n->m_next;
2424 			}
2425 		}
2426 
2427 	/* Prepend it to the head. */
2428 	m->m_next = control;
2429 	return (m);
2430 }
2431 
2432 static struct unpcb *
2433 fptounp(struct file *fp)
2434 {
2435 	struct socket *so;
2436 
2437 	if (fp->f_type != DTYPE_SOCKET)
2438 		return (NULL);
2439 	if ((so = fp->f_data) == NULL)
2440 		return (NULL);
2441 	if (so->so_proto->pr_domain != &localdomain)
2442 		return (NULL);
2443 	return sotounpcb(so);
2444 }
2445 
2446 static void
2447 unp_discard(struct file *fp)
2448 {
2449 	struct unp_defer *dr;
2450 
2451 	if (unp_externalize_fp(fp)) {
2452 		dr = malloc(sizeof(*dr), M_TEMP, M_WAITOK);
2453 		dr->ud_fp = fp;
2454 		UNP_DEFERRED_LOCK();
2455 		SLIST_INSERT_HEAD(&unp_defers, dr, ud_link);
2456 		UNP_DEFERRED_UNLOCK();
2457 		atomic_add_int(&unp_defers_count, 1);
2458 		taskqueue_enqueue(taskqueue_thread, &unp_defer_task);
2459 	} else
2460 		(void) closef(fp, (struct thread *)NULL);
2461 }
2462 
2463 static void
2464 unp_process_defers(void *arg __unused, int pending)
2465 {
2466 	struct unp_defer *dr;
2467 	SLIST_HEAD(, unp_defer) drl;
2468 	int count;
2469 
2470 	SLIST_INIT(&drl);
2471 	for (;;) {
2472 		UNP_DEFERRED_LOCK();
2473 		if (SLIST_FIRST(&unp_defers) == NULL) {
2474 			UNP_DEFERRED_UNLOCK();
2475 			break;
2476 		}
2477 		SLIST_SWAP(&unp_defers, &drl, unp_defer);
2478 		UNP_DEFERRED_UNLOCK();
2479 		count = 0;
2480 		while ((dr = SLIST_FIRST(&drl)) != NULL) {
2481 			SLIST_REMOVE_HEAD(&drl, ud_link);
2482 			closef(dr->ud_fp, NULL);
2483 			free(dr, M_TEMP);
2484 			count++;
2485 		}
2486 		atomic_add_int(&unp_defers_count, -count);
2487 	}
2488 }
2489 
2490 static void
2491 unp_internalize_fp(struct file *fp)
2492 {
2493 	struct unpcb *unp;
2494 
2495 	UNP_LINK_WLOCK();
2496 	if ((unp = fptounp(fp)) != NULL) {
2497 		unp->unp_file = fp;
2498 		unp->unp_msgcount++;
2499 	}
2500 	unp_rights++;
2501 	UNP_LINK_WUNLOCK();
2502 }
2503 
2504 static int
2505 unp_externalize_fp(struct file *fp)
2506 {
2507 	struct unpcb *unp;
2508 	int ret;
2509 
2510 	UNP_LINK_WLOCK();
2511 	if ((unp = fptounp(fp)) != NULL) {
2512 		unp->unp_msgcount--;
2513 		ret = 1;
2514 	} else
2515 		ret = 0;
2516 	unp_rights--;
2517 	UNP_LINK_WUNLOCK();
2518 	return (ret);
2519 }
2520 
2521 /*
2522  * unp_defer indicates whether additional work has been defered for a future
2523  * pass through unp_gc().  It is thread local and does not require explicit
2524  * synchronization.
2525  */
2526 static int	unp_marked;
2527 
2528 static void
2529 unp_remove_dead_ref(struct filedescent **fdep, int fdcount)
2530 {
2531 	struct unpcb *unp;
2532 	struct file *fp;
2533 	int i;
2534 
2535 	/*
2536 	 * This function can only be called from the gc task.
2537 	 */
2538 	KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0,
2539 	    ("%s: not on gc callout", __func__));
2540 	UNP_LINK_LOCK_ASSERT();
2541 
2542 	for (i = 0; i < fdcount; i++) {
2543 		fp = fdep[i]->fde_file;
2544 		if ((unp = fptounp(fp)) == NULL)
2545 			continue;
2546 		if ((unp->unp_gcflag & UNPGC_DEAD) == 0)
2547 			continue;
2548 		unp->unp_gcrefs--;
2549 	}
2550 }
2551 
2552 static void
2553 unp_restore_undead_ref(struct filedescent **fdep, int fdcount)
2554 {
2555 	struct unpcb *unp;
2556 	struct file *fp;
2557 	int i;
2558 
2559 	/*
2560 	 * This function can only be called from the gc task.
2561 	 */
2562 	KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0,
2563 	    ("%s: not on gc callout", __func__));
2564 	UNP_LINK_LOCK_ASSERT();
2565 
2566 	for (i = 0; i < fdcount; i++) {
2567 		fp = fdep[i]->fde_file;
2568 		if ((unp = fptounp(fp)) == NULL)
2569 			continue;
2570 		if ((unp->unp_gcflag & UNPGC_DEAD) == 0)
2571 			continue;
2572 		unp->unp_gcrefs++;
2573 		unp_marked++;
2574 	}
2575 }
2576 
2577 static void
2578 unp_gc_scan(struct unpcb *unp, void (*op)(struct filedescent **, int))
2579 {
2580 	struct socket *so, *soa;
2581 
2582 	so = unp->unp_socket;
2583 	SOCK_LOCK(so);
2584 	if (SOLISTENING(so)) {
2585 		/*
2586 		 * Mark all sockets in our accept queue.
2587 		 */
2588 		TAILQ_FOREACH(soa, &so->sol_comp, so_list) {
2589 			if (sotounpcb(soa)->unp_gcflag & UNPGC_IGNORE_RIGHTS)
2590 				continue;
2591 			SOCKBUF_LOCK(&soa->so_rcv);
2592 			unp_scan(soa->so_rcv.sb_mb, op);
2593 			SOCKBUF_UNLOCK(&soa->so_rcv);
2594 		}
2595 	} else {
2596 		/*
2597 		 * Mark all sockets we reference with RIGHTS.
2598 		 */
2599 		if ((unp->unp_gcflag & UNPGC_IGNORE_RIGHTS) == 0) {
2600 			SOCKBUF_LOCK(&so->so_rcv);
2601 			unp_scan(so->so_rcv.sb_mb, op);
2602 			SOCKBUF_UNLOCK(&so->so_rcv);
2603 		}
2604 	}
2605 	SOCK_UNLOCK(so);
2606 }
2607 
2608 static int unp_recycled;
2609 SYSCTL_INT(_net_local, OID_AUTO, recycled, CTLFLAG_RD, &unp_recycled, 0,
2610     "Number of unreachable sockets claimed by the garbage collector.");
2611 
2612 static int unp_taskcount;
2613 SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0,
2614     "Number of times the garbage collector has run.");
2615 
2616 SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0,
2617     "Number of active local sockets.");
2618 
2619 static void
2620 unp_gc(__unused void *arg, int pending)
2621 {
2622 	struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead,
2623 				    NULL };
2624 	struct unp_head **head;
2625 	struct unp_head unp_deadhead;	/* List of potentially-dead sockets. */
2626 	struct file *f, **unref;
2627 	struct unpcb *unp, *unptmp;
2628 	int i, total, unp_unreachable;
2629 
2630 	LIST_INIT(&unp_deadhead);
2631 	unp_taskcount++;
2632 	UNP_LINK_RLOCK();
2633 	/*
2634 	 * First determine which sockets may be in cycles.
2635 	 */
2636 	unp_unreachable = 0;
2637 
2638 	for (head = heads; *head != NULL; head++)
2639 		LIST_FOREACH(unp, *head, unp_link) {
2640 			KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0,
2641 			    ("%s: unp %p has unexpected gc flags 0x%x",
2642 			    __func__, unp, (unsigned int)unp->unp_gcflag));
2643 
2644 			f = unp->unp_file;
2645 
2646 			/*
2647 			 * Check for an unreachable socket potentially in a
2648 			 * cycle.  It must be in a queue as indicated by
2649 			 * msgcount, and this must equal the file reference
2650 			 * count.  Note that when msgcount is 0 the file is
2651 			 * NULL.
2652 			 */
2653 			if (f != NULL && unp->unp_msgcount != 0 &&
2654 			    refcount_load(&f->f_count) == unp->unp_msgcount) {
2655 				LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead);
2656 				unp->unp_gcflag |= UNPGC_DEAD;
2657 				unp->unp_gcrefs = unp->unp_msgcount;
2658 				unp_unreachable++;
2659 			}
2660 		}
2661 
2662 	/*
2663 	 * Scan all sockets previously marked as potentially being in a cycle
2664 	 * and remove the references each socket holds on any UNPGC_DEAD
2665 	 * sockets in its queue.  After this step, all remaining references on
2666 	 * sockets marked UNPGC_DEAD should not be part of any cycle.
2667 	 */
2668 	LIST_FOREACH(unp, &unp_deadhead, unp_dead)
2669 		unp_gc_scan(unp, unp_remove_dead_ref);
2670 
2671 	/*
2672 	 * If a socket still has a non-negative refcount, it cannot be in a
2673 	 * cycle.  In this case increment refcount of all children iteratively.
2674 	 * Stop the scan once we do a complete loop without discovering
2675 	 * a new reachable socket.
2676 	 */
2677 	do {
2678 		unp_marked = 0;
2679 		LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp)
2680 			if (unp->unp_gcrefs > 0) {
2681 				unp->unp_gcflag &= ~UNPGC_DEAD;
2682 				LIST_REMOVE(unp, unp_dead);
2683 				KASSERT(unp_unreachable > 0,
2684 				    ("%s: unp_unreachable underflow.",
2685 				    __func__));
2686 				unp_unreachable--;
2687 				unp_gc_scan(unp, unp_restore_undead_ref);
2688 			}
2689 	} while (unp_marked);
2690 
2691 	UNP_LINK_RUNLOCK();
2692 
2693 	if (unp_unreachable == 0)
2694 		return;
2695 
2696 	/*
2697 	 * Allocate space for a local array of dead unpcbs.
2698 	 * TODO: can this path be simplified by instead using the local
2699 	 * dead list at unp_deadhead, after taking out references
2700 	 * on the file object and/or unpcb and dropping the link lock?
2701 	 */
2702 	unref = malloc(unp_unreachable * sizeof(struct file *),
2703 	    M_TEMP, M_WAITOK);
2704 
2705 	/*
2706 	 * Iterate looking for sockets which have been specifically marked
2707 	 * as unreachable and store them locally.
2708 	 */
2709 	UNP_LINK_RLOCK();
2710 	total = 0;
2711 	LIST_FOREACH(unp, &unp_deadhead, unp_dead) {
2712 		KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0,
2713 		    ("%s: unp %p not marked UNPGC_DEAD", __func__, unp));
2714 		unp->unp_gcflag &= ~UNPGC_DEAD;
2715 		f = unp->unp_file;
2716 		if (unp->unp_msgcount == 0 || f == NULL ||
2717 		    refcount_load(&f->f_count) != unp->unp_msgcount ||
2718 		    !fhold(f))
2719 			continue;
2720 		unref[total++] = f;
2721 		KASSERT(total <= unp_unreachable,
2722 		    ("%s: incorrect unreachable count.", __func__));
2723 	}
2724 	UNP_LINK_RUNLOCK();
2725 
2726 	/*
2727 	 * Now flush all sockets, free'ing rights.  This will free the
2728 	 * struct files associated with these sockets but leave each socket
2729 	 * with one remaining ref.
2730 	 */
2731 	for (i = 0; i < total; i++) {
2732 		struct socket *so;
2733 
2734 		so = unref[i]->f_data;
2735 		CURVNET_SET(so->so_vnet);
2736 		sorflush(so);
2737 		CURVNET_RESTORE();
2738 	}
2739 
2740 	/*
2741 	 * And finally release the sockets so they can be reclaimed.
2742 	 */
2743 	for (i = 0; i < total; i++)
2744 		fdrop(unref[i], NULL);
2745 	unp_recycled += total;
2746 	free(unref, M_TEMP);
2747 }
2748 
2749 static void
2750 unp_dispose_mbuf(struct mbuf *m)
2751 {
2752 
2753 	if (m)
2754 		unp_scan(m, unp_freerights);
2755 }
2756 
2757 /*
2758  * Synchronize against unp_gc, which can trip over data as we are freeing it.
2759  */
2760 static void
2761 unp_dispose(struct socket *so)
2762 {
2763 	struct unpcb *unp;
2764 
2765 	unp = sotounpcb(so);
2766 	UNP_LINK_WLOCK();
2767 	unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS;
2768 	UNP_LINK_WUNLOCK();
2769 	if (!SOLISTENING(so))
2770 		unp_dispose_mbuf(so->so_rcv.sb_mb);
2771 }
2772 
2773 static void
2774 unp_scan(struct mbuf *m0, void (*op)(struct filedescent **, int))
2775 {
2776 	struct mbuf *m;
2777 	struct cmsghdr *cm;
2778 	void *data;
2779 	socklen_t clen, datalen;
2780 
2781 	while (m0 != NULL) {
2782 		for (m = m0; m; m = m->m_next) {
2783 			if (m->m_type != MT_CONTROL)
2784 				continue;
2785 
2786 			cm = mtod(m, struct cmsghdr *);
2787 			clen = m->m_len;
2788 
2789 			while (cm != NULL) {
2790 				if (sizeof(*cm) > clen || cm->cmsg_len > clen)
2791 					break;
2792 
2793 				data = CMSG_DATA(cm);
2794 				datalen = (caddr_t)cm + cm->cmsg_len
2795 				    - (caddr_t)data;
2796 
2797 				if (cm->cmsg_level == SOL_SOCKET &&
2798 				    cm->cmsg_type == SCM_RIGHTS) {
2799 					(*op)(data, datalen /
2800 					    sizeof(struct filedescent *));
2801 				}
2802 
2803 				if (CMSG_SPACE(datalen) < clen) {
2804 					clen -= CMSG_SPACE(datalen);
2805 					cm = (struct cmsghdr *)
2806 					    ((caddr_t)cm + CMSG_SPACE(datalen));
2807 				} else {
2808 					clen = 0;
2809 					cm = NULL;
2810 				}
2811 			}
2812 		}
2813 		m0 = m0->m_nextpkt;
2814 	}
2815 }
2816 
2817 /*
2818  * A helper function called by VFS before socket-type vnode reclamation.
2819  * For an active vnode it clears unp_vnode pointer and decrements unp_vnode
2820  * use count.
2821  */
2822 void
2823 vfs_unp_reclaim(struct vnode *vp)
2824 {
2825 	struct unpcb *unp;
2826 	int active;
2827 	struct mtx *vplock;
2828 
2829 	ASSERT_VOP_ELOCKED(vp, "vfs_unp_reclaim");
2830 	KASSERT(vp->v_type == VSOCK,
2831 	    ("vfs_unp_reclaim: vp->v_type != VSOCK"));
2832 
2833 	active = 0;
2834 	vplock = mtx_pool_find(mtxpool_sleep, vp);
2835 	mtx_lock(vplock);
2836 	VOP_UNP_CONNECT(vp, &unp);
2837 	if (unp == NULL)
2838 		goto done;
2839 	UNP_PCB_LOCK(unp);
2840 	if (unp->unp_vnode == vp) {
2841 		VOP_UNP_DETACH(vp);
2842 		unp->unp_vnode = NULL;
2843 		active = 1;
2844 	}
2845 	UNP_PCB_UNLOCK(unp);
2846  done:
2847 	mtx_unlock(vplock);
2848 	if (active)
2849 		vunref(vp);
2850 }
2851 
2852 #ifdef DDB
2853 static void
2854 db_print_indent(int indent)
2855 {
2856 	int i;
2857 
2858 	for (i = 0; i < indent; i++)
2859 		db_printf(" ");
2860 }
2861 
2862 static void
2863 db_print_unpflags(int unp_flags)
2864 {
2865 	int comma;
2866 
2867 	comma = 0;
2868 	if (unp_flags & UNP_HAVEPC) {
2869 		db_printf("%sUNP_HAVEPC", comma ? ", " : "");
2870 		comma = 1;
2871 	}
2872 	if (unp_flags & UNP_WANTCRED_ALWAYS) {
2873 		db_printf("%sUNP_WANTCRED_ALWAYS", comma ? ", " : "");
2874 		comma = 1;
2875 	}
2876 	if (unp_flags & UNP_WANTCRED_ONESHOT) {
2877 		db_printf("%sUNP_WANTCRED_ONESHOT", comma ? ", " : "");
2878 		comma = 1;
2879 	}
2880 	if (unp_flags & UNP_CONNWAIT) {
2881 		db_printf("%sUNP_CONNWAIT", comma ? ", " : "");
2882 		comma = 1;
2883 	}
2884 	if (unp_flags & UNP_CONNECTING) {
2885 		db_printf("%sUNP_CONNECTING", comma ? ", " : "");
2886 		comma = 1;
2887 	}
2888 	if (unp_flags & UNP_BINDING) {
2889 		db_printf("%sUNP_BINDING", comma ? ", " : "");
2890 		comma = 1;
2891 	}
2892 }
2893 
2894 static void
2895 db_print_xucred(int indent, struct xucred *xu)
2896 {
2897 	int comma, i;
2898 
2899 	db_print_indent(indent);
2900 	db_printf("cr_version: %u   cr_uid: %u   cr_pid: %d   cr_ngroups: %d\n",
2901 	    xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups);
2902 	db_print_indent(indent);
2903 	db_printf("cr_groups: ");
2904 	comma = 0;
2905 	for (i = 0; i < xu->cr_ngroups; i++) {
2906 		db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]);
2907 		comma = 1;
2908 	}
2909 	db_printf("\n");
2910 }
2911 
2912 static void
2913 db_print_unprefs(int indent, struct unp_head *uh)
2914 {
2915 	struct unpcb *unp;
2916 	int counter;
2917 
2918 	counter = 0;
2919 	LIST_FOREACH(unp, uh, unp_reflink) {
2920 		if (counter % 4 == 0)
2921 			db_print_indent(indent);
2922 		db_printf("%p  ", unp);
2923 		if (counter % 4 == 3)
2924 			db_printf("\n");
2925 		counter++;
2926 	}
2927 	if (counter != 0 && counter % 4 != 0)
2928 		db_printf("\n");
2929 }
2930 
2931 DB_SHOW_COMMAND(unpcb, db_show_unpcb)
2932 {
2933 	struct unpcb *unp;
2934 
2935         if (!have_addr) {
2936                 db_printf("usage: show unpcb <addr>\n");
2937                 return;
2938         }
2939         unp = (struct unpcb *)addr;
2940 
2941 	db_printf("unp_socket: %p   unp_vnode: %p\n", unp->unp_socket,
2942 	    unp->unp_vnode);
2943 
2944 	db_printf("unp_ino: %ju   unp_conn: %p\n", (uintmax_t)unp->unp_ino,
2945 	    unp->unp_conn);
2946 
2947 	db_printf("unp_refs:\n");
2948 	db_print_unprefs(2, &unp->unp_refs);
2949 
2950 	/* XXXRW: Would be nice to print the full address, if any. */
2951 	db_printf("unp_addr: %p\n", unp->unp_addr);
2952 
2953 	db_printf("unp_gencnt: %llu\n",
2954 	    (unsigned long long)unp->unp_gencnt);
2955 
2956 	db_printf("unp_flags: %x (", unp->unp_flags);
2957 	db_print_unpflags(unp->unp_flags);
2958 	db_printf(")\n");
2959 
2960 	db_printf("unp_peercred:\n");
2961 	db_print_xucred(2, &unp->unp_peercred);
2962 
2963 	db_printf("unp_refcount: %u\n", unp->unp_refcount);
2964 }
2965 #endif
2966