Lines Matching +full:multi +full:- +full:subsystems
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
6 * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved.
8 * Copyright (c) 2022-2025 Gleb Smirnoff <glebius@FreeBSD.org>
56 * need a proper out-of-band
155 * large enough for at least one max-size datagram plus address.
198 * - a global linkage lock
199 * - a global connection list lock
200 * - the mtxpool lock
201 * - per-unpcb mutexes
228 * has a back-pointer to its socket, unp_socket, which will be stable under
235 * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee
239 * protocols, bind() is a non-atomic operation, and connect() requires
245 * Another tricky issue is simultaneous multi-threaded or multi-process
278 #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \
281 #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx)
282 #define UNP_PCB_LOCKPTR(unp) (&(unp)->unp_mtx)
283 #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx)
284 #define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx)
285 #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx)
286 #define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx)
287 #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED)
288 #define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED)
321 old = refcount_acquire(&unp->unp_refcount); in unp_pcb_hold()
332 if ((ret = refcount_release(&unp->unp_refcount))) { in unp_pcb_rele()
345 ret = refcount_release(&unp->unp_refcount); in unp_pcb_rele_notlast()
386 unp2 = unp->unp_conn; in unp_pcb_lock_peer()
400 unp->unp_pairbusy++; in unp_pcb_lock_peer()
406 KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL, in unp_pcb_lock_peer()
408 if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) { in unp_pcb_lock_peer()
409 unp->unp_flags &= ~UNP_WAITING; in unp_pcb_lock_peer()
416 if (unp->unp_conn == NULL) { in unp_pcb_lock_peer()
435 if (__predict_false(so->so_error != 0)) { in uipc_lock_peer()
436 error = so->so_error; in uipc_lock_peer()
437 so->so_error = 0; in uipc_lock_peer()
449 error = so->so_state & SS_ISDISCONNECTED ? EPIPE : ENOTCONN; in uipc_lock_peer()
468 unp2 = unp->unp_conn; in uipc_abort()
485 KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); in uipc_attach()
486 switch (so->so_type) { in uipc_attach()
488 STAILQ_INIT(&so->so_rcv.uxdg_mb); in uipc_attach()
489 STAILQ_INIT(&so->so_snd.uxdg_mb); in uipc_attach()
490 TAILQ_INIT(&so->so_rcv.uxdg_conns); in uipc_attach()
493 * of one-to-many receive buffer, we assign both space in uipc_attach()
514 mtx_destroy(&so->so_rcv_mtx); in uipc_attach()
515 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF | MTX_DUPOK); in uipc_attach()
516 knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, in uipc_attach()
518 STAILQ_INIT(&so->so_rcv.uxst_mbq); in uipc_attach()
529 LIST_INIT(&unp->unp_refs); in uipc_attach()
531 unp->unp_socket = so; in uipc_attach()
532 so->so_pcb = unp; in uipc_attach()
533 refcount_init(&unp->unp_refcount, 1); in uipc_attach()
534 unp->unp_mode = ACCESSPERMS; in uipc_attach()
539 unp->unp_gencnt = ++unp_gencnt; in uipc_attach()
540 unp->unp_ino = ++unp_ino; in uipc_attach()
542 switch (so->so_type) { in uipc_attach()
579 if (nam->sa_family != AF_UNIX) in uipc_bindat()
585 if (soun->sun_len > sizeof(struct sockaddr_un)) in uipc_bindat()
587 namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); in uipc_bindat()
593 * socket, so flag in-progress operations, and return an error if an in uipc_bindat()
597 * also returns an error. Not allowing re-binding simplifies the in uipc_bindat()
601 if (unp->unp_vnode != NULL) { in uipc_bindat()
605 if (unp->unp_flags & UNP_BINDING) { in uipc_bindat()
609 unp->unp_flags |= UNP_BINDING; in uipc_bindat()
610 mode = unp->unp_mode & ~td->td_proc->p_pd->pd_cmask; in uipc_bindat()
614 bcopy(soun->sun_path, buf, namelen); in uipc_bindat()
645 error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd, in uipc_bindat()
675 unp->unp_vnode = vp; in uipc_bindat()
676 unp->unp_addr = soun; in uipc_bindat()
677 unp->unp_flags &= ~UNP_BINDING; in uipc_bindat()
687 unp->unp_flags &= ~UNP_BINDING; in uipc_bindat()
732 if ((vp = unp->unp_vnode) != NULL) { in uipc_close()
737 if (vp && unp->unp_vnode == NULL) { in uipc_close()
743 unp->unp_vnode = NULL; in uipc_close()
768 if (unp->unp_vnode != NULL || (unp->unp_flags & UNP_BINDING) != 0) in uipc_chmod()
771 unp->unp_mode = mode; in uipc_chmod()
781 if (so1->so_type != so2->so_type) in uipc_connect2()
784 unp = so1->so_pcb; in uipc_connect2()
786 unp2 = so2->so_pcb; in uipc_connect2()
814 if (unp->unp_gcflag & UNPGC_DEAD) in uipc_detach()
816 unp->unp_gencnt = ++unp_gencnt; in uipc_detach()
817 --unp_count; in uipc_detach()
822 if ((vp = unp->unp_vnode) != NULL) { in uipc_detach()
827 if (unp->unp_vnode != vp && unp->unp_vnode != NULL) { in uipc_detach()
833 if ((vp = unp->unp_vnode) != NULL) { in uipc_detach()
835 unp->unp_vnode = NULL; in uipc_detach()
843 while (!LIST_EMPTY(&unp->unp_refs)) { in uipc_detach()
844 struct unpcb *ref = LIST_FIRST(&unp->unp_refs); in uipc_detach()
858 unp->unp_socket->so_pcb = NULL; in uipc_detach()
859 unp->unp_socket = NULL; in uipc_detach()
860 free(unp->unp_addr, M_SONAME); in uipc_detach()
861 unp->unp_addr = NULL; in uipc_detach()
869 taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); in uipc_detach()
871 switch (so->so_type) { in uipc_detach()
874 MPASS(SOLISTENING(so) || (STAILQ_EMPTY(&so->so_rcv.uxst_mbq) && in uipc_detach()
875 so->so_rcv.uxst_peer == NULL)); in uipc_detach()
882 MPASS(so->so_rcv.uxdg_peeked == NULL); in uipc_detach()
883 MPASS(STAILQ_EMPTY(&so->so_rcv.uxdg_mb)); in uipc_detach()
884 MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); in uipc_detach()
885 MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); in uipc_detach()
911 MPASS(so->so_type != SOCK_DGRAM); in uipc_listen()
919 if (unp->unp_conn != NULL || (unp->unp_flags & UNP_CONNECTING) != 0) in uipc_listen()
921 else if (unp->unp_vnode == NULL) in uipc_listen()
931 cru2xt(td, &unp->unp_peercred); in uipc_listen()
933 (void)chgsbsize(so->so_cred->cr_uidinfo, in uipc_listen()
934 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); in uipc_listen()
935 (void)chgsbsize(so->so_cred->cr_uidinfo, in uipc_listen()
936 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); in uipc_listen()
957 if (unp2->unp_addr != NULL) in uipc_peeraddr()
958 sa = (struct sockaddr *)unp2->unp_addr; in uipc_peeraddr()
961 bcopy(sa, ret, sa->sa_len); in uipc_peeraddr()
966 bcopy(sa, ret, sa->sa_len); in uipc_peeraddr()
973 * netgraph(4) and other subsystems can call into socket code. The
988 m->m_pkthdr.rcvif = NULL; in uipc_reset_kernel_mbuf()
989 m->m_pkthdr.flowid = 0; in uipc_reset_kernel_mbuf()
990 m->m_pkthdr.csum_flags = 0; in uipc_reset_kernel_mbuf()
991 m->m_pkthdr.fibnum = 0; in uipc_reset_kernel_mbuf()
992 m->m_pkthdr.rsstype = 0; in uipc_reset_kernel_mbuf()
995 MPASS(m->m_pkthdr.len == mc->mc_len); in uipc_reset_kernel_mbuf()
1007 STAILQ_FOREACH(d, &sb->uxst_mbq, m_stailq) { in uipc_stream_sbcheck()
1008 if (d == sb->uxst_fnrdy) { in uipc_stream_sbcheck()
1009 MPASS(d->m_flags & M_NOTREADY); in uipc_stream_sbcheck()
1012 if (d->m_type == MT_CONTROL) in uipc_stream_sbcheck()
1013 dctl += d->m_len; in uipc_stream_sbcheck()
1014 else if (d->m_type == MT_DATA) { in uipc_stream_sbcheck()
1015 dccc += d->m_len; in uipc_stream_sbcheck()
1017 dacc += d->m_len; in uipc_stream_sbcheck()
1021 if (d->m_flags & M_EXT) in uipc_stream_sbcheck()
1022 dmbcnt += d->m_ext.ext_size; in uipc_stream_sbcheck()
1023 if (d->m_stailq.stqe_next == NULL) in uipc_stream_sbcheck()
1024 MPASS(sb->uxst_mbq.stqh_last == &d->m_stailq.stqe_next); in uipc_stream_sbcheck()
1026 MPASS(sb->uxst_fnrdy == NULL || notready); in uipc_stream_sbcheck()
1027 MPASS(dacc == sb->sb_acc); in uipc_stream_sbcheck()
1028 MPASS(dccc == sb->sb_ccc); in uipc_stream_sbcheck()
1029 MPASS(dctl == sb->sb_ctl); in uipc_stream_sbcheck()
1030 MPASS(dmbcnt == sb->sb_mbcnt); in uipc_stream_sbcheck()
1031 (void)STAILQ_EMPTY(&sb->uxst_mbq); in uipc_stream_sbcheck()
1060 if (__predict_true(sb->sb_hiwat >= sb->sb_ccc + sb->sb_ctl)) in uipc_stream_sbspace()
1061 space = sb->sb_hiwat - sb->sb_ccc - sb->sb_ctl; in uipc_stream_sbspace()
1064 if (__predict_true(sb->sb_mbmax >= sb->sb_mbcnt)) in uipc_stream_sbspace()
1065 mbspace = sb->sb_mbmax - sb->sb_mbcnt; in uipc_stream_sbspace()
1079 struct sockbuf *sb = &so->so_rcv; in uipc_stream_sbwait()
1082 sb->sb_flags |= SB_WAIT; in uipc_stream_sbwait()
1083 return (msleep_sbt(&sb->sb_acc, SOCK_RECVBUF_MTX(so), PSOCK | PCATCH, in uipc_stream_sbwait()
1107 nonblock = (so->so_state & SS_NBIO) || in uipc_sosend_stream_or_seqpacket()
1136 if (__predict_false(so->so_snd.sb_flags & SB_AIO_RUNNING)) { in uipc_sosend_stream_or_seqpacket()
1141 resid = uio->uio_resid; in uipc_sosend_stream_or_seqpacket()
1145 * buffer - do the copyin before taking any locks, sized to our in uipc_sosend_stream_or_seqpacket()
1149 error = mc_uiotomc(&mc, uio, so->so_snd.sb_hiwat, 0, M_WAITOK, in uipc_sosend_stream_or_seqpacket()
1163 if (unp2->unp_flags & UNP_WANTCRED_MASK) { in uipc_sosend_stream_or_seqpacket()
1169 unp_addsockcred(td, &cmc, unp2->unp_flags); in uipc_sosend_stream_or_seqpacket()
1170 unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT; in uipc_sosend_stream_or_seqpacket()
1180 so2 = unp2->unp_socket; in uipc_sosend_stream_or_seqpacket()
1183 sb = &so2->so_rcv; in uipc_sosend_stream_or_seqpacket()
1191 if (__predict_false(cmc.mc_len > sb->sb_hiwat)) { in uipc_sosend_stream_or_seqpacket()
1196 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { in uipc_sosend_stream_or_seqpacket()
1207 if (space < sb->sb_lowat || space < cmc.mc_len) { in uipc_sosend_stream_or_seqpacket()
1210 sb->uxst_flags |= UXST_PEER_AIO; in uipc_sosend_stream_or_seqpacket()
1214 so->so_snd.sb_ccc = in uipc_sosend_stream_or_seqpacket()
1215 so->so_snd.sb_hiwat - space; in uipc_sosend_stream_or_seqpacket()
1222 so->so_snd.sb_timeo)) != 0) { in uipc_sosend_stream_or_seqpacket()
1229 space -= cmc.mc_len; in uipc_sosend_stream_or_seqpacket()
1255 STAILQ_CONCAT(&sb->uxst_mbq, &cmc.mc_q); in uipc_sosend_stream_or_seqpacket()
1256 sb->sb_ctl += cmc.mc_len; in uipc_sosend_stream_or_seqpacket()
1257 sb->sb_mbcnt += cmc.mc_mlen; in uipc_sosend_stream_or_seqpacket()
1261 if (sb->uxst_fnrdy == NULL) in uipc_sosend_stream_or_seqpacket()
1262 sb->sb_acc += mc.mc_len; in uipc_sosend_stream_or_seqpacket()
1263 sb->sb_ccc += mc.mc_len; in uipc_sosend_stream_or_seqpacket()
1264 sb->sb_mbcnt += mc.mc_mlen; in uipc_sosend_stream_or_seqpacket()
1265 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); in uipc_sosend_stream_or_seqpacket()
1276 } else if (uio != NULL && uio->uio_resid > 0) { in uipc_sosend_stream_or_seqpacket()
1289 atomic_load_int(&so->so_snd.sb_hiwat), 0, M_WAITOK, in uipc_sosend_stream_or_seqpacket()
1299 td->td_ru.ru_msgsnd++; in uipc_sosend_stream_or_seqpacket()
1309 uio->uio_resid = resid - sent; in uipc_sosend_stream_or_seqpacket()
1332 struct sockbuf *sb = &so->so_rcv; in uipc_wakeup_writer()
1336 MPASS(sb->uxst_peer != NULL); in uipc_wakeup_writer()
1338 sel = &sb->uxst_peer->so_wrsel; in uipc_wakeup_writer()
1340 if (sb->uxst_flags & UXST_PEER_SEL) { in uipc_wakeup_writer()
1346 sb->uxst_flags &= ~UXST_PEER_SEL; in uipc_wakeup_writer()
1348 if (sb->sb_flags & SB_WAIT) { in uipc_wakeup_writer()
1349 sb->sb_flags &= ~SB_WAIT; in uipc_wakeup_writer()
1350 wakeup(&sb->sb_acc); in uipc_wakeup_writer()
1352 KNOTE_LOCKED(&sel->si_note, 0); in uipc_wakeup_writer()
1361 so->so_rcv.sb_state |= SBS_CANTRCVMORE; in uipc_cantrcvmore()
1362 selwakeuppri(&so->so_rdsel, PSOCK); in uipc_cantrcvmore()
1363 KNOTE_LOCKED(&so->so_rdsel.si_note, 0); in uipc_cantrcvmore()
1364 if (so->so_rcv.uxst_peer != NULL) in uipc_cantrcvmore()
1374 struct sockbuf *sb = &so->so_rcv; in uipc_soreceive_stream_or_seqpacket()
1388 nonblock = (so->so_state & SS_NBIO) || in uipc_soreceive_stream_or_seqpacket()
1401 if (__predict_false((atomic_load_short(&so->so_state) & in uipc_soreceive_stream_or_seqpacket()
1412 while (sb->sb_acc < sb->sb_lowat && in uipc_soreceive_stream_or_seqpacket()
1413 (sb->sb_ctl == 0 || controlp == NULL)) { in uipc_soreceive_stream_or_seqpacket()
1414 if (so->so_error) { in uipc_soreceive_stream_or_seqpacket()
1415 error = so->so_error; in uipc_soreceive_stream_or_seqpacket()
1417 so->so_error = 0; in uipc_soreceive_stream_or_seqpacket()
1422 if (sb->sb_state & SBS_CANTRCVMORE) { in uipc_soreceive_stream_or_seqpacket()
1440 MPASS(STAILQ_FIRST(&sb->uxst_mbq)); in uipc_soreceive_stream_or_seqpacket()
1441 MPASS(sb->sb_acc > 0 || sb->sb_ctl > 0); in uipc_soreceive_stream_or_seqpacket()
1445 first = STAILQ_FIRST(&sb->uxst_mbq); in uipc_soreceive_stream_or_seqpacket()
1446 if (first->m_type == MT_CONTROL) { in uipc_soreceive_stream_or_seqpacket()
1448 STAILQ_FOREACH_FROM(first, &sb->uxst_mbq, m_stailq) { in uipc_soreceive_stream_or_seqpacket()
1449 if (first->m_type != MT_CONTROL) in uipc_soreceive_stream_or_seqpacket()
1451 ctl += first->m_len; in uipc_soreceive_stream_or_seqpacket()
1453 if (first->m_flags & M_EXT) in uipc_soreceive_stream_or_seqpacket()
1454 mbcnt += first->m_ext.ext_size; in uipc_soreceive_stream_or_seqpacket()
1461 * last == NULL - socket to be flushed in uipc_soreceive_stream_or_seqpacket()
1463 * lastlen > last->m_len - uio to be filled, last to be adjusted in uipc_soreceive_stream_or_seqpacket()
1464 * lastlen == 0 - MT_CONTROL, M_EOR or M_NOTREADY encountered in uipc_soreceive_stream_or_seqpacket()
1466 space = uio->uio_resid; in uipc_soreceive_stream_or_seqpacket()
1468 for (m = first, last = sb->uxst_fnrdy, lastlen = 0; in uipc_soreceive_stream_or_seqpacket()
1469 m != sb->uxst_fnrdy; in uipc_soreceive_stream_or_seqpacket()
1471 if (m->m_type != MT_DATA) { in uipc_soreceive_stream_or_seqpacket()
1476 if (space >= m->m_len) { in uipc_soreceive_stream_or_seqpacket()
1477 space -= m->m_len; in uipc_soreceive_stream_or_seqpacket()
1478 datalen += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1480 if (m->m_flags & M_EXT) in uipc_soreceive_stream_or_seqpacket()
1481 mbcnt += m->m_ext.ext_size; in uipc_soreceive_stream_or_seqpacket()
1482 if (m->m_flags & M_EOR) { in uipc_soreceive_stream_or_seqpacket()
1499 STAILQ_INIT(&sb->uxst_mbq); in uipc_soreceive_stream_or_seqpacket()
1501 STAILQ_FIRST(&sb->uxst_mbq) = last; in uipc_soreceive_stream_or_seqpacket()
1502 MPASS(last->m_len > lastlen); in uipc_soreceive_stream_or_seqpacket()
1503 last->m_len -= lastlen; in uipc_soreceive_stream_or_seqpacket()
1504 last->m_data += lastlen; in uipc_soreceive_stream_or_seqpacket()
1506 MPASS(sb->sb_acc >= datalen); in uipc_soreceive_stream_or_seqpacket()
1507 sb->sb_acc -= datalen; in uipc_soreceive_stream_or_seqpacket()
1508 sb->sb_ccc -= datalen; in uipc_soreceive_stream_or_seqpacket()
1509 MPASS(sb->sb_ctl >= ctl); in uipc_soreceive_stream_or_seqpacket()
1510 sb->sb_ctl -= ctl; in uipc_soreceive_stream_or_seqpacket()
1511 MPASS(sb->sb_mbcnt >= mbcnt); in uipc_soreceive_stream_or_seqpacket()
1512 sb->sb_mbcnt -= mbcnt; in uipc_soreceive_stream_or_seqpacket()
1514 if (__predict_true(sb->uxst_peer != NULL)) { in uipc_soreceive_stream_or_seqpacket()
1518 if ((aio = sb->uxst_flags & UXST_PEER_AIO)) in uipc_soreceive_stream_or_seqpacket()
1519 sb->uxst_flags &= ~UXST_PEER_AIO; in uipc_soreceive_stream_or_seqpacket()
1529 struct socket *so2 = unp2->unp_socket; in uipc_soreceive_stream_or_seqpacket()
1532 so2->so_snd.sb_ccc -= datalen; in uipc_soreceive_stream_or_seqpacket()
1542 while (control != NULL && control->m_type == MT_CONTROL) { in uipc_soreceive_stream_or_seqpacket()
1563 (sb->sb_state & SBS_CANTRCVMORE) || in uipc_soreceive_stream_or_seqpacket()
1564 cmc.mc_len + sb->sb_ccc + sb->sb_ctl > in uipc_soreceive_stream_or_seqpacket()
1565 sb->sb_hiwat)) { in uipc_soreceive_stream_or_seqpacket()
1586 STAILQ_CONCAT(&cmc.mc_q, &sb->uxst_mbq); in uipc_soreceive_stream_or_seqpacket()
1587 STAILQ_SWAP(&cmc.mc_q, &sb->uxst_mbq, mbuf); in uipc_soreceive_stream_or_seqpacket()
1589 sb->sb_ctl = sb->sb_acc = sb->sb_ccc = in uipc_soreceive_stream_or_seqpacket()
1590 sb->sb_mbcnt = 0; in uipc_soreceive_stream_or_seqpacket()
1591 STAILQ_FOREACH(m, &sb->uxst_mbq, m_stailq) { in uipc_soreceive_stream_or_seqpacket()
1592 if (m->m_type == MT_DATA) { in uipc_soreceive_stream_or_seqpacket()
1593 sb->sb_acc += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1594 sb->sb_ccc += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1596 sb->sb_ctl += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1598 sb->sb_mbcnt += MSIZE; in uipc_soreceive_stream_or_seqpacket()
1599 if (m->m_flags & M_EXT) in uipc_soreceive_stream_or_seqpacket()
1600 sb->sb_mbcnt += in uipc_soreceive_stream_or_seqpacket()
1601 m->m_ext.ext_size; in uipc_soreceive_stream_or_seqpacket()
1610 controlp = &(*controlp)->m_next; in uipc_soreceive_stream_or_seqpacket()
1618 * userland. They are useless to a law-abiding in uipc_soreceive_stream_or_seqpacket()
1624 *controlp = m_copym(control, 0, control->m_len, in uipc_soreceive_stream_or_seqpacket()
1626 controlp = &(*controlp)->m_next; in uipc_soreceive_stream_or_seqpacket()
1634 error = uiomove(mtod(m, char *), m->m_len, uio); in uipc_soreceive_stream_or_seqpacket()
1649 MPASS(!(m->m_flags & M_PKTHDR)); in uipc_soreceive_stream_or_seqpacket()
1650 MPASS(last->m_data - M_START(last) >= lastlen); in uipc_soreceive_stream_or_seqpacket()
1651 error = uiomove(mtod(last, char *) - lastlen, in uipc_soreceive_stream_or_seqpacket()
1660 if (waitall && !(flags & MSG_EOR) && uio->uio_resid > 0) in uipc_soreceive_stream_or_seqpacket()
1667 uio->uio_td->td_ru.ru_msgrcv++; in uipc_soreceive_stream_or_seqpacket()
1687 else if (!TAILQ_EMPTY(&so->sol_comp)) in uipc_sopoll_stream_or_seqpacket()
1689 else if (so->so_error) in uipc_sopoll_stream_or_seqpacket()
1692 selrecord(td, &so->so_rdsel); in uipc_sopoll_stream_or_seqpacket()
1697 if (so->so_state & SS_ISDISCONNECTED) in uipc_sopoll_stream_or_seqpacket()
1703 if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat || in uipc_sopoll_stream_or_seqpacket()
1704 so->so_error || so->so_rerror) in uipc_sopoll_stream_or_seqpacket()
1706 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) in uipc_sopoll_stream_or_seqpacket()
1710 selrecord(td, &so->so_rdsel); in uipc_sopoll_stream_or_seqpacket()
1711 so->so_rcv.sb_flags |= SB_SEL; in uipc_sopoll_stream_or_seqpacket()
1716 struct socket *so2 = so->so_rcv.uxst_peer; in uipc_sopoll_stream_or_seqpacket()
1719 struct sockbuf *sb = &so2->so_rcv; in uipc_sopoll_stream_or_seqpacket()
1722 if (uipc_stream_sbspace(sb) >= sb->sb_lowat) in uipc_sopoll_stream_or_seqpacket()
1725 if (sb->sb_state & SBS_CANTRCVMORE) in uipc_sopoll_stream_or_seqpacket()
1728 so2->so_rcv.uxst_flags |= UXST_PEER_SEL; in uipc_sopoll_stream_or_seqpacket()
1729 selrecord(td, &so->so_wrsel); in uipc_sopoll_stream_or_seqpacket()
1733 selrecord(td, &so->so_wrsel); in uipc_sopoll_stream_or_seqpacket()
1755 if (so->so_rcv.uxst_peer != NULL) in uipc_wrknl_lock()
1756 SOCK_RECVBUF_LOCK(so->so_rcv.uxst_peer); in uipc_wrknl_lock()
1769 if (so->so_rcv.uxst_peer != NULL) in uipc_wrknl_unlock()
1770 SOCK_RECVBUF_UNLOCK(so->so_rcv.uxst_peer); in uipc_wrknl_unlock()
1792 if (what == LA_LOCKED && so->so_rcv.uxst_peer != NULL) in uipc_wrknl_assert_lock()
1793 SOCK_RECVBUF_LOCK_ASSERT(so->so_rcv.uxst_peer); in uipc_wrknl_assert_lock()
1800 struct socket *so = kn->kn_fp->f_data; in uipc_filt_sowdetach()
1803 knlist_remove(&so->so_wrsel.si_note, kn, 1); in uipc_filt_sowdetach()
1810 struct socket *so = kn->kn_fp->f_data, *so2; in uipc_filt_sowrite()
1811 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; in uipc_filt_sowrite()
1817 if (so->so_state & SS_ISDISCONNECTED) { in uipc_filt_sowrite()
1818 kn->kn_flags |= EV_EOF; in uipc_filt_sowrite()
1819 kn->kn_fflags = so->so_error; in uipc_filt_sowrite()
1825 so2 = unp2->unp_socket; in uipc_filt_sowrite()
1827 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); in uipc_filt_sowrite()
1829 if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { in uipc_filt_sowrite()
1830 kn->kn_flags |= EV_EOF; in uipc_filt_sowrite()
1832 } else if (kn->kn_sfflags & NOTE_LOWAT) in uipc_filt_sowrite()
1833 return (kn->kn_data >= kn->kn_sdata); in uipc_filt_sowrite()
1835 return (kn->kn_data >= so2->so_rcv.sb_lowat); in uipc_filt_sowrite()
1841 struct socket *so = kn->kn_fp->f_data, *so2; in uipc_filt_soempty()
1842 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; in uipc_filt_soempty()
1847 so2 = unp2->unp_socket; in uipc_filt_soempty()
1849 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); in uipc_filt_soempty()
1851 return (kn->kn_data == 0 ? 1 : 0); in uipc_filt_soempty()
1873 switch (kn->kn_filter) { in uipc_kqfilter_stream_or_seqpacket()
1877 kn->kn_fop = &uipc_write_filtops; in uipc_kqfilter_stream_or_seqpacket()
1880 kn->kn_fop = &uipc_empty_filtops; in uipc_kqfilter_stream_or_seqpacket()
1886 knl = &so->so_wrsel.si_note; in uipc_kqfilter_stream_or_seqpacket()
1893 struct socket *so2 = so->so_rcv.uxst_peer; in uipc_kqfilter_stream_or_seqpacket()
1915 if (__predict_false(sb->sb_hiwat < sb->uxdg_cc || in uipc_dgram_sbspace()
1916 sb->sb_mbmax < sb->uxdg_mbcnt)) in uipc_dgram_sbspace()
1919 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) in uipc_dgram_sbspace()
1922 bleft = sb->sb_hiwat - sb->uxdg_cc; in uipc_dgram_sbspace()
1923 mleft = sb->sb_mbmax - sb->uxdg_mbcnt; in uipc_dgram_sbspace()
1932 * from -> control -> data and append it to the socket buffer.
1964 if (__predict_false(uio->uio_resid > unpdg_maxdgram)) { in uipc_sosend_dgram()
1974 cc = m->m_pkthdr.len; in uipc_sosend_dgram()
1975 mbcnt = MSIZE + m->m_pkthdr.memlen; in uipc_sosend_dgram()
1984 if (__predict_false(m->m_pkthdr.len > unpdg_maxdgram)) { in uipc_sosend_dgram()
2008 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { in uipc_sosend_dgram()
2013 if (so->so_error != 0) { in uipc_sosend_dgram()
2014 error = so->so_error; in uipc_sosend_dgram()
2015 so->so_error = 0; in uipc_sosend_dgram()
2019 if (((so->so_state & SS_ISCONNECTED) == 0) && addr == NULL) { in uipc_sosend_dgram()
2030 unp2 = unp->unp_conn; in uipc_sosend_dgram()
2042 if (unp2->unp_flags & UNP_WANTCRED_MASK) in uipc_sosend_dgram()
2043 unp_addsockcred(td, &cmc, unp2->unp_flags); in uipc_sosend_dgram()
2044 if (unp->unp_addr != NULL) in uipc_sosend_dgram()
2045 from = (struct sockaddr *)unp->unp_addr; in uipc_sosend_dgram()
2048 f->m_len = from->sa_len; in uipc_sosend_dgram()
2049 MPASS(from->sa_len <= MLEN); in uipc_sosend_dgram()
2050 bcopy(from, mtod(f, void *), from->sa_len); in uipc_sosend_dgram()
2053 * Concatenate mbufs: from -> control -> data. in uipc_sosend_dgram()
2057 f->m_next = mc_first(&cmc); in uipc_sosend_dgram()
2058 mc_last(&cmc)->m_next = m; in uipc_sosend_dgram()
2062 f->m_next = m; in uipc_sosend_dgram()
2064 ctl = f->m_len + cmc.mc_len; in uipc_sosend_dgram()
2068 for (struct mbuf *mb = f; mb != NULL; mb = mb->m_next) { in uipc_sosend_dgram()
2069 if (mb->m_type == MT_DATA) in uipc_sosend_dgram()
2070 dcc += mb->m_len; in uipc_sosend_dgram()
2072 dctl += mb->m_len; in uipc_sosend_dgram()
2074 if (mb->m_flags & M_EXT) in uipc_sosend_dgram()
2075 dmbcnt += mb->m_ext.ext_size; in uipc_sosend_dgram()
2081 f->m_pkthdr.len = cc + ctl; in uipc_sosend_dgram()
2082 f->m_pkthdr.memlen = mbcnt; in uipc_sosend_dgram()
2083 f->m_pkthdr.ctllen = ctl; in uipc_sosend_dgram()
2088 * Unconnected sends, when !(so->so_state & SS_ISCONNECTED) and the in uipc_sosend_dgram()
2110 so2 = unp2->unp_socket; in uipc_sosend_dgram()
2111 sb = (addr == NULL) ? &so->so_snd : &so2->so_rcv; in uipc_sosend_dgram()
2114 if (addr == NULL && STAILQ_EMPTY(&sb->uxdg_mb)) in uipc_sosend_dgram()
2115 TAILQ_INSERT_HEAD(&so2->so_rcv.uxdg_conns, &so->so_snd, in uipc_sosend_dgram()
2117 STAILQ_INSERT_TAIL(&sb->uxdg_mb, f, m_stailqpkt); in uipc_sosend_dgram()
2118 sb->uxdg_cc += cc + ctl; in uipc_sosend_dgram()
2119 sb->uxdg_ctl += ctl; in uipc_sosend_dgram()
2120 sb->uxdg_mbcnt += mbcnt; in uipc_sosend_dgram()
2121 so2->so_rcv.sb_acc += cc + ctl; in uipc_sosend_dgram()
2122 so2->so_rcv.sb_ccc += cc + ctl; in uipc_sosend_dgram()
2123 so2->so_rcv.sb_ctl += ctl; in uipc_sosend_dgram()
2124 so2->so_rcv.sb_mbcnt += mbcnt; in uipc_sosend_dgram()
2130 if (f->m_next->m_type == MT_CONTROL) { in uipc_sosend_dgram()
2131 STAILQ_FIRST(&cmc.mc_q) = f->m_next; in uipc_sosend_dgram()
2132 f->m_next = NULL; in uipc_sosend_dgram()
2141 td->td_ru.ru_msgsnd++; in uipc_sosend_dgram()
2170 so->so_rcv.uxdg_peeked = m; in uipc_peek_dgram()
2171 so->so_rcv.uxdg_cc += m->m_pkthdr.len; in uipc_peek_dgram()
2172 so->so_rcv.uxdg_ctl += m->m_pkthdr.ctllen; in uipc_peek_dgram()
2173 so->so_rcv.uxdg_mbcnt += m->m_pkthdr.memlen; in uipc_peek_dgram()
2176 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); in uipc_peek_dgram()
2180 m = m->m_next; in uipc_peek_dgram()
2186 while (m != NULL && m->m_type == MT_CONTROL) { in uipc_peek_dgram()
2188 *controlp = m_copym(m, 0, m->m_len, M_WAITOK); in uipc_peek_dgram()
2189 controlp = &(*controlp)->m_next; in uipc_peek_dgram()
2191 m = m->m_next; in uipc_peek_dgram()
2193 KASSERT(m == NULL || m->m_type == MT_DATA, in uipc_peek_dgram()
2195 while (m != NULL && uio->uio_resid > 0) { in uipc_peek_dgram()
2196 len = uio->uio_resid; in uipc_peek_dgram()
2197 if (len > m->m_len) in uipc_peek_dgram()
2198 len = m->m_len; in uipc_peek_dgram()
2204 if (len == m->m_len) in uipc_peek_dgram()
2205 m = m->m_next; in uipc_peek_dgram()
2213 uio->uio_resid -= m_length(m, NULL) - len; in uipc_peek_dgram()
2244 nonblock = (so->so_state & SS_NBIO) || in uipc_soreceive_dgram()
2258 while ((m = so->so_rcv.uxdg_peeked) == NULL && in uipc_soreceive_dgram()
2259 (sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) == NULL && in uipc_soreceive_dgram()
2260 (m = STAILQ_FIRST(&so->so_rcv.uxdg_mb)) == NULL) { in uipc_soreceive_dgram()
2261 if (so->so_error) { in uipc_soreceive_dgram()
2262 error = so->so_error; in uipc_soreceive_dgram()
2264 so->so_error = 0; in uipc_soreceive_dgram()
2269 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || in uipc_soreceive_dgram()
2270 uio->uio_resid == 0) { in uipc_soreceive_dgram()
2289 sb = &so->so_rcv; in uipc_soreceive_dgram()
2291 m = STAILQ_FIRST(&sb->uxdg_mb); in uipc_soreceive_dgram()
2293 MPASS(m == so->so_rcv.uxdg_peeked); in uipc_soreceive_dgram()
2295 MPASS(sb->uxdg_cc > 0); in uipc_soreceive_dgram()
2297 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); in uipc_soreceive_dgram()
2299 if (uio->uio_td) in uipc_soreceive_dgram()
2300 uio->uio_td->td_ru.ru_msgrcv++; in uipc_soreceive_dgram()
2302 if (__predict_true(m != so->so_rcv.uxdg_peeked)) { in uipc_soreceive_dgram()
2303 STAILQ_REMOVE_HEAD(&sb->uxdg_mb, m_stailqpkt); in uipc_soreceive_dgram()
2304 if (STAILQ_EMPTY(&sb->uxdg_mb) && sb != &so->so_rcv) in uipc_soreceive_dgram()
2305 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); in uipc_soreceive_dgram()
2307 so->so_rcv.uxdg_peeked = NULL; in uipc_soreceive_dgram()
2309 sb->uxdg_cc -= m->m_pkthdr.len; in uipc_soreceive_dgram()
2310 sb->uxdg_ctl -= m->m_pkthdr.ctllen; in uipc_soreceive_dgram()
2311 sb->uxdg_mbcnt -= m->m_pkthdr.memlen; in uipc_soreceive_dgram()
2316 so->so_rcv.sb_acc -= m->m_pkthdr.len; in uipc_soreceive_dgram()
2317 so->so_rcv.sb_ccc -= m->m_pkthdr.len; in uipc_soreceive_dgram()
2318 so->so_rcv.sb_ctl -= m->m_pkthdr.ctllen; in uipc_soreceive_dgram()
2319 so->so_rcv.sb_mbcnt -= m->m_pkthdr.memlen; in uipc_soreceive_dgram()
2337 while (m != NULL && m->m_type == MT_CONTROL) { in uipc_soreceive_dgram()
2348 controlp = &(*controlp)->m_next; in uipc_soreceive_dgram()
2351 KASSERT(m == NULL || m->m_type == MT_DATA, in uipc_soreceive_dgram()
2353 while (m != NULL && uio->uio_resid > 0) { in uipc_soreceive_dgram()
2354 len = uio->uio_resid; in uipc_soreceive_dgram()
2355 if (len > m->m_len) in uipc_soreceive_dgram()
2356 len = m->m_len; in uipc_soreceive_dgram()
2363 if (len == m->m_len) in uipc_soreceive_dgram()
2366 m->m_data += len; in uipc_soreceive_dgram()
2367 m->m_len -= len; in uipc_soreceive_dgram()
2376 uio->uio_resid -= m_length(m, NULL); in uipc_soreceive_dgram()
2396 MPASS(so->so_type == SOCK_STREAM); in uipc_sendfile_wait()
2400 nonblock = so->so_state & SS_NBIO; in uipc_sendfile_wait()
2403 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) in uipc_sendfile_wait()
2409 so2 = unp2->unp_socket; in uipc_sendfile_wait()
2410 sb = &so2->so_rcv; in uipc_sendfile_wait()
2414 (*space < so->so_snd.sb_hiwat / 2)) { in uipc_sendfile_wait()
2422 error = uipc_stream_sbwait(so2, so->so_snd.sb_timeo); in uipc_sendfile_wait()
2424 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) in uipc_sendfile_wait()
2456 MPASS(so->so_type == SOCK_STREAM); in uipc_sendfile()
2458 KASSERT(!(m->m_flags & M_EXTPG), in uipc_sendfile()
2463 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) { in uipc_sendfile()
2473 so2 = unp2->unp_socket; in uipc_sendfile()
2474 sb = &so2->so_rcv; in uipc_sendfile()
2478 sb->sb_ccc += mc.mc_len; in uipc_sendfile()
2479 sb->sb_mbcnt += mc.mc_mlen; in uipc_sendfile()
2480 if (sb->uxst_fnrdy == NULL) { in uipc_sendfile()
2484 if (m->m_flags & M_NOTREADY) { in uipc_sendfile()
2485 sb->uxst_fnrdy = m; in uipc_sendfile()
2488 sb->sb_acc += m->m_len; in uipc_sendfile()
2494 sb->sb_acc += mc.mc_len; in uipc_sendfile()
2499 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); in uipc_sendfile()
2525 blocker = (sb->uxst_fnrdy == m); in uipc_sbready()
2526 STAILQ_FOREACH_FROM(m, &sb->uxst_mbq, m_stailq) { in uipc_sbready()
2528 MPASS(m->m_flags & M_NOTREADY); in uipc_sbready()
2529 m->m_flags &= ~M_NOTREADY; in uipc_sbready()
2531 sb->sb_acc += m->m_len; in uipc_sbready()
2532 count--; in uipc_sbready()
2533 } else if (m->m_flags & M_NOTREADY) in uipc_sbready()
2536 sb->sb_acc += m->m_len; in uipc_sbready()
2539 sb->uxst_fnrdy = m; in uipc_sbready()
2557 sb = &so->so_rcv; in uipc_ready_scan()
2559 if (sb->uxst_fnrdy != NULL) { in uipc_ready_scan()
2560 STAILQ_FOREACH(mb, &sb->uxst_mbq, m_stailq) { in uipc_ready_scan()
2578 MPASS(so->so_type == SOCK_STREAM); in uipc_ready()
2584 so2 = unp2->unp_socket; in uipc_ready()
2585 sb = &so2->so_rcv; in uipc_ready()
2598 * be valid. In this case, the not-ready mbufs are still in uipc_ready()
2604 if (uipc_ready_scan(unp->unp_socket, m, count, &error)) in uipc_ready()
2626 sb->st_blksize = so->so_snd.sb_hiwat; in uipc_sense()
2627 sb->st_dev = NODEV; in uipc_sense()
2628 sb->st_ino = unp->unp_ino; in uipc_sense()
2641 so->so_error = ECONNABORTED; in uipc_shutdown()
2646 } else if ((so->so_state & in uipc_shutdown()
2654 * both backward-compatibility and POSIX requirements by forcing in uipc_shutdown()
2662 if (so->so_type != SOCK_DGRAM) { in uipc_shutdown()
2672 if (so->so_type == SOCK_DGRAM) in uipc_shutdown()
2679 if (so->so_type == SOCK_DGRAM) in uipc_shutdown()
2686 if (so->so_type == SOCK_DGRAM) { in uipc_shutdown()
2690 if (unp->unp_conn != NULL) in uipc_shutdown()
2691 uipc_cantrcvmore(unp->unp_conn->unp_socket); in uipc_shutdown()
2695 wakeup(&so->so_timeo); in uipc_shutdown()
2710 if (unp->unp_addr != NULL) in uipc_sockaddr()
2711 sa = (struct sockaddr *) unp->unp_addr; in uipc_sockaddr()
2714 bcopy(sa, ret, sa->sa_len); in uipc_sockaddr()
2726 if (sopt->sopt_level != SOL_LOCAL) in uipc_ctloutput()
2732 switch (sopt->sopt_dir) { in uipc_ctloutput()
2734 switch (sopt->sopt_name) { in uipc_ctloutput()
2737 if (unp->unp_flags & UNP_HAVEPC) in uipc_ctloutput()
2738 xu = unp->unp_peercred; in uipc_ctloutput()
2740 if (so->so_proto->pr_flags & PR_CONNREQUIRED) in uipc_ctloutput()
2752 optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0; in uipc_ctloutput()
2758 optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0; in uipc_ctloutput()
2769 switch (sopt->sopt_name) { in uipc_ctloutput()
2780 if ((unp->unp_flags & (exclusive)) != 0) { \ in uipc_ctloutput()
2785 unp->unp_flags |= (bit); \ in uipc_ctloutput()
2787 unp->unp_flags &= ~(bit); \ in uipc_ctloutput()
2791 switch (sopt->sopt_name) { in uipc_ctloutput()
2843 if (nam->sa_family != AF_UNIX) in unp_connectat()
2845 if (nam->sa_len > sizeof(struct sockaddr_un)) in unp_connectat()
2847 len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); in unp_connectat()
2851 bcopy(soun->sun_path, buf, len); in unp_connectat()
2872 else if (unp->unp_conn != NULL) in unp_connectat()
2874 else if ((unp->unp_flags & UNP_CONNECTING) != 0) { in unp_connectat()
2881 if (unp->unp_pairbusy > 0) { in unp_connectat()
2882 unp->unp_flags |= UNP_WAITING; in unp_connectat()
2888 unp->unp_flags |= UNP_CONNECTING; in unp_connectat()
2891 connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0; in unp_connectat()
2908 if (vp->v_type != VSOCK) { in unp_connectat()
2913 error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD); in unp_connectat()
2917 error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td); in unp_connectat()
2931 so2 = unp2->unp_socket; in unp_connectat()
2932 if (so->so_type != so2->so_type) { in unp_connectat()
2951 if (unp2->unp_addr != NULL) { in unp_connectat()
2952 bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); in unp_connectat()
2953 unp3->unp_addr = (struct sockaddr_un *) sa; in unp_connectat()
2974 KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && in unp_connectat()
2980 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, in unp_connectat()
2982 unp->unp_flags &= ~UNP_CONNECTING; in unp_connectat()
3001 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, in unp_connectat()
3003 unp->unp_flags &= ~UNP_CONNECTING; in unp_connectat()
3020 cru2xt(td, &client_unp->unp_peercred); in unp_copy_peercred()
3021 client_unp->unp_flags |= UNP_HAVEPC; in unp_copy_peercred()
3023 memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred, in unp_copy_peercred()
3024 sizeof(server_unp->unp_peercred)); in unp_copy_peercred()
3025 server_unp->unp_flags |= UNP_HAVEPC; in unp_copy_peercred()
3026 client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK); in unp_copy_peercred()
3044 * 1) socketpair(2) - in this case socket has not been yet reported to userland
3056 struct socket *so2 = sotounpcb(so)->unp_conn->unp_socket; in unp_soisconnected()
3065 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); in unp_soisconnected()
3066 MPASS((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | in unp_soisconnected()
3068 MPASS(so->so_qstate == SQ_NONE); in unp_soisconnected()
3070 so->so_state &= ~SS_ISDISCONNECTED; in unp_soisconnected()
3071 so->so_state |= SS_ISCONNECTED; in unp_soisconnected()
3073 sb = &so2->so_rcv; in unp_soisconnected()
3074 sb->uxst_peer = so; in unp_soisconnected()
3077 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); in unp_soisconnected()
3078 sb = &so->so_rcv; in unp_soisconnected()
3079 selwakeuppri(sb->sb_sel, PSOCK); in unp_soisconnected()
3081 sb = &so->so_snd; in unp_soisconnected()
3082 selwakeuppri(sb->sb_sel, PSOCK); in unp_soisconnected()
3093 MPASS(so2->so_type == so->so_type); in unp_connect2()
3101 KASSERT(unp->unp_conn == NULL, in unp_connect2()
3104 unp->unp_conn = unp2; in unp_connect2()
3107 switch (so->so_type) { in unp_connect2()
3110 LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); in unp_connect2()
3117 KASSERT(unp2->unp_conn == NULL, in unp_connect2()
3119 unp2->unp_conn = unp; in unp_connect2()
3144 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); in unp_soisdisconnected()
3146 MPASS((so->so_state & (SS_ISCONNECTING | SS_ISDISCONNECTING | in unp_soisdisconnected()
3148 MPASS(so->so_state & SS_ISCONNECTED); in unp_soisdisconnected()
3150 so->so_state |= SS_ISDISCONNECTED; in unp_soisdisconnected()
3151 so->so_state &= ~SS_ISCONNECTED; in unp_soisdisconnected()
3152 so->so_rcv.uxst_peer = NULL; in unp_soisdisconnected()
3167 KASSERT(unp->unp_conn == unp2, in unp_disconnect()
3170 unp->unp_conn = NULL; in unp_disconnect()
3171 so = unp->unp_socket; in unp_disconnect()
3172 so2 = unp2->unp_socket; in unp_disconnect()
3173 switch (unp->unp_socket->so_type) { in unp_disconnect()
3183 if (!STAILQ_EMPTY(&so->so_snd.uxdg_mb)) { in unp_disconnect()
3184 TAILQ_REMOVE(&so2->so_rcv.uxdg_conns, &so->so_snd, in unp_disconnect()
3186 if (__predict_true((so2->so_rcv.sb_state & in unp_disconnect()
3188 STAILQ_EMPTY(&so2->so_rcv.uxdg_mb)) { in unp_disconnect()
3189 STAILQ_CONCAT(&so2->so_rcv.uxdg_mb, in unp_disconnect()
3190 &so->so_snd.uxdg_mb); in unp_disconnect()
3191 so2->so_rcv.uxdg_cc += so->so_snd.uxdg_cc; in unp_disconnect()
3192 so2->so_rcv.uxdg_ctl += so->so_snd.uxdg_ctl; in unp_disconnect()
3193 so2->so_rcv.uxdg_mbcnt += so->so_snd.uxdg_mbcnt; in unp_disconnect()
3195 m = STAILQ_FIRST(&so->so_snd.uxdg_mb); in unp_disconnect()
3196 STAILQ_INIT(&so->so_snd.uxdg_mb); in unp_disconnect()
3197 so2->so_rcv.sb_acc -= so->so_snd.uxdg_cc; in unp_disconnect()
3198 so2->so_rcv.sb_ccc -= so->so_snd.uxdg_cc; in unp_disconnect()
3199 so2->so_rcv.sb_ctl -= so->so_snd.uxdg_ctl; in unp_disconnect()
3200 so2->so_rcv.sb_mbcnt -= so->so_snd.uxdg_mbcnt; in unp_disconnect()
3203 so->so_snd.uxdg_cc = 0; in unp_disconnect()
3204 so->so_snd.uxdg_ctl = 0; in unp_disconnect()
3205 so->so_snd.uxdg_mbcnt = 0; in unp_disconnect()
3210 LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) { in unp_disconnect()
3221 so->so_state &= ~SS_ISCONNECTED; in unp_disconnect()
3233 MPASS(unp2->unp_conn == unp); in unp_disconnect()
3234 unp2->unp_conn = NULL; in unp_disconnect()
3294 * The process of preparing the PCB list is too time-consuming and in unp_pcblist()
3295 * resource-intensive to repeat twice on every request. in unp_pcblist()
3297 if (req->oldptr == NULL) { in unp_pcblist()
3299 req->oldidx = 2 * (sizeof *xug) in unp_pcblist()
3304 if (req->newptr != NULL) in unp_pcblist()
3316 xug->xug_len = sizeof *xug; in unp_pcblist()
3317 xug->xug_count = n; in unp_pcblist()
3318 xug->xug_gen = gencnt; in unp_pcblist()
3319 xug->xug_sogen = so_gencnt; in unp_pcblist()
3332 if (unp->unp_gencnt <= gencnt) { in unp_pcblist()
3333 if (cr_cansee(req->td->td_ucred, in unp_pcblist()
3334 unp->unp_socket->so_cred)) { in unp_pcblist()
3354 if (unp->unp_gencnt <= gencnt) { in unp_pcblist()
3355 xu->xu_len = sizeof *xu; in unp_pcblist()
3356 xu->xu_unpp = (uintptr_t)unp; in unp_pcblist()
3358 * XXX - need more locking here to protect against in unp_pcblist()
3361 if (unp->unp_addr != NULL) in unp_pcblist()
3362 bcopy(unp->unp_addr, &xu->xu_addr, in unp_pcblist()
3363 unp->unp_addr->sun_len); in unp_pcblist()
3365 bzero(&xu->xu_addr, sizeof(xu->xu_addr)); in unp_pcblist()
3366 if (unp->unp_conn != NULL && in unp_pcblist()
3367 unp->unp_conn->unp_addr != NULL) in unp_pcblist()
3368 bcopy(unp->unp_conn->unp_addr, in unp_pcblist()
3369 &xu->xu_caddr, in unp_pcblist()
3370 unp->unp_conn->unp_addr->sun_len); in unp_pcblist()
3372 bzero(&xu->xu_caddr, sizeof(xu->xu_caddr)); in unp_pcblist()
3373 xu->unp_vnode = (uintptr_t)unp->unp_vnode; in unp_pcblist()
3374 xu->unp_conn = (uintptr_t)unp->unp_conn; in unp_pcblist()
3375 xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs); in unp_pcblist()
3376 xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink); in unp_pcblist()
3377 xu->unp_gencnt = unp->unp_gencnt; in unp_pcblist()
3378 sotoxsocket(unp->unp_socket, &xu->xu_socket); in unp_pcblist()
3393 xug->xug_gen = unp_gencnt; in unp_pcblist()
3394 xug->xug_sogen = so_gencnt; in unp_pcblist()
3395 xug->xug_count = unp_count; in unp_pcblist()
3429 if ((so = unp->unp_socket) != NULL) in unp_drop()
3430 so->so_error = in unp_drop()
3431 so->so_proto->pr_type == SOCK_DGRAM ? ECONNRESET : EPIPE; in unp_drop()
3450 fp = fdep[i]->fde_file; in unp_freerights()
3451 filecaps_free(&fdep[i]->fde_caps); in unp_freerights()
3462 prison1 = fp->f_cred->cr_prison; in restrict_rights()
3463 prison2 = td->td_ucred->cr_prison; in restrict_rights()
3464 return (prison1 != prison2 && prison1->pr_root != prison2->pr_root && in restrict_rights()
3474 struct filedesc *fdesc = td->td_proc->p_fd; in unp_externalize()
3477 socklen_t clen = control->m_len, datalen; in unp_externalize()
3490 MPASS(clen >= sizeof(*cm) && clen >= cm->cmsg_len); in unp_externalize()
3493 datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; in unp_externalize()
3494 if (cm->cmsg_level == SOL_SOCKET in unp_externalize()
3495 && cm->cmsg_type == SCM_RIGHTS) { in unp_externalize()
3530 fp = fdep[i]->fde_file; in unp_externalize()
3533 O_RESOLVE_BENEATH : 0), &fdep[i]->fde_caps); in unp_externalize()
3550 cm->cmsg_type, cm->cmsg_level, M_WAITOK); in unp_externalize()
3555 controlp = &(*controlp)->m_next; in unp_externalize()
3559 clen -= CMSG_SPACE(datalen); in unp_externalize()
3586 KASSERT(LIST_EMPTY(&unp->unp_refs), in unp_zdtor()
3588 KASSERT(unp->unp_socket == NULL, in unp_zdtor()
3590 KASSERT(unp->unp_vnode == NULL, in unp_zdtor()
3592 KASSERT(unp->unp_conn == NULL, in unp_zdtor()
3594 KASSERT(unp->unp_addr == NULL, in unp_zdtor()
3635 for (m = control; m != NULL; m = m->m_next) { in unp_internalize_cleanup_rights()
3637 if (cp->cmsg_level != SOL_SOCKET || in unp_internalize_cleanup_rights()
3638 cp->cmsg_type != SCM_RIGHTS) in unp_internalize_cleanup_rights()
3641 datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data; in unp_internalize_cleanup_rights()
3664 MPASS(control->m_next == NULL); /* COMPAT_OLDSOCK may violate */ in unp_internalize()
3667 p = td->td_proc; in unp_internalize()
3668 fdesc = p->p_fd; in unp_internalize()
3671 for (clen = control->m_len, cm = mtod(control, struct cmsghdr *), in unp_internalize()
3674 clen >= sizeof(*cm) && cm->cmsg_level == SOL_SOCKET && in unp_internalize()
3675 clen >= cm->cmsg_len && cm->cmsg_len >= sizeof(*cm) && in unp_internalize()
3676 (char *)cm + cm->cmsg_len >= (char *)data; in unp_internalize()
3678 clen -= min(CMSG_SPACE(datalen), clen), in unp_internalize()
3681 datalen = (char *)cm + cm->cmsg_len - (char *)data; in unp_internalize()
3682 switch (cm->cmsg_type) { in unp_internalize()
3688 cmcred->cmcred_pid = p->p_pid; in unp_internalize()
3689 cmcred->cmcred_uid = td->td_ucred->cr_ruid; in unp_internalize()
3690 cmcred->cmcred_gid = td->td_ucred->cr_rgid; in unp_internalize()
3691 cmcred->cmcred_euid = td->td_ucred->cr_uid; in unp_internalize()
3694 cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups + 1, in unp_internalize()
3696 cmcred->cmcred_groups[0] = td->td_ucred->cr_gid; in unp_internalize()
3697 for (i = 1; i < cmcred->cmcred_ngroups; i++) in unp_internalize()
3698 cmcred->cmcred_groups[i] = in unp_internalize()
3699 td->td_ucred->cr_groups[i - 1]; in unp_internalize()
3732 if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) { in unp_internalize()
3747 if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) { in unp_internalize()
3750 fdrop(fdesc->fd_ofiles[*fdp]. in unp_internalize()
3764 fde = &fdesc->fd_ofiles[*fdp]; in unp_internalize()
3766 fdep[i]->fde_file = fde->fde_file; in unp_internalize()
3767 filecaps_copy(&fde->fde_caps, in unp_internalize()
3768 &fdep[i]->fde_caps, true); in unp_internalize()
3769 unp_internalize_fp(fdep[i]->fde_file); in unp_internalize()
3831 ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX); in unp_addsockcred()
3844 MPASS((m->m_flags & M_EXT) == 0 && m->m_next == NULL); in unp_addsockcred()
3850 sc->sc_version = 0; in unp_addsockcred()
3851 sc->sc_pid = td->td_proc->p_pid; in unp_addsockcred()
3852 sc->sc_uid = td->td_ucred->cr_ruid; in unp_addsockcred()
3853 sc->sc_euid = td->td_ucred->cr_uid; in unp_addsockcred()
3854 sc->sc_gid = td->td_ucred->cr_rgid; in unp_addsockcred()
3855 sc->sc_egid = td->td_ucred->cr_gid; in unp_addsockcred()
3856 sc->sc_ngroups = ngroups; in unp_addsockcred()
3857 for (i = 0; i < sc->sc_ngroups; i++) in unp_addsockcred()
3858 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; in unp_addsockcred()
3863 sc->sc_uid = td->td_ucred->cr_ruid; in unp_addsockcred()
3864 sc->sc_euid = td->td_ucred->cr_uid; in unp_addsockcred()
3865 sc->sc_gid = td->td_ucred->cr_rgid; in unp_addsockcred()
3866 sc->sc_egid = td->td_ucred->cr_gid; in unp_addsockcred()
3867 sc->sc_ngroups = ngroups; in unp_addsockcred()
3868 for (i = 0; i < sc->sc_ngroups; i++) in unp_addsockcred()
3869 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; in unp_addsockcred()
3877 if (!STAILQ_EMPTY(&mc->mc_q) && cmsgtype == SCM_CREDS) in unp_addsockcred()
3878 STAILQ_FOREACH_SAFE(n, &mc->mc_q, m_stailq, n_prev) { in unp_addsockcred()
3880 if (cm->cmsg_level == SOL_SOCKET && in unp_addsockcred()
3881 cm->cmsg_type == SCM_CREDS) { in unp_addsockcred()
3896 if (fp->f_type != DTYPE_SOCKET) in fptounp()
3898 if ((so = fp->f_data) == NULL) in fptounp()
3900 if (so->so_proto->pr_domain != &localdomain) in fptounp()
3912 dr->ud_fp = fp; in unp_discard()
3941 closef_nothread(dr->ud_fp); in unp_process_defers()
3945 atomic_add_int(&unp_defers_count, -count); in unp_process_defers()
3956 unp->unp_file = fp; in unp_internalize_fp()
3957 unp->unp_msgcount++; in unp_internalize_fp()
3971 unp->unp_msgcount--; in unp_externalize_fp()
3975 unp_rights--; in unp_externalize_fp()
4002 fp = fdep[i]->fde_file; in unp_remove_dead_ref()
4005 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) in unp_remove_dead_ref()
4007 unp->unp_gcrefs--; in unp_remove_dead_ref()
4026 fp = fdep[i]->fde_file; in unp_restore_undead_ref()
4029 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) in unp_restore_undead_ref()
4031 unp->unp_gcrefs++; in unp_restore_undead_ref()
4043 if (sotounpcb(so)->unp_gcflag & UNPGC_IGNORE_RIGHTS) in unp_scan_socket()
4047 switch (so->so_type) { in unp_scan_socket()
4049 unp_scan(STAILQ_FIRST(&so->so_rcv.uxdg_mb), op); in unp_scan_socket()
4050 unp_scan(so->so_rcv.uxdg_peeked, op); in unp_scan_socket()
4051 TAILQ_FOREACH(sb, &so->so_rcv.uxdg_conns, uxdg_clist) in unp_scan_socket()
4052 unp_scan(STAILQ_FIRST(&sb->uxdg_mb), op); in unp_scan_socket()
4056 unp_scan(STAILQ_FIRST(&so->so_rcv.uxst_mbq), op); in unp_scan_socket()
4067 so = unp->unp_socket; in unp_gc_scan()
4073 TAILQ_FOREACH(soa, &so->sol_comp, so_list) in unp_gc_scan()
4101 struct unp_head unp_deadhead; /* List of potentially-dead sockets. */ in unp_gc()
4116 KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0, in unp_gc()
4118 __func__, unp, (unsigned int)unp->unp_gcflag)); in unp_gc()
4120 f = unp->unp_file; in unp_gc()
4129 if (f != NULL && unp->unp_msgcount != 0 && in unp_gc()
4130 refcount_load(&f->f_count) == unp->unp_msgcount) { in unp_gc()
4132 unp->unp_gcflag |= UNPGC_DEAD; in unp_gc()
4133 unp->unp_gcrefs = unp->unp_msgcount; in unp_gc()
4148 * If a socket still has a non-negative refcount, it cannot be in a in unp_gc()
4156 if (unp->unp_gcrefs > 0) { in unp_gc()
4157 unp->unp_gcflag &= ~UNPGC_DEAD; in unp_gc()
4162 unp_unreachable--; in unp_gc()
4188 KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0, in unp_gc()
4190 unp->unp_gcflag &= ~UNPGC_DEAD; in unp_gc()
4191 f = unp->unp_file; in unp_gc()
4192 if (unp->unp_msgcount == 0 || f == NULL || in unp_gc()
4193 refcount_load(&f->f_count) != unp->unp_msgcount || in unp_gc()
4210 so = unref[i]->f_data; in unp_gc()
4211 CURVNET_SET(so->so_vnet); in unp_gc()
4241 unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS; in unp_dispose()
4250 switch (so->so_type) { in unp_dispose()
4252 while ((sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) != NULL) { in unp_dispose()
4253 STAILQ_CONCAT(&so->so_rcv.uxdg_mb, &sb->uxdg_mb); in unp_dispose()
4254 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); in unp_dispose()
4256 sb->uxdg_cc = sb->uxdg_ctl = sb->uxdg_mbcnt = 0; in unp_dispose()
4258 sb = &so->so_rcv; in unp_dispose()
4259 if (sb->uxdg_peeked != NULL) { in unp_dispose()
4260 STAILQ_INSERT_HEAD(&sb->uxdg_mb, sb->uxdg_peeked, in unp_dispose()
4262 sb->uxdg_peeked = NULL; in unp_dispose()
4264 m = STAILQ_FIRST(&sb->uxdg_mb); in unp_dispose()
4265 STAILQ_INIT(&sb->uxdg_mb); in unp_dispose()
4269 sb = &so->so_rcv; in unp_dispose()
4270 m = STAILQ_FIRST(&sb->uxst_mbq); in unp_dispose()
4271 STAILQ_INIT(&sb->uxst_mbq); in unp_dispose()
4272 sb->sb_acc = sb->sb_ccc = sb->sb_ctl = sb->sb_mbcnt = 0; in unp_dispose()
4277 if (sb->uxst_fnrdy != NULL) { in unp_dispose()
4280 while (m != NULL && m->m_flags & M_NOTREADY) in unp_dispose()
4281 m = m->m_next; in unp_dispose()
4282 for (prev = n = m; n != NULL; n = n->m_next) { in unp_dispose()
4283 if (n->m_flags & M_NOTREADY) in unp_dispose()
4284 prev->m_next = n->m_next; in unp_dispose()
4288 sb->uxst_fnrdy = NULL; in unp_dispose()
4308 sb->sb_state |= SBS_CANTRCVMORE; in unp_dispose()
4309 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, in unp_dispose()
4329 for (m = m0; m; m = m->m_next) { in unp_scan()
4330 if (m->m_type != MT_CONTROL) in unp_scan()
4334 clen = m->m_len; in unp_scan()
4337 if (sizeof(*cm) > clen || cm->cmsg_len > clen) in unp_scan()
4341 datalen = (caddr_t)cm + cm->cmsg_len in unp_scan()
4342 - (caddr_t)data; in unp_scan()
4344 if (cm->cmsg_level == SOL_SOCKET && in unp_scan()
4345 cm->cmsg_type == SCM_RIGHTS) { in unp_scan()
4351 clen -= CMSG_SPACE(datalen); in unp_scan()
4360 m0 = m0->m_nextpkt; in unp_scan()
4461 * A helper function called by VFS before socket-type vnode reclamation.
4473 KASSERT(vp->v_type == VSOCK, in vfs_unp_reclaim()
4474 ("vfs_unp_reclaim: vp->v_type != VSOCK")); in vfs_unp_reclaim()
4483 if (unp->unp_vnode == vp) { in vfs_unp_reclaim()
4485 unp->unp_vnode = NULL; in vfs_unp_reclaim()
4540 xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups); in db_print_xucred()
4544 for (i = 0; i < xu->cr_ngroups; i++) { in db_print_xucred()
4545 db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]); in db_print_xucred()
4580 db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket, in DB_SHOW_COMMAND()
4581 unp->unp_vnode); in DB_SHOW_COMMAND()
4583 db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino, in DB_SHOW_COMMAND()
4584 unp->unp_conn); in DB_SHOW_COMMAND()
4587 db_print_unprefs(2, &unp->unp_refs); in DB_SHOW_COMMAND()
4590 db_printf("unp_addr: %p\n", unp->unp_addr); in DB_SHOW_COMMAND()
4593 (unsigned long long)unp->unp_gencnt); in DB_SHOW_COMMAND()
4595 db_printf("unp_flags: %x (", unp->unp_flags); in DB_SHOW_COMMAND()
4596 db_print_unpflags(unp->unp_flags); in DB_SHOW_COMMAND()
4600 db_print_xucred(2, &unp->unp_peercred); in DB_SHOW_COMMAND()
4602 db_printf("unp_refcount: %u\n", unp->unp_refcount); in DB_SHOW_COMMAND()