Lines Matching +full:cm +full:- +full:poll +full:- +full:mode
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
6 * Copyright (c) 2004-2009 Robert N. M. Watson All Rights Reserved.
8 * Copyright (c) 2022-2025 Gleb Smirnoff <glebius@FreeBSD.org>
56 * need a proper out-of-band
76 #include <sys/poll.h>
155 * large enough for at least one max-size datagram plus address.
198 * - a global linkage lock
199 * - a global connection list lock
200 * - the mtxpool lock
201 * - per-unpcb mutexes
228 * has a back-pointer to its socket, unp_socket, which will be stable under
235 * unp->unp_conn->unp_socket, you need to hold a lock on unp_conn to guarantee
239 * protocols, bind() is a non-atomic operation, and connect() requires
245 * Another tricky issue is simultaneous multi-threaded or multi-process
278 #define UNP_PCB_LOCK_INIT(unp) mtx_init(&(unp)->unp_mtx, \
281 #define UNP_PCB_LOCK_DESTROY(unp) mtx_destroy(&(unp)->unp_mtx)
282 #define UNP_PCB_LOCKPTR(unp) (&(unp)->unp_mtx)
283 #define UNP_PCB_LOCK(unp) mtx_lock(&(unp)->unp_mtx)
284 #define UNP_PCB_TRYLOCK(unp) mtx_trylock(&(unp)->unp_mtx)
285 #define UNP_PCB_UNLOCK(unp) mtx_unlock(&(unp)->unp_mtx)
286 #define UNP_PCB_OWNED(unp) mtx_owned(&(unp)->unp_mtx)
287 #define UNP_PCB_LOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_OWNED)
288 #define UNP_PCB_UNLOCK_ASSERT(unp) mtx_assert(&(unp)->unp_mtx, MA_NOTOWNED)
321 old = refcount_acquire(&unp->unp_refcount); in unp_pcb_hold()
332 if ((ret = refcount_release(&unp->unp_refcount))) { in unp_pcb_rele()
345 ret = refcount_release(&unp->unp_refcount); in unp_pcb_rele_notlast()
386 unp2 = unp->unp_conn; in unp_pcb_lock_peer()
400 unp->unp_pairbusy++; in unp_pcb_lock_peer()
406 KASSERT(unp->unp_conn == unp2 || unp->unp_conn == NULL, in unp_pcb_lock_peer()
408 if (--unp->unp_pairbusy == 0 && (unp->unp_flags & UNP_WAITING) != 0) { in unp_pcb_lock_peer()
409 unp->unp_flags &= ~UNP_WAITING; in unp_pcb_lock_peer()
416 if (unp->unp_conn == NULL) { in unp_pcb_lock_peer()
435 if (__predict_false(so->so_error != 0)) { in uipc_lock_peer()
436 error = so->so_error; in uipc_lock_peer()
437 so->so_error = 0; in uipc_lock_peer()
449 error = so->so_state & SS_ISDISCONNECTED ? EPIPE : ENOTCONN; in uipc_lock_peer()
468 unp2 = unp->unp_conn; in uipc_abort()
485 KASSERT(so->so_pcb == NULL, ("uipc_attach: so_pcb != NULL")); in uipc_attach()
486 switch (so->so_type) { in uipc_attach()
488 STAILQ_INIT(&so->so_rcv.uxdg_mb); in uipc_attach()
489 STAILQ_INIT(&so->so_snd.uxdg_mb); in uipc_attach()
490 TAILQ_INIT(&so->so_rcv.uxdg_conns); in uipc_attach()
493 * of one-to-many receive buffer, we assign both space in uipc_attach()
514 mtx_destroy(&so->so_rcv_mtx); in uipc_attach()
515 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF | MTX_DUPOK); in uipc_attach()
516 knlist_init(&so->so_wrsel.si_note, so, uipc_wrknl_lock, in uipc_attach()
518 STAILQ_INIT(&so->so_rcv.uxst_mbq); in uipc_attach()
529 LIST_INIT(&unp->unp_refs); in uipc_attach()
531 unp->unp_socket = so; in uipc_attach()
532 so->so_pcb = unp; in uipc_attach()
533 refcount_init(&unp->unp_refcount, 1); in uipc_attach()
534 unp->unp_mode = ACCESSPERMS; in uipc_attach()
539 unp->unp_gencnt = ++unp_gencnt; in uipc_attach()
540 unp->unp_ino = ++unp_ino; in uipc_attach()
542 switch (so->so_type) { in uipc_attach()
577 mode_t mode; in uipc_bindat() local
579 if (nam->sa_family != AF_UNIX) in uipc_bindat()
585 if (soun->sun_len > sizeof(struct sockaddr_un)) in uipc_bindat()
587 namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path); in uipc_bindat()
593 * socket, so flag in-progress operations, and return an error if an in uipc_bindat()
597 * also returns an error. Not allowing re-binding simplifies the in uipc_bindat()
601 if (unp->unp_vnode != NULL) { in uipc_bindat()
605 if (unp->unp_flags & UNP_BINDING) { in uipc_bindat()
609 unp->unp_flags |= UNP_BINDING; in uipc_bindat()
610 mode = unp->unp_mode & ~td->td_proc->p_pd->pd_cmask; in uipc_bindat()
614 bcopy(soun->sun_path, buf, namelen); in uipc_bindat()
643 vattr.va_mode = mode; in uipc_bindat()
645 error = mac_vnode_check_create(td->td_ucred, nd.ni_dvp, &nd.ni_cnd, in uipc_bindat()
675 unp->unp_vnode = vp; in uipc_bindat()
676 unp->unp_addr = soun; in uipc_bindat()
677 unp->unp_flags &= ~UNP_BINDING; in uipc_bindat()
687 unp->unp_flags &= ~UNP_BINDING; in uipc_bindat()
732 if ((vp = unp->unp_vnode) != NULL) { in uipc_close()
737 if (vp && unp->unp_vnode == NULL) { in uipc_close()
743 unp->unp_vnode = NULL; in uipc_close()
756 uipc_chmod(struct socket *so, mode_t mode, struct ucred *cred __unused, in uipc_chmod() argument
762 if ((mode & ~ACCESSPERMS) != 0) in uipc_chmod()
768 if (unp->unp_vnode != NULL || (unp->unp_flags & UNP_BINDING) != 0) in uipc_chmod()
771 unp->unp_mode = mode; in uipc_chmod()
781 if (so1->so_type != so2->so_type) in uipc_connect2()
784 unp = so1->so_pcb; in uipc_connect2()
786 unp2 = so2->so_pcb; in uipc_connect2()
799 taskqueue_enqueue_timeout(taskqueue_thread, &unp_gc_task, -1); in maybe_schedule_gc()
820 if (unp->unp_gcflag & UNPGC_DEAD) in uipc_detach()
822 unp->unp_gencnt = ++unp_gencnt; in uipc_detach()
823 --unp_count; in uipc_detach()
828 if ((vp = unp->unp_vnode) != NULL) { in uipc_detach()
833 if (unp->unp_vnode != vp && unp->unp_vnode != NULL) { in uipc_detach()
839 if ((vp = unp->unp_vnode) != NULL) { in uipc_detach()
841 unp->unp_vnode = NULL; in uipc_detach()
849 while (!LIST_EMPTY(&unp->unp_refs)) { in uipc_detach()
850 struct unpcb *ref = LIST_FIRST(&unp->unp_refs); in uipc_detach()
863 unp->unp_socket->so_pcb = NULL; in uipc_detach()
864 unp->unp_socket = NULL; in uipc_detach()
865 free(unp->unp_addr, M_SONAME); in uipc_detach()
866 unp->unp_addr = NULL; in uipc_detach()
875 switch (so->so_type) { in uipc_detach()
878 MPASS(SOLISTENING(so) || (STAILQ_EMPTY(&so->so_rcv.uxst_mbq) && in uipc_detach()
879 so->so_rcv.uxst_peer == NULL)); in uipc_detach()
886 MPASS(so->so_rcv.uxdg_peeked == NULL); in uipc_detach()
887 MPASS(STAILQ_EMPTY(&so->so_rcv.uxdg_mb)); in uipc_detach()
888 MPASS(TAILQ_EMPTY(&so->so_rcv.uxdg_conns)); in uipc_detach()
889 MPASS(STAILQ_EMPTY(&so->so_snd.uxdg_mb)); in uipc_detach()
927 MPASS(so->so_type != SOCK_DGRAM); in uipc_listen()
935 if (unp->unp_conn != NULL || (unp->unp_flags & UNP_CONNECTING) != 0) in uipc_listen()
937 else if (unp->unp_vnode == NULL) in uipc_listen()
947 cru2xt(td, &unp->unp_peercred); in uipc_listen()
949 (void)chgsbsize(so->so_cred->cr_uidinfo, in uipc_listen()
950 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); in uipc_listen()
951 (void)chgsbsize(so->so_cred->cr_uidinfo, in uipc_listen()
952 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); in uipc_listen()
973 if (unp2->unp_addr != NULL) in uipc_peeraddr()
974 sa = (struct sockaddr *)unp2->unp_addr; in uipc_peeraddr()
977 bcopy(sa, ret, sa->sa_len); in uipc_peeraddr()
982 bcopy(sa, ret, sa->sa_len); in uipc_peeraddr()
1004 m->m_pkthdr.rcvif = NULL; in uipc_reset_kernel_mbuf()
1005 m->m_pkthdr.flowid = 0; in uipc_reset_kernel_mbuf()
1006 m->m_pkthdr.csum_flags = 0; in uipc_reset_kernel_mbuf()
1007 m->m_pkthdr.fibnum = 0; in uipc_reset_kernel_mbuf()
1008 m->m_pkthdr.rsstype = 0; in uipc_reset_kernel_mbuf()
1011 MPASS(m->m_pkthdr.len == mc->mc_len); in uipc_reset_kernel_mbuf()
1023 STAILQ_FOREACH(d, &sb->uxst_mbq, m_stailq) { in uipc_stream_sbcheck()
1024 if (d == sb->uxst_fnrdy) { in uipc_stream_sbcheck()
1025 MPASS(d->m_flags & M_NOTREADY); in uipc_stream_sbcheck()
1028 if (d->m_type == MT_CONTROL) in uipc_stream_sbcheck()
1029 dctl += d->m_len; in uipc_stream_sbcheck()
1030 else if (d->m_type == MT_DATA) { in uipc_stream_sbcheck()
1031 dccc += d->m_len; in uipc_stream_sbcheck()
1033 dacc += d->m_len; in uipc_stream_sbcheck()
1037 if (d->m_flags & M_EXT) in uipc_stream_sbcheck()
1038 dmbcnt += d->m_ext.ext_size; in uipc_stream_sbcheck()
1039 if (d->m_stailq.stqe_next == NULL) in uipc_stream_sbcheck()
1040 MPASS(sb->uxst_mbq.stqh_last == &d->m_stailq.stqe_next); in uipc_stream_sbcheck()
1042 MPASS(sb->uxst_fnrdy == NULL || notready); in uipc_stream_sbcheck()
1043 MPASS(dacc == sb->sb_acc); in uipc_stream_sbcheck()
1044 MPASS(dccc == sb->sb_ccc); in uipc_stream_sbcheck()
1045 MPASS(dctl == sb->sb_ctl); in uipc_stream_sbcheck()
1046 MPASS(dmbcnt == sb->sb_mbcnt); in uipc_stream_sbcheck()
1047 (void)STAILQ_EMPTY(&sb->uxst_mbq); in uipc_stream_sbcheck()
1076 if (__predict_true(sb->sb_hiwat >= sb->sb_ccc + sb->sb_ctl)) in uipc_stream_sbspace()
1077 space = sb->sb_hiwat - sb->sb_ccc - sb->sb_ctl; in uipc_stream_sbspace()
1080 if (__predict_true(sb->sb_mbmax >= sb->sb_mbcnt)) in uipc_stream_sbspace()
1081 mbspace = sb->sb_mbmax - sb->sb_mbcnt; in uipc_stream_sbspace()
1095 struct sockbuf *sb = &so->so_rcv; in uipc_stream_sbwait()
1098 sb->sb_flags |= SB_WAIT; in uipc_stream_sbwait()
1099 return (msleep_sbt(&sb->sb_acc, SOCK_RECVBUF_MTX(so), PSOCK | PCATCH, in uipc_stream_sbwait()
1123 nonblock = (so->so_state & SS_NBIO) || in uipc_sosend_stream_or_seqpacket()
1152 if (__predict_false(so->so_snd.sb_flags & SB_AIO_RUNNING)) { in uipc_sosend_stream_or_seqpacket()
1157 resid = uio->uio_resid; in uipc_sosend_stream_or_seqpacket()
1161 * buffer - do the copyin before taking any locks, sized to our in uipc_sosend_stream_or_seqpacket()
1165 error = mc_uiotomc(&mc, uio, so->so_snd.sb_hiwat, 0, M_WAITOK, in uipc_sosend_stream_or_seqpacket()
1179 if (unp2->unp_flags & UNP_WANTCRED_MASK) { in uipc_sosend_stream_or_seqpacket()
1185 unp_addsockcred(td, &cmc, unp2->unp_flags); in uipc_sosend_stream_or_seqpacket()
1186 unp2->unp_flags &= ~UNP_WANTCRED_ONESHOT; in uipc_sosend_stream_or_seqpacket()
1196 so2 = unp2->unp_socket; in uipc_sosend_stream_or_seqpacket()
1199 sb = &so2->so_rcv; in uipc_sosend_stream_or_seqpacket()
1207 if (__predict_false(cmc.mc_len > sb->sb_hiwat)) { in uipc_sosend_stream_or_seqpacket()
1212 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { in uipc_sosend_stream_or_seqpacket()
1223 if (space < sb->sb_lowat || space < cmc.mc_len) { in uipc_sosend_stream_or_seqpacket()
1226 sb->uxst_flags |= UXST_PEER_AIO; in uipc_sosend_stream_or_seqpacket()
1230 so->so_snd.sb_ccc = in uipc_sosend_stream_or_seqpacket()
1231 so->so_snd.sb_hiwat - space; in uipc_sosend_stream_or_seqpacket()
1238 so->so_snd.sb_timeo)) != 0) { in uipc_sosend_stream_or_seqpacket()
1245 space -= cmc.mc_len; in uipc_sosend_stream_or_seqpacket()
1271 STAILQ_CONCAT(&sb->uxst_mbq, &cmc.mc_q); in uipc_sosend_stream_or_seqpacket()
1272 sb->sb_ctl += cmc.mc_len; in uipc_sosend_stream_or_seqpacket()
1273 sb->sb_mbcnt += cmc.mc_mlen; in uipc_sosend_stream_or_seqpacket()
1277 if (sb->uxst_fnrdy == NULL) in uipc_sosend_stream_or_seqpacket()
1278 sb->sb_acc += mc.mc_len; in uipc_sosend_stream_or_seqpacket()
1279 sb->sb_ccc += mc.mc_len; in uipc_sosend_stream_or_seqpacket()
1280 sb->sb_mbcnt += mc.mc_mlen; in uipc_sosend_stream_or_seqpacket()
1281 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); in uipc_sosend_stream_or_seqpacket()
1292 } else if (uio != NULL && uio->uio_resid > 0) { in uipc_sosend_stream_or_seqpacket()
1305 atomic_load_int(&so->so_snd.sb_hiwat), 0, M_WAITOK, in uipc_sosend_stream_or_seqpacket()
1315 td->td_ru.ru_msgsnd++; in uipc_sosend_stream_or_seqpacket()
1325 uio->uio_resid = resid - sent; in uipc_sosend_stream_or_seqpacket()
1340 * In a blocking mode peer is sleeping on our receive buffer, and we need just
1348 struct sockbuf *sb = &so->so_rcv; in uipc_wakeup_writer()
1352 MPASS(sb->uxst_peer != NULL); in uipc_wakeup_writer()
1354 sel = &sb->uxst_peer->so_wrsel; in uipc_wakeup_writer()
1356 if (sb->uxst_flags & UXST_PEER_SEL) { in uipc_wakeup_writer()
1362 sb->uxst_flags &= ~UXST_PEER_SEL; in uipc_wakeup_writer()
1364 if (sb->sb_flags & SB_WAIT) { in uipc_wakeup_writer()
1365 sb->sb_flags &= ~SB_WAIT; in uipc_wakeup_writer()
1366 wakeup(&sb->sb_acc); in uipc_wakeup_writer()
1368 KNOTE_LOCKED(&sel->si_note, 0); in uipc_wakeup_writer()
1377 so->so_rcv.sb_state |= SBS_CANTRCVMORE; in uipc_cantrcvmore()
1378 selwakeuppri(&so->so_rdsel, PSOCK); in uipc_cantrcvmore()
1379 KNOTE_LOCKED(&so->so_rdsel.si_note, 0); in uipc_cantrcvmore()
1380 if (so->so_rcv.uxst_peer != NULL) in uipc_cantrcvmore()
1390 struct sockbuf *sb = &so->so_rcv; in uipc_soreceive_stream_or_seqpacket()
1404 nonblock = (so->so_state & SS_NBIO) || in uipc_soreceive_stream_or_seqpacket()
1417 if (__predict_false((atomic_load_short(&so->so_state) & in uipc_soreceive_stream_or_seqpacket()
1428 while (sb->sb_acc < sb->sb_lowat && in uipc_soreceive_stream_or_seqpacket()
1429 (sb->sb_ctl == 0 || controlp == NULL)) { in uipc_soreceive_stream_or_seqpacket()
1430 if (so->so_error) { in uipc_soreceive_stream_or_seqpacket()
1431 error = so->so_error; in uipc_soreceive_stream_or_seqpacket()
1433 so->so_error = 0; in uipc_soreceive_stream_or_seqpacket()
1438 if (sb->sb_state & SBS_CANTRCVMORE) { in uipc_soreceive_stream_or_seqpacket()
1456 MPASS(STAILQ_FIRST(&sb->uxst_mbq)); in uipc_soreceive_stream_or_seqpacket()
1457 MPASS(sb->sb_acc > 0 || sb->sb_ctl > 0); in uipc_soreceive_stream_or_seqpacket()
1461 first = STAILQ_FIRST(&sb->uxst_mbq); in uipc_soreceive_stream_or_seqpacket()
1462 if (first->m_type == MT_CONTROL) { in uipc_soreceive_stream_or_seqpacket()
1464 STAILQ_FOREACH_FROM(first, &sb->uxst_mbq, m_stailq) { in uipc_soreceive_stream_or_seqpacket()
1465 if (first->m_type != MT_CONTROL) in uipc_soreceive_stream_or_seqpacket()
1467 ctl += first->m_len; in uipc_soreceive_stream_or_seqpacket()
1469 if (first->m_flags & M_EXT) in uipc_soreceive_stream_or_seqpacket()
1470 mbcnt += first->m_ext.ext_size; in uipc_soreceive_stream_or_seqpacket()
1483 for (space = uio->uio_resid, m = next = first, part = NULL, datalen = 0; in uipc_soreceive_stream_or_seqpacket()
1484 space > 0 && m != sb->uxst_fnrdy && m->m_type == MT_DATA; in uipc_soreceive_stream_or_seqpacket()
1486 if (space >= m->m_len) { in uipc_soreceive_stream_or_seqpacket()
1487 space -= m->m_len; in uipc_soreceive_stream_or_seqpacket()
1488 datalen += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1490 if (m->m_flags & M_EXT) in uipc_soreceive_stream_or_seqpacket()
1491 mbcnt += m->m_ext.ext_size; in uipc_soreceive_stream_or_seqpacket()
1492 if (m->m_flags & M_EOR) { in uipc_soreceive_stream_or_seqpacket()
1501 m->m_len -= partlen; in uipc_soreceive_stream_or_seqpacket()
1502 m->m_data += partlen; in uipc_soreceive_stream_or_seqpacket()
1512 STAILQ_INIT(&sb->uxst_mbq); in uipc_soreceive_stream_or_seqpacket()
1514 STAILQ_FIRST(&sb->uxst_mbq) = next; in uipc_soreceive_stream_or_seqpacket()
1515 MPASS(sb->sb_acc >= datalen); in uipc_soreceive_stream_or_seqpacket()
1516 sb->sb_acc -= datalen; in uipc_soreceive_stream_or_seqpacket()
1517 sb->sb_ccc -= datalen; in uipc_soreceive_stream_or_seqpacket()
1518 MPASS(sb->sb_ctl >= ctl); in uipc_soreceive_stream_or_seqpacket()
1519 sb->sb_ctl -= ctl; in uipc_soreceive_stream_or_seqpacket()
1520 MPASS(sb->sb_mbcnt >= mbcnt); in uipc_soreceive_stream_or_seqpacket()
1521 sb->sb_mbcnt -= mbcnt; in uipc_soreceive_stream_or_seqpacket()
1523 if (__predict_true(sb->uxst_peer != NULL)) { in uipc_soreceive_stream_or_seqpacket()
1527 if ((aio = sb->uxst_flags & UXST_PEER_AIO)) in uipc_soreceive_stream_or_seqpacket()
1528 sb->uxst_flags &= ~UXST_PEER_AIO; in uipc_soreceive_stream_or_seqpacket()
1538 struct socket *so2 = unp2->unp_socket; in uipc_soreceive_stream_or_seqpacket()
1541 so2->so_snd.sb_ccc -= datalen; in uipc_soreceive_stream_or_seqpacket()
1551 while (control != NULL && control->m_type == MT_CONTROL) { in uipc_soreceive_stream_or_seqpacket()
1572 (sb->sb_state & SBS_CANTRCVMORE) || in uipc_soreceive_stream_or_seqpacket()
1573 cmc.mc_len + sb->sb_ccc + sb->sb_ctl > in uipc_soreceive_stream_or_seqpacket()
1574 sb->sb_hiwat)) { in uipc_soreceive_stream_or_seqpacket()
1595 STAILQ_CONCAT(&cmc.mc_q, &sb->uxst_mbq); in uipc_soreceive_stream_or_seqpacket()
1596 STAILQ_SWAP(&cmc.mc_q, &sb->uxst_mbq, mbuf); in uipc_soreceive_stream_or_seqpacket()
1598 sb->sb_ctl = sb->sb_acc = sb->sb_ccc = in uipc_soreceive_stream_or_seqpacket()
1599 sb->sb_mbcnt = 0; in uipc_soreceive_stream_or_seqpacket()
1600 STAILQ_FOREACH(m, &sb->uxst_mbq, m_stailq) { in uipc_soreceive_stream_or_seqpacket()
1601 if (m->m_type == MT_DATA) { in uipc_soreceive_stream_or_seqpacket()
1602 sb->sb_acc += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1603 sb->sb_ccc += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1605 sb->sb_ctl += m->m_len; in uipc_soreceive_stream_or_seqpacket()
1607 sb->sb_mbcnt += MSIZE; in uipc_soreceive_stream_or_seqpacket()
1608 if (m->m_flags & M_EXT) in uipc_soreceive_stream_or_seqpacket()
1609 sb->sb_mbcnt += in uipc_soreceive_stream_or_seqpacket()
1610 m->m_ext.ext_size; in uipc_soreceive_stream_or_seqpacket()
1619 controlp = &(*controlp)->m_next; in uipc_soreceive_stream_or_seqpacket()
1627 * userland. They are useless to a law-abiding in uipc_soreceive_stream_or_seqpacket()
1633 *controlp = m_copym(control, 0, control->m_len, in uipc_soreceive_stream_or_seqpacket()
1635 controlp = &(*controlp)->m_next; in uipc_soreceive_stream_or_seqpacket()
1648 mtod(m, char *) : mtod(m, char *) - partlen; in uipc_soreceive_stream_or_seqpacket()
1652 len = m->m_len; in uipc_soreceive_stream_or_seqpacket()
1659 MPASS(datalen >= m->m_len); in uipc_soreceive_stream_or_seqpacket()
1660 datalen -= m->m_len; in uipc_soreceive_stream_or_seqpacket()
1666 datalen -= len; in uipc_soreceive_stream_or_seqpacket()
1670 if (waitall && !(flags & MSG_EOR) && uio->uio_resid > 0) in uipc_soreceive_stream_or_seqpacket()
1677 uio->uio_td->td_ru.ru_msgrcv++; in uipc_soreceive_stream_or_seqpacket()
1697 else if (!TAILQ_EMPTY(&so->sol_comp)) in uipc_sopoll_stream_or_seqpacket()
1699 else if (so->so_error) in uipc_sopoll_stream_or_seqpacket()
1702 selrecord(td, &so->so_rdsel); in uipc_sopoll_stream_or_seqpacket()
1707 if (so->so_state & SS_ISDISCONNECTED) in uipc_sopoll_stream_or_seqpacket()
1713 if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat || in uipc_sopoll_stream_or_seqpacket()
1714 so->so_error || so->so_rerror) in uipc_sopoll_stream_or_seqpacket()
1716 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) in uipc_sopoll_stream_or_seqpacket()
1720 selrecord(td, &so->so_rdsel); in uipc_sopoll_stream_or_seqpacket()
1721 so->so_rcv.sb_flags |= SB_SEL; in uipc_sopoll_stream_or_seqpacket()
1726 struct socket *so2 = so->so_rcv.uxst_peer; in uipc_sopoll_stream_or_seqpacket()
1729 struct sockbuf *sb = &so2->so_rcv; in uipc_sopoll_stream_or_seqpacket()
1732 if (uipc_stream_sbspace(sb) >= sb->sb_lowat) in uipc_sopoll_stream_or_seqpacket()
1735 if (sb->sb_state & SBS_CANTRCVMORE) in uipc_sopoll_stream_or_seqpacket()
1738 so2->so_rcv.uxst_flags |= UXST_PEER_SEL; in uipc_sopoll_stream_or_seqpacket()
1739 selrecord(td, &so->so_wrsel); in uipc_sopoll_stream_or_seqpacket()
1743 selrecord(td, &so->so_wrsel); in uipc_sopoll_stream_or_seqpacket()
1765 if (so->so_rcv.uxst_peer != NULL) in uipc_wrknl_lock()
1766 SOCK_RECVBUF_LOCK(so->so_rcv.uxst_peer); in uipc_wrknl_lock()
1779 if (so->so_rcv.uxst_peer != NULL) in uipc_wrknl_unlock()
1780 SOCK_RECVBUF_UNLOCK(so->so_rcv.uxst_peer); in uipc_wrknl_unlock()
1802 if (what == LA_LOCKED && so->so_rcv.uxst_peer != NULL) in uipc_wrknl_assert_lock()
1803 SOCK_RECVBUF_LOCK_ASSERT(so->so_rcv.uxst_peer); in uipc_wrknl_assert_lock()
1810 struct socket *so = kn->kn_fp->f_data; in uipc_filt_sowdetach()
1813 knlist_remove(&so->so_wrsel.si_note, kn, 1); in uipc_filt_sowdetach()
1820 struct socket *so = kn->kn_fp->f_data, *so2; in uipc_filt_sowrite()
1821 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; in uipc_filt_sowrite()
1827 if (so->so_state & SS_ISDISCONNECTED) { in uipc_filt_sowrite()
1828 kn->kn_flags |= EV_EOF; in uipc_filt_sowrite()
1829 kn->kn_fflags = so->so_error; in uipc_filt_sowrite()
1835 so2 = unp2->unp_socket; in uipc_filt_sowrite()
1837 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); in uipc_filt_sowrite()
1839 if (so2->so_rcv.sb_state & SBS_CANTRCVMORE) { in uipc_filt_sowrite()
1840 kn->kn_flags |= EV_EOF; in uipc_filt_sowrite()
1842 } else if (kn->kn_sfflags & NOTE_LOWAT) in uipc_filt_sowrite()
1843 return (kn->kn_data >= kn->kn_sdata); in uipc_filt_sowrite()
1845 return (kn->kn_data >= so2->so_rcv.sb_lowat); in uipc_filt_sowrite()
1851 struct socket *so = kn->kn_fp->f_data, *so2; in uipc_filt_soempty()
1852 struct unpcb *unp = sotounpcb(so), *unp2 = unp->unp_conn; in uipc_filt_soempty()
1857 so2 = unp2->unp_socket; in uipc_filt_soempty()
1859 kn->kn_data = uipc_stream_sbspace(&so2->so_rcv); in uipc_filt_soempty()
1861 return (kn->kn_data == 0 ? 1 : 0); in uipc_filt_soempty()
1883 switch (kn->kn_filter) { in uipc_kqfilter_stream_or_seqpacket()
1887 kn->kn_fop = &uipc_write_filtops; in uipc_kqfilter_stream_or_seqpacket()
1890 kn->kn_fop = &uipc_empty_filtops; in uipc_kqfilter_stream_or_seqpacket()
1896 knl = &so->so_wrsel.si_note; in uipc_kqfilter_stream_or_seqpacket()
1903 struct socket *so2 = so->so_rcv.uxst_peer; in uipc_kqfilter_stream_or_seqpacket()
1925 if (__predict_false(sb->sb_hiwat < sb->uxdg_cc || in uipc_dgram_sbspace()
1926 sb->sb_mbmax < sb->uxdg_mbcnt)) in uipc_dgram_sbspace()
1929 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) in uipc_dgram_sbspace()
1932 bleft = sb->sb_hiwat - sb->uxdg_cc; in uipc_dgram_sbspace()
1933 mleft = sb->sb_mbmax - sb->uxdg_mbcnt; in uipc_dgram_sbspace()
1942 * from -> control -> data and append it to the socket buffer.
1974 if (__predict_false(uio->uio_resid > unpdg_maxdgram)) { in uipc_sosend_dgram()
1984 cc = m->m_pkthdr.len; in uipc_sosend_dgram()
1985 mbcnt = MSIZE + m->m_pkthdr.memlen; in uipc_sosend_dgram()
1994 if (__predict_false(m->m_pkthdr.len > unpdg_maxdgram)) { in uipc_sosend_dgram()
2018 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { in uipc_sosend_dgram()
2023 if (so->so_error != 0) { in uipc_sosend_dgram()
2024 error = so->so_error; in uipc_sosend_dgram()
2025 so->so_error = 0; in uipc_sosend_dgram()
2029 if (((so->so_state & SS_ISCONNECTED) == 0) && addr == NULL) { in uipc_sosend_dgram()
2040 unp2 = unp->unp_conn; in uipc_sosend_dgram()
2052 if (unp2->unp_flags & UNP_WANTCRED_MASK) in uipc_sosend_dgram()
2053 unp_addsockcred(td, &cmc, unp2->unp_flags); in uipc_sosend_dgram()
2054 if (unp->unp_addr != NULL) in uipc_sosend_dgram()
2055 from = (struct sockaddr *)unp->unp_addr; in uipc_sosend_dgram()
2058 f->m_len = from->sa_len; in uipc_sosend_dgram()
2059 MPASS(from->sa_len <= MLEN); in uipc_sosend_dgram()
2060 bcopy(from, mtod(f, void *), from->sa_len); in uipc_sosend_dgram()
2063 * Concatenate mbufs: from -> control -> data. in uipc_sosend_dgram()
2067 f->m_next = mc_first(&cmc); in uipc_sosend_dgram()
2068 mc_last(&cmc)->m_next = m; in uipc_sosend_dgram()
2072 f->m_next = m; in uipc_sosend_dgram()
2074 ctl = f->m_len + cmc.mc_len; in uipc_sosend_dgram()
2078 for (struct mbuf *mb = f; mb != NULL; mb = mb->m_next) { in uipc_sosend_dgram()
2079 if (mb->m_type == MT_DATA) in uipc_sosend_dgram()
2080 dcc += mb->m_len; in uipc_sosend_dgram()
2082 dctl += mb->m_len; in uipc_sosend_dgram()
2084 if (mb->m_flags & M_EXT) in uipc_sosend_dgram()
2085 dmbcnt += mb->m_ext.ext_size; in uipc_sosend_dgram()
2091 f->m_pkthdr.len = cc + ctl; in uipc_sosend_dgram()
2092 f->m_pkthdr.memlen = mbcnt; in uipc_sosend_dgram()
2093 f->m_pkthdr.ctllen = ctl; in uipc_sosend_dgram()
2098 * Unconnected sends, when !(so->so_state & SS_ISCONNECTED) and the in uipc_sosend_dgram()
2120 so2 = unp2->unp_socket; in uipc_sosend_dgram()
2121 sb = (addr == NULL) ? &so->so_snd : &so2->so_rcv; in uipc_sosend_dgram()
2124 if (addr == NULL && STAILQ_EMPTY(&sb->uxdg_mb)) in uipc_sosend_dgram()
2125 TAILQ_INSERT_HEAD(&so2->so_rcv.uxdg_conns, &so->so_snd, in uipc_sosend_dgram()
2127 STAILQ_INSERT_TAIL(&sb->uxdg_mb, f, m_stailqpkt); in uipc_sosend_dgram()
2128 sb->uxdg_cc += cc + ctl; in uipc_sosend_dgram()
2129 sb->uxdg_ctl += ctl; in uipc_sosend_dgram()
2130 sb->uxdg_mbcnt += mbcnt; in uipc_sosend_dgram()
2131 so2->so_rcv.sb_acc += cc + ctl; in uipc_sosend_dgram()
2132 so2->so_rcv.sb_ccc += cc + ctl; in uipc_sosend_dgram()
2133 so2->so_rcv.sb_ctl += ctl; in uipc_sosend_dgram()
2134 so2->so_rcv.sb_mbcnt += mbcnt; in uipc_sosend_dgram()
2140 if (f->m_next->m_type == MT_CONTROL) { in uipc_sosend_dgram()
2141 STAILQ_FIRST(&cmc.mc_q) = f->m_next; in uipc_sosend_dgram()
2142 f->m_next = NULL; in uipc_sosend_dgram()
2151 td->td_ru.ru_msgsnd++; in uipc_sosend_dgram()
2180 so->so_rcv.uxdg_peeked = m; in uipc_peek_dgram()
2181 so->so_rcv.uxdg_cc += m->m_pkthdr.len; in uipc_peek_dgram()
2182 so->so_rcv.uxdg_ctl += m->m_pkthdr.ctllen; in uipc_peek_dgram()
2183 so->so_rcv.uxdg_mbcnt += m->m_pkthdr.memlen; in uipc_peek_dgram()
2186 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); in uipc_peek_dgram()
2190 m = m->m_next; in uipc_peek_dgram()
2196 while (m != NULL && m->m_type == MT_CONTROL) { in uipc_peek_dgram()
2198 *controlp = m_copym(m, 0, m->m_len, M_WAITOK); in uipc_peek_dgram()
2199 controlp = &(*controlp)->m_next; in uipc_peek_dgram()
2201 m = m->m_next; in uipc_peek_dgram()
2203 KASSERT(m == NULL || m->m_type == MT_DATA, in uipc_peek_dgram()
2205 while (m != NULL && uio->uio_resid > 0) { in uipc_peek_dgram()
2206 len = uio->uio_resid; in uipc_peek_dgram()
2207 if (len > m->m_len) in uipc_peek_dgram()
2208 len = m->m_len; in uipc_peek_dgram()
2214 if (len == m->m_len) in uipc_peek_dgram()
2215 m = m->m_next; in uipc_peek_dgram()
2223 uio->uio_resid -= m_length(m, NULL) - len; in uipc_peek_dgram()
2254 nonblock = (so->so_state & SS_NBIO) || in uipc_soreceive_dgram()
2268 while ((m = so->so_rcv.uxdg_peeked) == NULL && in uipc_soreceive_dgram()
2269 (sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) == NULL && in uipc_soreceive_dgram()
2270 (m = STAILQ_FIRST(&so->so_rcv.uxdg_mb)) == NULL) { in uipc_soreceive_dgram()
2271 if (so->so_error) { in uipc_soreceive_dgram()
2272 error = so->so_error; in uipc_soreceive_dgram()
2274 so->so_error = 0; in uipc_soreceive_dgram()
2279 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || in uipc_soreceive_dgram()
2280 uio->uio_resid == 0) { in uipc_soreceive_dgram()
2299 sb = &so->so_rcv; in uipc_soreceive_dgram()
2301 m = STAILQ_FIRST(&sb->uxdg_mb); in uipc_soreceive_dgram()
2303 MPASS(m == so->so_rcv.uxdg_peeked); in uipc_soreceive_dgram()
2305 MPASS(sb->uxdg_cc > 0); in uipc_soreceive_dgram()
2307 KASSERT(m->m_type == MT_SONAME, ("m->m_type == %d", m->m_type)); in uipc_soreceive_dgram()
2309 if (uio->uio_td) in uipc_soreceive_dgram()
2310 uio->uio_td->td_ru.ru_msgrcv++; in uipc_soreceive_dgram()
2312 if (__predict_true(m != so->so_rcv.uxdg_peeked)) { in uipc_soreceive_dgram()
2313 STAILQ_REMOVE_HEAD(&sb->uxdg_mb, m_stailqpkt); in uipc_soreceive_dgram()
2314 if (STAILQ_EMPTY(&sb->uxdg_mb) && sb != &so->so_rcv) in uipc_soreceive_dgram()
2315 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); in uipc_soreceive_dgram()
2317 so->so_rcv.uxdg_peeked = NULL; in uipc_soreceive_dgram()
2319 sb->uxdg_cc -= m->m_pkthdr.len; in uipc_soreceive_dgram()
2320 sb->uxdg_ctl -= m->m_pkthdr.ctllen; in uipc_soreceive_dgram()
2321 sb->uxdg_mbcnt -= m->m_pkthdr.memlen; in uipc_soreceive_dgram()
2326 so->so_rcv.sb_acc -= m->m_pkthdr.len; in uipc_soreceive_dgram()
2327 so->so_rcv.sb_ccc -= m->m_pkthdr.len; in uipc_soreceive_dgram()
2328 so->so_rcv.sb_ctl -= m->m_pkthdr.ctllen; in uipc_soreceive_dgram()
2329 so->so_rcv.sb_mbcnt -= m->m_pkthdr.memlen; in uipc_soreceive_dgram()
2347 while (m != NULL && m->m_type == MT_CONTROL) { in uipc_soreceive_dgram()
2358 controlp = &(*controlp)->m_next; in uipc_soreceive_dgram()
2361 KASSERT(m == NULL || m->m_type == MT_DATA, in uipc_soreceive_dgram()
2363 while (m != NULL && uio->uio_resid > 0) { in uipc_soreceive_dgram()
2364 len = uio->uio_resid; in uipc_soreceive_dgram()
2365 if (len > m->m_len) in uipc_soreceive_dgram()
2366 len = m->m_len; in uipc_soreceive_dgram()
2373 if (len == m->m_len) in uipc_soreceive_dgram()
2376 m->m_data += len; in uipc_soreceive_dgram()
2377 m->m_len -= len; in uipc_soreceive_dgram()
2386 uio->uio_resid -= m_length(m, NULL); in uipc_soreceive_dgram()
2406 MPASS(so->so_type == SOCK_STREAM); in uipc_sendfile_wait()
2410 nonblock = so->so_state & SS_NBIO; in uipc_sendfile_wait()
2413 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) in uipc_sendfile_wait()
2419 so2 = unp2->unp_socket; in uipc_sendfile_wait()
2420 sb = &so2->so_rcv; in uipc_sendfile_wait()
2424 (*space < so->so_snd.sb_hiwat / 2)) { in uipc_sendfile_wait()
2432 error = uipc_stream_sbwait(so2, so->so_snd.sb_timeo); in uipc_sendfile_wait()
2434 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) in uipc_sendfile_wait()
2466 MPASS(so->so_type == SOCK_STREAM); in uipc_sendfile()
2468 KASSERT(!(m->m_flags & M_EXTPG), in uipc_sendfile()
2473 if (__predict_false((so->so_state & SS_ISCONNECTED) == 0)) { in uipc_sendfile()
2483 so2 = unp2->unp_socket; in uipc_sendfile()
2484 sb = &so2->so_rcv; in uipc_sendfile()
2488 sb->sb_ccc += mc.mc_len; in uipc_sendfile()
2489 sb->sb_mbcnt += mc.mc_mlen; in uipc_sendfile()
2490 if (sb->uxst_fnrdy == NULL) { in uipc_sendfile()
2494 if (m->m_flags & M_NOTREADY) { in uipc_sendfile()
2495 sb->uxst_fnrdy = m; in uipc_sendfile()
2498 sb->sb_acc += m->m_len; in uipc_sendfile()
2504 sb->sb_acc += mc.mc_len; in uipc_sendfile()
2509 STAILQ_CONCAT(&sb->uxst_mbq, &mc.mc_q); in uipc_sendfile()
2535 blocker = (sb->uxst_fnrdy == m); in uipc_sbready()
2536 STAILQ_FOREACH_FROM(m, &sb->uxst_mbq, m_stailq) { in uipc_sbready()
2538 MPASS(m->m_flags & M_NOTREADY); in uipc_sbready()
2539 m->m_flags &= ~M_NOTREADY; in uipc_sbready()
2541 sb->sb_acc += m->m_len; in uipc_sbready()
2542 count--; in uipc_sbready()
2543 } else if (m->m_flags & M_NOTREADY) in uipc_sbready()
2546 sb->sb_acc += m->m_len; in uipc_sbready()
2549 sb->uxst_fnrdy = m; in uipc_sbready()
2567 sb = &so->so_rcv; in uipc_ready_scan()
2569 if (sb->uxst_fnrdy != NULL) { in uipc_ready_scan()
2570 STAILQ_FOREACH(mb, &sb->uxst_mbq, m_stailq) { in uipc_ready_scan()
2588 MPASS(so->so_type == SOCK_STREAM); in uipc_ready()
2594 so2 = unp2->unp_socket; in uipc_ready()
2595 sb = &so2->so_rcv; in uipc_ready()
2608 * be valid. In this case, the not-ready mbufs are still in uipc_ready()
2614 if (uipc_ready_scan(unp->unp_socket, m, count, &error)) in uipc_ready()
2636 sb->st_blksize = so->so_snd.sb_hiwat; in uipc_sense()
2637 sb->st_dev = NODEV; in uipc_sense()
2638 sb->st_ino = unp->unp_ino; in uipc_sense()
2651 so->so_error = ECONNABORTED; in uipc_shutdown()
2656 } else if ((so->so_state & in uipc_shutdown()
2664 * both backward-compatibility and POSIX requirements by forcing in uipc_shutdown()
2672 if (so->so_type != SOCK_DGRAM) { in uipc_shutdown()
2682 if (so->so_type == SOCK_DGRAM) in uipc_shutdown()
2689 if (so->so_type == SOCK_DGRAM) in uipc_shutdown()
2696 if (so->so_type == SOCK_DGRAM) { in uipc_shutdown()
2700 if (unp->unp_conn != NULL) in uipc_shutdown()
2701 uipc_cantrcvmore(unp->unp_conn->unp_socket); in uipc_shutdown()
2705 wakeup(&so->so_timeo); in uipc_shutdown()
2720 if (unp->unp_addr != NULL) in uipc_sockaddr()
2721 sa = (struct sockaddr *) unp->unp_addr; in uipc_sockaddr()
2724 bcopy(sa, ret, sa->sa_len); in uipc_sockaddr()
2736 if (sopt->sopt_level != SOL_LOCAL) in uipc_ctloutput()
2742 switch (sopt->sopt_dir) { in uipc_ctloutput()
2744 switch (sopt->sopt_name) { in uipc_ctloutput()
2747 if (unp->unp_flags & UNP_HAVEPC) in uipc_ctloutput()
2748 xu = unp->unp_peercred; in uipc_ctloutput()
2750 if (so->so_proto->pr_flags & PR_CONNREQUIRED) in uipc_ctloutput()
2762 optval = unp->unp_flags & UNP_WANTCRED_ONESHOT ? 1 : 0; in uipc_ctloutput()
2768 optval = unp->unp_flags & UNP_WANTCRED_ALWAYS ? 1 : 0; in uipc_ctloutput()
2779 switch (sopt->sopt_name) { in uipc_ctloutput()
2790 if ((unp->unp_flags & (exclusive)) != 0) { \ in uipc_ctloutput()
2795 unp->unp_flags |= (bit); \ in uipc_ctloutput()
2797 unp->unp_flags &= ~(bit); \ in uipc_ctloutput()
2801 switch (sopt->sopt_name) { in uipc_ctloutput()
2853 if (nam->sa_family != AF_UNIX) in unp_connectat()
2855 if (nam->sa_len > sizeof(struct sockaddr_un)) in unp_connectat()
2857 len = nam->sa_len - offsetof(struct sockaddr_un, sun_path); in unp_connectat()
2861 bcopy(soun->sun_path, buf, len); in unp_connectat()
2882 else if (unp->unp_conn != NULL) in unp_connectat()
2884 else if ((unp->unp_flags & UNP_CONNECTING) != 0) { in unp_connectat()
2891 if (unp->unp_pairbusy > 0) { in unp_connectat()
2892 unp->unp_flags |= UNP_WAITING; in unp_connectat()
2898 unp->unp_flags |= UNP_CONNECTING; in unp_connectat()
2901 connreq = (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0; in unp_connectat()
2918 if (vp->v_type != VSOCK) { in unp_connectat()
2923 error = mac_vnode_check_open(td->td_ucred, vp, VWRITE | VREAD); in unp_connectat()
2927 error = VOP_ACCESS(vp, VWRITE, td->td_ucred, td); in unp_connectat()
2941 so2 = unp2->unp_socket; in unp_connectat()
2942 if (so->so_type != so2->so_type) { in unp_connectat()
2961 if (unp2->unp_addr != NULL) { in unp_connectat()
2962 bcopy(unp2->unp_addr, sa, unp2->unp_addr->sun_len); in unp_connectat()
2963 unp3->unp_addr = (struct sockaddr_un *) sa; in unp_connectat()
2984 KASSERT(unp2 != NULL && so2 != NULL && unp2->unp_socket == so2 && in unp_connectat()
2990 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, in unp_connectat()
2992 unp->unp_flags &= ~UNP_CONNECTING; in unp_connectat()
3011 KASSERT((unp->unp_flags & UNP_CONNECTING) != 0, in unp_connectat()
3013 unp->unp_flags &= ~UNP_CONNECTING; in unp_connectat()
3030 cru2xt(td, &client_unp->unp_peercred); in unp_copy_peercred()
3031 client_unp->unp_flags |= UNP_HAVEPC; in unp_copy_peercred()
3033 memcpy(&server_unp->unp_peercred, &listen_unp->unp_peercred, in unp_copy_peercred()
3034 sizeof(server_unp->unp_peercred)); in unp_copy_peercred()
3035 server_unp->unp_flags |= UNP_HAVEPC; in unp_copy_peercred()
3036 client_unp->unp_flags |= (listen_unp->unp_flags & UNP_WANTCRED_MASK); in unp_copy_peercred()
3051 * select/poll/kevent.
3054 * 1) socketpair(2) - in this case socket has not been yet reported to userland
3066 struct socket *so2 = sotounpcb(so)->unp_conn->unp_socket; in unp_soisconnected()
3075 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); in unp_soisconnected()
3076 MPASS((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | in unp_soisconnected()
3078 MPASS(so->so_qstate == SQ_NONE); in unp_soisconnected()
3080 so->so_state &= ~SS_ISDISCONNECTED; in unp_soisconnected()
3081 so->so_state |= SS_ISCONNECTED; in unp_soisconnected()
3083 sb = &so2->so_rcv; in unp_soisconnected()
3084 sb->uxst_peer = so; in unp_soisconnected()
3087 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); in unp_soisconnected()
3088 sb = &so->so_rcv; in unp_soisconnected()
3089 selwakeuppri(sb->sb_sel, PSOCK); in unp_soisconnected()
3091 sb = &so->so_snd; in unp_soisconnected()
3092 selwakeuppri(sb->sb_sel, PSOCK); in unp_soisconnected()
3103 MPASS(so2->so_type == so->so_type); in unp_connect2()
3111 KASSERT(unp->unp_conn == NULL, in unp_connect2()
3114 unp->unp_conn = unp2; in unp_connect2()
3117 switch (so->so_type) { in unp_connect2()
3120 LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink); in unp_connect2()
3127 KASSERT(unp2->unp_conn == NULL, in unp_connect2()
3129 unp2->unp_conn = unp; in unp_connect2()
3154 MPASS(so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET); in unp_soisdisconnected()
3156 MPASS((so->so_state & (SS_ISCONNECTING | SS_ISDISCONNECTING | in unp_soisdisconnected()
3158 MPASS(so->so_state & SS_ISCONNECTED); in unp_soisdisconnected()
3160 so->so_state |= SS_ISDISCONNECTED; in unp_soisdisconnected()
3161 so->so_state &= ~SS_ISCONNECTED; in unp_soisdisconnected()
3162 so->so_rcv.uxst_peer = NULL; in unp_soisdisconnected()
3177 KASSERT(unp->unp_conn == unp2, in unp_disconnect()
3180 unp->unp_conn = NULL; in unp_disconnect()
3181 so = unp->unp_socket; in unp_disconnect()
3182 so2 = unp2->unp_socket; in unp_disconnect()
3183 switch (unp->unp_socket->so_type) { in unp_disconnect()
3193 if (!STAILQ_EMPTY(&so->so_snd.uxdg_mb)) { in unp_disconnect()
3194 TAILQ_REMOVE(&so2->so_rcv.uxdg_conns, &so->so_snd, in unp_disconnect()
3196 if (__predict_true((so2->so_rcv.sb_state & in unp_disconnect()
3198 STAILQ_EMPTY(&so2->so_rcv.uxdg_mb)) { in unp_disconnect()
3199 STAILQ_CONCAT(&so2->so_rcv.uxdg_mb, in unp_disconnect()
3200 &so->so_snd.uxdg_mb); in unp_disconnect()
3201 so2->so_rcv.uxdg_cc += so->so_snd.uxdg_cc; in unp_disconnect()
3202 so2->so_rcv.uxdg_ctl += so->so_snd.uxdg_ctl; in unp_disconnect()
3203 so2->so_rcv.uxdg_mbcnt += so->so_snd.uxdg_mbcnt; in unp_disconnect()
3205 m = STAILQ_FIRST(&so->so_snd.uxdg_mb); in unp_disconnect()
3206 STAILQ_INIT(&so->so_snd.uxdg_mb); in unp_disconnect()
3207 so2->so_rcv.sb_acc -= so->so_snd.uxdg_cc; in unp_disconnect()
3208 so2->so_rcv.sb_ccc -= so->so_snd.uxdg_cc; in unp_disconnect()
3209 so2->so_rcv.sb_ctl -= so->so_snd.uxdg_ctl; in unp_disconnect()
3210 so2->so_rcv.sb_mbcnt -= so->so_snd.uxdg_mbcnt; in unp_disconnect()
3213 so->so_snd.uxdg_cc = 0; in unp_disconnect()
3214 so->so_snd.uxdg_ctl = 0; in unp_disconnect()
3215 so->so_snd.uxdg_mbcnt = 0; in unp_disconnect()
3220 LIST_FOREACH(unptmp, &unp2->unp_refs, unp_reflink) { in unp_disconnect()
3230 so->so_state &= ~SS_ISCONNECTED; in unp_disconnect()
3241 MPASS(unp2->unp_conn == unp); in unp_disconnect()
3242 unp2->unp_conn = NULL; in unp_disconnect()
3302 * The process of preparing the PCB list is too time-consuming and in unp_pcblist()
3303 * resource-intensive to repeat twice on every request. in unp_pcblist()
3305 if (req->oldptr == NULL) { in unp_pcblist()
3307 req->oldidx = 2 * (sizeof *xug) in unp_pcblist()
3312 if (req->newptr != NULL) in unp_pcblist()
3324 xug->xug_len = sizeof *xug; in unp_pcblist()
3325 xug->xug_count = n; in unp_pcblist()
3326 xug->xug_gen = gencnt; in unp_pcblist()
3327 xug->xug_sogen = so_gencnt; in unp_pcblist()
3340 if (unp->unp_gencnt <= gencnt) { in unp_pcblist()
3341 if (cr_cansee(req->td->td_ucred, in unp_pcblist()
3342 unp->unp_socket->so_cred)) { in unp_pcblist()
3362 if (unp->unp_gencnt <= gencnt) { in unp_pcblist()
3363 xu->xu_len = sizeof *xu; in unp_pcblist()
3364 xu->xu_unpp = (uintptr_t)unp; in unp_pcblist()
3366 * XXX - need more locking here to protect against in unp_pcblist()
3369 if (unp->unp_addr != NULL) in unp_pcblist()
3370 bcopy(unp->unp_addr, &xu->xu_addr, in unp_pcblist()
3371 unp->unp_addr->sun_len); in unp_pcblist()
3373 bzero(&xu->xu_addr, sizeof(xu->xu_addr)); in unp_pcblist()
3374 if (unp->unp_conn != NULL && in unp_pcblist()
3375 unp->unp_conn->unp_addr != NULL) in unp_pcblist()
3376 bcopy(unp->unp_conn->unp_addr, in unp_pcblist()
3377 &xu->xu_caddr, in unp_pcblist()
3378 unp->unp_conn->unp_addr->sun_len); in unp_pcblist()
3380 bzero(&xu->xu_caddr, sizeof(xu->xu_caddr)); in unp_pcblist()
3381 xu->unp_vnode = (uintptr_t)unp->unp_vnode; in unp_pcblist()
3382 xu->unp_conn = (uintptr_t)unp->unp_conn; in unp_pcblist()
3383 xu->xu_firstref = (uintptr_t)LIST_FIRST(&unp->unp_refs); in unp_pcblist()
3384 xu->xu_nextref = (uintptr_t)LIST_NEXT(unp, unp_reflink); in unp_pcblist()
3385 xu->unp_gencnt = unp->unp_gencnt; in unp_pcblist()
3386 sotoxsocket(unp->unp_socket, &xu->xu_socket); in unp_pcblist()
3401 xug->xug_gen = unp_gencnt; in unp_pcblist()
3402 xug->xug_sogen = so_gencnt; in unp_pcblist()
3403 xug->xug_count = unp_count; in unp_pcblist()
3437 if ((so = unp->unp_socket) != NULL) in unp_drop()
3438 so->so_error = in unp_drop()
3439 so->so_proto->pr_type == SOCK_DGRAM ? ECONNRESET : EPIPE; in unp_drop()
3458 fp = fdep[i]->fde_file; in unp_freerights()
3459 filecaps_free(&fdep[i]->fde_caps); in unp_freerights()
3470 prison1 = fp->f_cred->cr_prison; in restrict_rights()
3471 prison2 = td->td_ucred->cr_prison; in restrict_rights()
3472 return (prison1 != prison2 && prison1->pr_root != prison2->pr_root && in restrict_rights()
3480 struct cmsghdr *cm = mtod(control, struct cmsghdr *); in unp_externalize() local
3482 struct filedesc *fdesc = td->td_proc->p_fd; in unp_externalize()
3485 socklen_t clen = control->m_len, datalen; in unp_externalize()
3497 while (cm != NULL) { in unp_externalize()
3498 MPASS(clen >= sizeof(*cm) && clen >= cm->cmsg_len); in unp_externalize()
3500 data = CMSG_DATA(cm); in unp_externalize()
3501 datalen = (caddr_t)cm + cm->cmsg_len - (caddr_t)data; in unp_externalize()
3502 if (cm->cmsg_level == SOL_SOCKET in unp_externalize()
3503 && cm->cmsg_type == SCM_RIGHTS) { in unp_externalize()
3538 fp = fdep[i]->fde_file; in unp_externalize()
3541 O_RESOLVE_BENEATH : 0), &fdep[i]->fde_caps); in unp_externalize()
3558 cm->cmsg_type, cm->cmsg_level, M_WAITOK); in unp_externalize()
3563 controlp = &(*controlp)->m_next; in unp_externalize()
3567 clen -= CMSG_SPACE(datalen); in unp_externalize()
3568 cm = (struct cmsghdr *) in unp_externalize()
3569 ((caddr_t)cm + CMSG_SPACE(datalen)); in unp_externalize()
3572 cm = NULL; in unp_externalize()
3594 KASSERT(LIST_EMPTY(&unp->unp_refs), in unp_zdtor()
3596 KASSERT(unp->unp_socket == NULL, in unp_zdtor()
3598 KASSERT(unp->unp_vnode == NULL, in unp_zdtor()
3600 KASSERT(unp->unp_conn == NULL, in unp_zdtor()
3602 KASSERT(unp->unp_addr == NULL, in unp_zdtor()
3643 for (m = control; m != NULL; m = m->m_next) { in unp_internalize_cleanup_rights()
3645 if (cp->cmsg_level != SOL_SOCKET || in unp_internalize_cleanup_rights()
3646 cp->cmsg_type != SCM_RIGHTS) in unp_internalize_cleanup_rights()
3649 datalen = (caddr_t)cp + cp->cmsg_len - (caddr_t)data; in unp_internalize_cleanup_rights()
3660 struct cmsghdr *cm; in unp_internalize() local
3672 MPASS(control->m_next == NULL); /* COMPAT_OLDSOCK may violate */ in unp_internalize()
3675 p = td->td_proc; in unp_internalize()
3676 fdesc = p->p_fd; in unp_internalize()
3679 for (clen = control->m_len, cm = mtod(control, struct cmsghdr *), in unp_internalize()
3680 data = CMSG_DATA(cm); in unp_internalize()
3682 clen >= sizeof(*cm) && cm->cmsg_level == SOL_SOCKET && in unp_internalize()
3683 clen >= cm->cmsg_len && cm->cmsg_len >= sizeof(*cm) && in unp_internalize()
3684 (char *)cm + cm->cmsg_len >= (char *)data; in unp_internalize()
3686 clen -= min(CMSG_SPACE(datalen), clen), in unp_internalize()
3687 cm = (struct cmsghdr *) ((char *)cm + CMSG_SPACE(datalen)), in unp_internalize()
3688 data = CMSG_DATA(cm)) { in unp_internalize()
3689 datalen = (char *)cm + cm->cmsg_len - (char *)data; in unp_internalize()
3690 switch (cm->cmsg_type) { in unp_internalize()
3696 cmcred->cmcred_pid = p->p_pid; in unp_internalize()
3697 cmcred->cmcred_uid = td->td_ucred->cr_ruid; in unp_internalize()
3698 cmcred->cmcred_gid = td->td_ucred->cr_rgid; in unp_internalize()
3699 cmcred->cmcred_euid = td->td_ucred->cr_uid; in unp_internalize()
3702 cmcred->cmcred_ngroups = MIN(td->td_ucred->cr_ngroups + 1, in unp_internalize()
3704 cmcred->cmcred_groups[0] = td->td_ucred->cr_gid; in unp_internalize()
3705 for (i = 1; i < cmcred->cmcred_ngroups; i++) in unp_internalize()
3706 cmcred->cmcred_groups[i] = in unp_internalize()
3707 td->td_ucred->cr_groups[i - 1]; in unp_internalize()
3740 if (!(fp->f_ops->fo_flags & DFLAG_PASSABLE)) { in unp_internalize()
3755 if (!fhold(fdesc->fd_ofiles[*fdp].fde_file)) { in unp_internalize()
3758 fdrop(fdesc->fd_ofiles[*fdp]. in unp_internalize()
3772 fde = &fdesc->fd_ofiles[*fdp]; in unp_internalize()
3774 fdep[i]->fde_file = fde->fde_file; in unp_internalize()
3775 filecaps_copy(&fde->fde_caps, in unp_internalize()
3776 &fdep[i]->fde_caps, true); in unp_internalize()
3777 unp_internalize_fp(fdep[i]->fde_file); in unp_internalize()
3832 unp_addsockcred(struct thread *td, struct mchain *mc, int mode) in unp_addsockcred() argument
3835 const struct cmsghdr *cm; in unp_addsockcred() local
3839 ngroups = MIN(td->td_ucred->cr_ngroups, CMGROUP_MAX); in unp_addsockcred()
3840 if (mode & UNP_WANTCRED_ALWAYS) { in unp_addsockcred()
3852 MPASS((m->m_flags & M_EXT) == 0 && m->m_next == NULL); in unp_addsockcred()
3854 if (mode & UNP_WANTCRED_ALWAYS) { in unp_addsockcred()
3858 sc->sc_version = 0; in unp_addsockcred()
3859 sc->sc_pid = td->td_proc->p_pid; in unp_addsockcred()
3860 sc->sc_uid = td->td_ucred->cr_ruid; in unp_addsockcred()
3861 sc->sc_euid = td->td_ucred->cr_uid; in unp_addsockcred()
3862 sc->sc_gid = td->td_ucred->cr_rgid; in unp_addsockcred()
3863 sc->sc_egid = td->td_ucred->cr_gid; in unp_addsockcred()
3864 sc->sc_ngroups = ngroups; in unp_addsockcred()
3865 for (i = 0; i < sc->sc_ngroups; i++) in unp_addsockcred()
3866 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; in unp_addsockcred()
3871 sc->sc_uid = td->td_ucred->cr_ruid; in unp_addsockcred()
3872 sc->sc_euid = td->td_ucred->cr_uid; in unp_addsockcred()
3873 sc->sc_gid = td->td_ucred->cr_rgid; in unp_addsockcred()
3874 sc->sc_egid = td->td_ucred->cr_gid; in unp_addsockcred()
3875 sc->sc_ngroups = ngroups; in unp_addsockcred()
3876 for (i = 0; i < sc->sc_ngroups; i++) in unp_addsockcred()
3877 sc->sc_groups[i] = td->td_ucred->cr_groups[i]; in unp_addsockcred()
3885 if (!STAILQ_EMPTY(&mc->mc_q) && cmsgtype == SCM_CREDS) in unp_addsockcred()
3886 STAILQ_FOREACH_SAFE(n, &mc->mc_q, m_stailq, n_prev) { in unp_addsockcred()
3887 cm = mtod(n, struct cmsghdr *); in unp_addsockcred()
3888 if (cm->cmsg_level == SOL_SOCKET && in unp_addsockcred()
3889 cm->cmsg_type == SCM_CREDS) { in unp_addsockcred()
3904 if (fp->f_type != DTYPE_SOCKET) in fptounp()
3906 if ((so = fp->f_data) == NULL) in fptounp()
3908 if (so->so_proto->pr_domain != &localdomain) in fptounp()
3920 dr->ud_fp = fp; in unp_discard()
3949 closef_nothread(dr->ud_fp); in unp_process_defers()
3953 atomic_add_int(&unp_defers_count, -count); in unp_process_defers()
3964 unp->unp_file = fp; in unp_internalize_fp()
3965 unp->unp_msgcount++; in unp_internalize_fp()
3979 unp->unp_msgcount--; in unp_externalize_fp()
3983 unp_rights--; in unp_externalize_fp()
4010 fp = fdep[i]->fde_file; in unp_remove_dead_ref()
4013 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) in unp_remove_dead_ref()
4015 unp->unp_gcrefs--; in unp_remove_dead_ref()
4034 fp = fdep[i]->fde_file; in unp_restore_undead_ref()
4037 if ((unp->unp_gcflag & UNPGC_DEAD) == 0) in unp_restore_undead_ref()
4039 unp->unp_gcrefs++; in unp_restore_undead_ref()
4051 if (sotounpcb(so)->unp_gcflag & UNPGC_IGNORE_RIGHTS) in unp_scan_socket()
4055 switch (so->so_type) { in unp_scan_socket()
4057 unp_scan(STAILQ_FIRST(&so->so_rcv.uxdg_mb), op); in unp_scan_socket()
4058 unp_scan(so->so_rcv.uxdg_peeked, op); in unp_scan_socket()
4059 TAILQ_FOREACH(sb, &so->so_rcv.uxdg_conns, uxdg_clist) in unp_scan_socket()
4060 unp_scan(STAILQ_FIRST(&sb->uxdg_mb), op); in unp_scan_socket()
4064 unp_scan(STAILQ_FIRST(&so->so_rcv.uxst_mbq), op); in unp_scan_socket()
4075 so = unp->unp_socket; in unp_gc_scan()
4081 TAILQ_FOREACH(soa, &so->sol_comp, so_list) in unp_gc_scan()
4109 struct unp_head unp_deadhead; /* List of potentially-dead sockets. */ in unp_gc()
4124 KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0, in unp_gc()
4126 __func__, unp, (unsigned int)unp->unp_gcflag)); in unp_gc()
4128 f = unp->unp_file; in unp_gc()
4137 if (f != NULL && unp->unp_msgcount != 0 && in unp_gc()
4138 refcount_load(&f->f_count) == unp->unp_msgcount) { in unp_gc()
4140 unp->unp_gcflag |= UNPGC_DEAD; in unp_gc()
4141 unp->unp_gcrefs = unp->unp_msgcount; in unp_gc()
4156 * If a socket still has a non-negative refcount, it cannot be in a in unp_gc()
4164 if (unp->unp_gcrefs > 0) { in unp_gc()
4165 unp->unp_gcflag &= ~UNPGC_DEAD; in unp_gc()
4170 unp_unreachable--; in unp_gc()
4196 KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0, in unp_gc()
4198 unp->unp_gcflag &= ~UNPGC_DEAD; in unp_gc()
4199 f = unp->unp_file; in unp_gc()
4200 if (unp->unp_msgcount == 0 || f == NULL || in unp_gc()
4201 refcount_load(&f->f_count) != unp->unp_msgcount || in unp_gc()
4218 so = unref[i]->f_data; in unp_gc()
4220 CURVNET_SET(so->so_vnet); in unp_gc()
4251 unp->unp_gcflag |= UNPGC_IGNORE_RIGHTS; in unp_dispose()
4260 switch (so->so_type) { in unp_dispose()
4262 while ((sb = TAILQ_FIRST(&so->so_rcv.uxdg_conns)) != NULL) { in unp_dispose()
4263 STAILQ_CONCAT(&so->so_rcv.uxdg_mb, &sb->uxdg_mb); in unp_dispose()
4264 TAILQ_REMOVE(&so->so_rcv.uxdg_conns, sb, uxdg_clist); in unp_dispose()
4266 sb->uxdg_cc = sb->uxdg_ctl = sb->uxdg_mbcnt = 0; in unp_dispose()
4268 sb = &so->so_rcv; in unp_dispose()
4269 if (sb->uxdg_peeked != NULL) { in unp_dispose()
4270 STAILQ_INSERT_HEAD(&sb->uxdg_mb, sb->uxdg_peeked, in unp_dispose()
4272 sb->uxdg_peeked = NULL; in unp_dispose()
4274 m = STAILQ_FIRST(&sb->uxdg_mb); in unp_dispose()
4275 STAILQ_INIT(&sb->uxdg_mb); in unp_dispose()
4279 sb = &so->so_rcv; in unp_dispose()
4280 m = STAILQ_FIRST(&sb->uxst_mbq); in unp_dispose()
4281 STAILQ_INIT(&sb->uxst_mbq); in unp_dispose()
4282 sb->sb_acc = sb->sb_ccc = sb->sb_ctl = sb->sb_mbcnt = 0; in unp_dispose()
4287 if (sb->uxst_fnrdy != NULL) { in unp_dispose()
4290 while (m != NULL && m->m_flags & M_NOTREADY) in unp_dispose()
4291 m = m->m_next; in unp_dispose()
4292 for (prev = n = m; n != NULL; n = n->m_next) { in unp_dispose()
4293 if (n->m_flags & M_NOTREADY) in unp_dispose()
4294 prev->m_next = n->m_next; in unp_dispose()
4298 sb->uxst_fnrdy = NULL; in unp_dispose()
4318 sb->sb_state |= SBS_CANTRCVMORE; in unp_dispose()
4319 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, in unp_dispose()
4334 struct cmsghdr *cm; in unp_scan() local
4339 for (m = m0; m; m = m->m_next) { in unp_scan()
4340 if (m->m_type != MT_CONTROL) in unp_scan()
4343 cm = mtod(m, struct cmsghdr *); in unp_scan()
4344 clen = m->m_len; in unp_scan()
4346 while (cm != NULL) { in unp_scan()
4347 if (sizeof(*cm) > clen || cm->cmsg_len > clen) in unp_scan()
4350 data = CMSG_DATA(cm); in unp_scan()
4351 datalen = (caddr_t)cm + cm->cmsg_len in unp_scan()
4352 - (caddr_t)data; in unp_scan()
4354 if (cm->cmsg_level == SOL_SOCKET && in unp_scan()
4355 cm->cmsg_type == SCM_RIGHTS) { in unp_scan()
4361 clen -= CMSG_SPACE(datalen); in unp_scan()
4362 cm = (struct cmsghdr *) in unp_scan()
4363 ((caddr_t)cm + CMSG_SPACE(datalen)); in unp_scan()
4366 cm = NULL; in unp_scan()
4370 m0 = m0->m_nextpkt; in unp_scan()
4474 * A helper function called by VFS before socket-type vnode reclamation.
4486 KASSERT(vp->v_type == VSOCK, in vfs_unp_reclaim()
4487 ("vfs_unp_reclaim: vp->v_type != VSOCK")); in vfs_unp_reclaim()
4496 if (unp->unp_vnode == vp) { in vfs_unp_reclaim()
4498 unp->unp_vnode = NULL; in vfs_unp_reclaim()
4553 xu->cr_version, xu->cr_uid, xu->cr_pid, xu->cr_ngroups); in db_print_xucred()
4557 for (i = 0; i < xu->cr_ngroups; i++) { in db_print_xucred()
4558 db_printf("%s%u", comma ? ", " : "", xu->cr_groups[i]); in db_print_xucred()
4593 db_printf("unp_socket: %p unp_vnode: %p\n", unp->unp_socket, in DB_SHOW_COMMAND()
4594 unp->unp_vnode); in DB_SHOW_COMMAND()
4596 db_printf("unp_ino: %ju unp_conn: %p\n", (uintmax_t)unp->unp_ino, in DB_SHOW_COMMAND()
4597 unp->unp_conn); in DB_SHOW_COMMAND()
4600 db_print_unprefs(2, &unp->unp_refs); in DB_SHOW_COMMAND()
4603 db_printf("unp_addr: %p\n", unp->unp_addr); in DB_SHOW_COMMAND()
4606 (unsigned long long)unp->unp_gencnt); in DB_SHOW_COMMAND()
4608 db_printf("unp_flags: %x (", unp->unp_flags); in DB_SHOW_COMMAND()
4609 db_print_unpflags(unp->unp_flags); in DB_SHOW_COMMAND()
4613 db_print_xucred(2, &unp->unp_peercred); in DB_SHOW_COMMAND()
4615 db_printf("unp_refcount: %u\n", unp->unp_refcount); in DB_SHOW_COMMAND()