Lines Matching full:so
166 static int soreceive_generic_locked(struct socket *so,
169 static int soreceive_rcvoob(struct socket *so, struct uio *uio,
171 static int soreceive_stream_locked(struct socket *so, struct sockbuf *sb,
174 static int sosend_generic_locked(struct socket *so, struct sockaddr *addr,
212 #define VNET_SO_ASSERT(so) \ argument
214 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
258 * The purpose of the UINT_MAX / 3 limit, is so that the formula in sysctl_somaxconn()
348 static int so_unsplice(struct socket *so, bool timeout);
598 * held, so we can only trylock the socket buffers; if this fails, we fall back
760 * of the change so that they can update their dependent limits as required.
801 struct socket *so; in soalloc() local
803 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); in soalloc()
804 if (so == NULL) in soalloc()
807 if (mac_socket_init(so, M_NOWAIT) != 0) { in soalloc()
808 uma_zfree(socket_zone, so); in soalloc()
812 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { in soalloc()
813 uma_zfree(socket_zone, so); in soalloc()
820 * a feature to change class of an existing lock, so we use DUPOK. in soalloc()
822 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); in soalloc()
823 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF); in soalloc()
824 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF); in soalloc()
825 so->so_rcv.sb_sel = &so->so_rdsel; in soalloc()
826 so->so_snd.sb_sel = &so->so_wrsel; in soalloc()
827 sx_init(&so->so_snd_sx, "so_snd_sx"); in soalloc()
828 sx_init(&so->so_rcv_sx, "so_rcv_sx"); in soalloc()
829 TAILQ_INIT(&so->so_snd.sb_aiojobq); in soalloc()
830 TAILQ_INIT(&so->so_rcv.sb_aiojobq); in soalloc()
831 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); in soalloc()
832 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); in soalloc()
834 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", in soalloc()
835 __func__, __LINE__, so)); in soalloc()
836 so->so_vnet = vnet; in soalloc()
840 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { in soalloc()
842 uma_zfree(socket_zone, so); in soalloc()
847 so->so_gencnt = ++so_gencnt; in soalloc()
854 return (so); in soalloc()
863 sodealloc(struct socket *so) in sodealloc() argument
866 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); in sodealloc()
867 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); in sodealloc()
870 so->so_gencnt = ++so_gencnt; in sodealloc()
873 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", in sodealloc()
874 __func__, __LINE__, so)); in sodealloc()
875 so->so_vnet->vnet_sockcnt--; in sodealloc()
879 mac_socket_destroy(so); in sodealloc()
882 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); in sodealloc()
885 khelp_destroy_osd(&so->osd); in sodealloc()
886 if (SOLISTENING(so)) { in sodealloc()
887 if (so->sol_accept_filter != NULL) in sodealloc()
888 accept_filt_setopt(so, NULL); in sodealloc()
890 if (so->so_rcv.sb_hiwat) in sodealloc()
891 (void)chgsbsize(so->so_cred->cr_uidinfo, in sodealloc()
892 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); in sodealloc()
893 if (so->so_snd.sb_hiwat) in sodealloc()
894 (void)chgsbsize(so->so_cred->cr_uidinfo, in sodealloc()
895 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); in sodealloc()
896 sx_destroy(&so->so_snd_sx); in sodealloc()
897 sx_destroy(&so->so_rcv_sx); in sodealloc()
898 mtx_destroy(&so->so_snd_mtx); in sodealloc()
899 mtx_destroy(&so->so_rcv_mtx); in sodealloc()
901 crfree(so->so_cred); in sodealloc()
902 mtx_destroy(&so->so_lock); in sodealloc()
903 uma_zfree(socket_zone, so); in sodealloc()
915 struct socket *so; in socreate() local
952 so = soalloc(CRED_TO_VNET(cred)); in socreate()
953 if (so == NULL) in socreate()
956 so->so_type = type; in socreate()
957 so->so_cred = crhold(cred); in socreate()
961 so->so_fibnum = td->td_proc->p_fibnum; in socreate()
963 so->so_fibnum = 0; in socreate()
964 so->so_proto = prp; in socreate()
966 mac_socket_create(cred, so); in socreate()
968 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, in socreate()
970 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, in socreate()
973 so->so_snd.sb_mtx = &so->so_snd_mtx; in socreate()
974 so->so_rcv.sb_mtx = &so->so_rcv_mtx; in socreate()
980 CURVNET_SET(so->so_vnet); in socreate()
981 error = prp->pr_attach(so, proto, td); in socreate()
984 sodealloc(so); in socreate()
987 soref(so); in socreate()
988 *aso = so; in socreate()
1021 struct socket *so; local
1159 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL",
1161 so = soalloc(head->so_vnet);
1162 if (so == NULL) {
1168 so->so_listen = head;
1169 so->so_type = head->so_type;
1185 so->so_options = head->so_options & (SO_ACCEPTFILTER | SO_KEEPALIVE |
1187 so->so_linger = head->so_linger;
1188 so->so_state = head->so_state;
1189 so->so_fibnum = head->so_fibnum;
1190 so->so_proto = head->so_proto;
1191 so->so_cred = crhold(head->so_cred);
1194 if (hhook_run_socket(so, head, HHOOK_SOCKET_NEWCONN)) {
1195 sodealloc(so);
1202 mac_socket_newconn(head, so);
1204 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
1206 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
1209 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) {
1210 sodealloc(so);
1215 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat;
1216 so->so_snd.sb_lowat = head->sol_sbsnd_lowat;
1217 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo;
1218 so->so_snd.sb_timeo = head->sol_sbsnd_timeo;
1219 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE;
1220 so->so_snd.sb_flags = head->sol_sbsnd_flags & SB_AUTOSIZE;
1221 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
1222 so->so_snd.sb_mtx = &so->so_snd_mtx;
1223 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
1226 return (so);
1233 struct socket *so; local
1235 if ((so = solisten_clone(head)) == NULL)
1238 if (so->so_proto->pr_attach(so, 0, NULL) != 0) {
1239 sodealloc(so);
1245 (void)solisten_enqueue(so, connstatus);
1247 return (so);
1257 solisten_enqueue(struct socket *so, int connstatus) argument
1259 struct socket *head = so->so_listen;
1261 MPASS(refcount_load(&so->so_count) == 0);
1262 refcount_init(&so->so_count, 1);
1267 so->so_state |= connstatus;
1270 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
1271 so->so_qstate = SQ_COMP;
1297 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list);
1298 so->so_qstate = SQ_INCOMP;
1315 struct socket *so; local
1319 so = soalloc(head->so_vnet);
1320 if (so == NULL) {
1326 so->so_type = head->so_type;
1327 so->so_options = head->so_options;
1328 so->so_linger = head->so_linger;
1329 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED;
1330 so->so_fibnum = head->so_fibnum;
1331 so->so_proto = head->so_proto;
1332 so->so_cred = crhold(head->so_cred);
1334 mac_socket_newconn(head, so);
1336 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
1338 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
1341 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
1342 sodealloc(so);
1347 if ((*so->so_proto->pr_attach)(so, 0, NULL)) {
1348 sodealloc(so);
1353 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
1354 so->so_snd.sb_lowat = head->so_snd.sb_lowat;
1355 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
1356 so->so_snd.sb_timeo = head->so_snd.sb_timeo;
1357 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE;
1358 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE;
1359 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
1360 so->so_snd.sb_mtx = &so->so_snd_mtx;
1361 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
1364 soref(so);
1366 return (so);
1371 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) argument
1375 CURVNET_SET(so->so_vnet);
1376 error = so->so_proto->pr_bind(so, nam, td);
1382 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) argument
1386 CURVNET_SET(so->so_vnet);
1387 error = so->so_proto->pr_bindat(fd, so, nam, td);
1397 * socket-layer listen state. Call backs are used so that the protocol can
1405 solisten(struct socket *so, int backlog, struct thread *td) argument
1409 CURVNET_SET(so->so_vnet);
1410 error = so->so_proto->pr_listen(so, backlog, td);
1420 solisten_proto_check(struct socket *so) argument
1422 SOCK_LOCK_ASSERT(so);
1424 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1429 * Sleeping is not permitted here, so simply fail if userspace is
1434 if (!sx_try_xlock(&so->so_snd_sx))
1436 if (!sx_try_xlock(&so->so_rcv_sx)) {
1437 sx_xunlock(&so->so_snd_sx);
1440 mtx_lock(&so->so_snd_mtx);
1441 mtx_lock(&so->so_rcv_mtx);
1444 if (!SOLISTENING(so)) {
1448 ktls = so->so_snd.sb_tls_info != NULL ||
1449 so->so_rcv.sb_tls_info != NULL;
1454 (so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 ||
1455 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0) {
1456 solisten_proto_abort(so);
1468 solisten_proto_abort(struct socket *so) argument
1470 mtx_unlock(&so->so_snd_mtx);
1471 mtx_unlock(&so->so_rcv_mtx);
1472 sx_xunlock(&so->so_snd_sx);
1473 sx_xunlock(&so->so_rcv_sx);
1477 solisten_proto(struct socket *so, int backlog) argument
1484 SOCK_LOCK_ASSERT(so);
1485 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1487 ("%s: bad socket state %p", __func__, so));
1489 if (SOLISTENING(so))
1495 sbrcv_lowat = so->so_rcv.sb_lowat;
1496 sbsnd_lowat = so->so_snd.sb_lowat;
1497 sbrcv_hiwat = so->so_rcv.sb_hiwat;
1498 sbsnd_hiwat = so->so_snd.sb_hiwat;
1499 sbrcv_flags = so->so_rcv.sb_flags;
1500 sbsnd_flags = so->so_snd.sb_flags;
1501 sbrcv_timeo = so->so_rcv.sb_timeo;
1502 sbsnd_timeo = so->so_snd.sb_timeo;
1505 mac_socketpeer_label_free(so->so_peerlabel);
1508 if (!(so->so_proto->pr_flags & PR_SOCKBUF)) {
1509 sbdestroy(so, SO_SND);
1510 sbdestroy(so, SO_RCV);
1514 bzero(&so->so_rcv,
1518 so->sol_sbrcv_lowat = sbrcv_lowat;
1519 so->sol_sbsnd_lowat = sbsnd_lowat;
1520 so->sol_sbrcv_hiwat = sbrcv_hiwat;
1521 so->sol_sbsnd_hiwat = sbsnd_hiwat;
1522 so->sol_sbrcv_flags = sbrcv_flags;
1523 so->sol_sbsnd_flags = sbsnd_flags;
1524 so->sol_sbrcv_timeo = sbrcv_timeo;
1525 so->sol_sbsnd_timeo = sbsnd_timeo;
1527 so->sol_qlen = so->sol_incqlen = 0;
1528 TAILQ_INIT(&so->sol_incomp);
1529 TAILQ_INIT(&so->sol_comp);
1531 so->sol_accept_filter = NULL;
1532 so->sol_accept_filter_arg = NULL;
1533 so->sol_accept_filter_str = NULL;
1535 so->sol_upcall = NULL;
1536 so->sol_upcallarg = NULL;
1538 so->so_options |= SO_ACCEPTCONN;
1543 so->sol_qlimit = backlog;
1545 mtx_unlock(&so->so_snd_mtx);
1546 mtx_unlock(&so->so_rcv_mtx);
1547 sx_xunlock(&so->so_snd_sx);
1548 sx_xunlock(&so->so_rcv_sx);
1584 struct socket *so; local
1609 so = TAILQ_FIRST(&head->sol_comp);
1610 SOCK_LOCK(so);
1611 KASSERT(so->so_qstate == SQ_COMP,
1612 ("%s: so %p not SQ_COMP", __func__, so));
1614 so->so_qstate = SQ_NONE;
1615 so->so_listen = NULL;
1616 TAILQ_REMOVE(&head->sol_comp, so, so_list);
1618 so->so_state |= (head->so_state & SS_NBIO);
1620 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
1621 SOCK_UNLOCK(so);
1624 *ret = so;
1665 * Splice the output from so to the input of so2.
1668 so_splice(struct socket *so, struct socket *so2, struct splice *splice) argument
1676 if (so->so_proto->pr_protocol != IPPROTO_TCP ||
1679 if (so->so_vnet != so2->so_vnet)
1683 KASSERT(so->so_proto->pr_sosend == sosend_generic,
1690 so->so_splice_sent = 0;
1691 sp->src = so;
1695 SOCK_LOCK(so);
1696 if (SOLISTENING(so))
1698 else if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING)) == 0)
1700 else if (so->so_splice != NULL)
1703 SOCK_UNLOCK(so);
1707 soref(so);
1708 so->so_splice = sp;
1709 SOCK_RECVBUF_LOCK(so);
1710 so->so_rcv.sb_flags |= SB_SPLICED;
1711 SOCK_RECVBUF_UNLOCK(so);
1712 SOCK_UNLOCK(so);
1724 SOCK_LOCK(so);
1725 so->so_splice = NULL;
1726 SOCK_RECVBUF_LOCK(so);
1727 so->so_rcv.sb_flags &= ~SB_SPLICED;
1728 SOCK_RECVBUF_UNLOCK(so);
1729 SOCK_UNLOCK(so);
1730 sorele(so);
1756 so_unsplice(struct socket *so, bool timeout) argument
1763 * First unset SB_SPLICED and hide the splice structure so that
1767 SOCK_LOCK(so);
1768 if (SOLISTENING(so)) {
1769 SOCK_UNLOCK(so);
1772 SOCK_RECVBUF_LOCK(so);
1773 if ((so->so_rcv.sb_flags & SB_SPLICED) == 0) {
1774 SOCK_RECVBUF_UNLOCK(so);
1775 SOCK_UNLOCK(so);
1778 so->so_rcv.sb_flags &= ~SB_SPLICED;
1779 sp = so->so_splice;
1780 so->so_splice = NULL;
1781 SOCK_RECVBUF_UNLOCK(so);
1782 SOCK_UNLOCK(so);
1831 sorwakeup(so);
1832 CURVNET_SET(so->so_vnet);
1833 sorele(so);
1845 sofree(struct socket *so) argument
1847 struct protosw *pr = so->so_proto;
1849 SOCK_LOCK_ASSERT(so);
1850 KASSERT(refcount_load(&so->so_count) == 0,
1851 ("%s: so %p has references", __func__, so));
1852 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE,
1853 ("%s: so %p is on listen queue", __func__, so));
1854 KASSERT(SOLISTENING(so) || (so->so_rcv.sb_flags & SB_SPLICED) == 0,
1855 ("%s: so %p rcvbuf is spliced", __func__, so));
1856 KASSERT(SOLISTENING(so) || (so->so_snd.sb_flags & SB_SPLICED) == 0,
1857 ("%s: so %p sndbuf is spliced", __func__, so));
1858 KASSERT(so->so_splice == NULL && so->so_splice_back == NULL,
1859 ("%s: so %p has spliced data", __func__, so));
1861 SOCK_UNLOCK(so);
1863 if (so->so_dtor != NULL)
1864 so->so_dtor(so);
1866 VNET_SO_ASSERT(so);
1868 pr->pr_detach(so);
1875 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) {
1876 sbdestroy(so, SO_SND);
1877 sbdestroy(so, SO_RCV);
1879 seldrain(&so->so_rdsel);
1880 seldrain(&so->so_wrsel);
1881 knlist_destroy(&so->so_rdsel.si_note);
1882 knlist_destroy(&so->so_wrsel.si_note);
1883 sodealloc(so);
1891 sorele_locked(struct socket *so) argument
1893 SOCK_LOCK_ASSERT(so);
1894 if (refcount_release(&so->so_count))
1895 sofree(so);
1897 SOCK_UNLOCK(so);
1909 soclose(struct socket *so) argument
1915 CURVNET_SET(so->so_vnet);
1916 funsetown(&so->so_sigio);
1917 if (so->so_state & SS_ISCONNECTED) {
1918 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
1919 error = sodisconnect(so);
1927 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) {
1928 if ((so->so_state & SS_ISDISCONNECTING) &&
1929 (so->so_state & SS_NBIO))
1931 while (so->so_state & SS_ISCONNECTED) {
1932 error = tsleep(&so->so_timeo,
1934 so->so_linger * hz);
1942 if (so->so_proto->pr_close != NULL)
1943 so->so_proto->pr_close(so);
1945 SOCK_LOCK(so);
1946 if ((listening = SOLISTENING(so))) {
1950 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list);
1951 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list);
1953 so->sol_qlen = so->sol_incqlen = 0;
1960 last = refcount_release(&so->so_count);
1962 __func__, so));
1965 sorele_locked(so);
1988 * This interface will call into the protocol code, so must not be called
1998 soabort(struct socket *so) argument
2001 VNET_SO_ASSERT(so);
2003 if (so->so_proto->pr_abort != NULL)
2004 so->so_proto->pr_abort(so);
2005 SOCK_LOCK(so);
2006 sorele_locked(so);
2010 soaccept(struct socket *so, struct sockaddr *sa) argument
2017 CURVNET_SET(so->so_vnet);
2018 error = so->so_proto->pr_accept(so, sa);
2020 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto));
2026 sopeeraddr(struct socket *so, struct sockaddr *sa) argument
2035 error = so->so_proto->pr_peeraddr(so, sa);
2037 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto));
2043 sosockaddr(struct socket *so, struct sockaddr *sa) argument
2050 CURVNET_SET(so->so_vnet);
2051 error = so->so_proto->pr_sockaddr(so, sa);
2053 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto));
2060 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) argument
2063 return (soconnectat(AT_FDCWD, so, nam, td));
2067 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) argument
2071 CURVNET_SET(so->so_vnet);
2081 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
2082 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
2083 (error = sodisconnect(so)))) {
2090 so->so_error = 0;
2092 error = so->so_proto->pr_connect(so, nam, td);
2094 error = so->so_proto->pr_connectat(fd, so, nam, td);
2114 sodisconnect(struct socket *so) argument
2118 if ((so->so_state & SS_ISCONNECTED) == 0)
2120 if (so->so_state & SS_ISDISCONNECTING)
2122 VNET_SO_ASSERT(so);
2123 error = so->so_proto->pr_disconnect(so);
2128 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2135 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM"));
2136 KASSERT(so->so_proto->pr_flags & PR_ATOMIC,
2156 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0;
2162 SOCKBUF_LOCK(&so->so_snd);
2163 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2164 SOCKBUF_UNLOCK(&so->so_snd);
2168 if (so->so_error) {
2169 error = so->so_error;
2170 so->so_error = 0;
2171 SOCKBUF_UNLOCK(&so->so_snd);
2174 if ((so->so_state & SS_ISCONNECTED) == 0) {
2180 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
2181 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
2183 SOCKBUF_UNLOCK(&so->so_snd);
2188 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
2192 SOCKBUF_UNLOCK(&so->so_snd);
2201 space = sbspace(&so->so_snd);
2205 SOCKBUF_UNLOCK(&so->so_snd);
2235 SOCK_LOCK(so);
2236 so->so_options |= SO_DONTROUTE;
2237 SOCK_UNLOCK(so);
2247 VNET_SO_ASSERT(so);
2248 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB :
2254 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
2262 SOCK_LOCK(so);
2263 so->so_options &= ~SO_DONTROUTE;
2264 SOCK_UNLOCK(so);
2291 sosend_generic_locked(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2297 int atomic = sosendallatonce(so) || top;
2308 SOCK_IO_SEND_ASSERT_LOCKED(so);
2326 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
2332 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
2333 (so->so_proto->pr_flags & PR_ATOMIC);
2341 tls = ktls_hold(so->so_snd.sb_tls_info);
2368 SOCKBUF_LOCK(&so->so_snd);
2369 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
2370 SOCKBUF_UNLOCK(&so->so_snd);
2374 if (so->so_error) {
2375 error = so->so_error;
2376 so->so_error = 0;
2377 SOCKBUF_UNLOCK(&so->so_snd);
2380 if ((so->so_state & SS_ISCONNECTED) == 0) {
2387 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
2388 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
2390 SOCKBUF_UNLOCK(&so->so_snd);
2395 SOCKBUF_UNLOCK(&so->so_snd);
2396 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
2403 space = sbspace(&so->so_snd);
2406 if ((atomic && resid > so->so_snd.sb_hiwat) ||
2407 clen > so->so_snd.sb_hiwat) {
2408 SOCKBUF_UNLOCK(&so->so_snd);
2413 (atomic || space < so->so_snd.sb_lowat || space < clen)) {
2414 if ((so->so_state & SS_NBIO) ||
2416 SOCKBUF_UNLOCK(&so->so_snd);
2420 error = sbwait(so, SO_SND);
2421 SOCKBUF_UNLOCK(&so->so_snd);
2426 SOCKBUF_UNLOCK(&so->so_snd);
2474 SOCK_LOCK(so);
2475 so->so_options |= SO_DONTROUTE;
2476 SOCK_UNLOCK(so);
2488 VNET_SO_ASSERT(so);
2497 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
2508 error = so->so_proto->pr_send(so, pr_send_flag, top,
2512 SOCK_LOCK(so);
2513 so->so_options &= ~SO_DONTROUTE;
2514 SOCK_UNLOCK(so);
2523 soref(so);
2524 ktls_enqueue(top, so, tls_enq_cnt);
2549 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2554 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags));
2557 error = sosend_generic_locked(so, addr, uio, top, control, flags, td);
2558 SOCK_IO_SEND_UNLOCK(so);
2571 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2576 CURVNET_SET(so->so_vnet);
2577 error = so->so_proto->pr_sosend(so, addr, uio,
2587 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio, argument
2596 CURVNET_SET(so->so_vnet);
2597 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags,
2608 (so->so_proto->pr_flags & PR_ATOMIC) == 0 &&
2614 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 &&
2640 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) argument
2642 struct protosw *pr = so->so_proto;
2647 VNET_SO_ASSERT(so);
2650 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK);
2667 * buffer so that other consumers see the values consistently. 'nextrecord'
2703 * so specifies, followed by an optional mbuf or mbufs containing ancillary
2716 soreceive_generic_locked(struct socket *so, struct sockaddr **psa, argument
2722 struct protosw *pr = so->so_proto;
2728 SOCK_IO_RECV_ASSERT_LOCKED(so);
2739 SOCKBUF_LOCK(&so->so_rcv);
2740 m = so->so_rcv.sb_mb;
2748 sbavail(&so->so_rcv) < uio->uio_resid) &&
2749 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat &&
2751 KASSERT(m != NULL || !sbavail(&so->so_rcv),
2753 m, sbavail(&so->so_rcv)));
2754 if (so->so_error || so->so_rerror) {
2757 if (so->so_error)
2758 error = so->so_error;
2760 error = so->so_rerror;
2762 if (so->so_error)
2763 so->so_error = 0;
2765 so->so_rerror = 0;
2767 SOCKBUF_UNLOCK(&so->so_rcv);
2770 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2771 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
2775 else if (so->so_rcv.sb_tlsdcc == 0 &&
2776 so->so_rcv.sb_tlscc == 0) {
2780 SOCKBUF_UNLOCK(&so->so_rcv);
2786 m = so->so_rcv.sb_mb;
2789 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED |
2791 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
2792 SOCKBUF_UNLOCK(&so->so_rcv);
2797 SOCKBUF_UNLOCK(&so->so_rcv);
2800 if ((so->so_state & SS_NBIO) ||
2802 SOCKBUF_UNLOCK(&so->so_rcv);
2806 SBLASTRECORDCHK(&so->so_rcv);
2807 SBLASTMBUFCHK(&so->so_rcv);
2808 error = sbwait(so, SO_RCV);
2809 SOCKBUF_UNLOCK(&so->so_rcv);
2830 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2833 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb"));
2834 SBLASTRECORDCHK(&so->so_rcv);
2835 SBLASTMBUFCHK(&so->so_rcv);
2847 sbfree(&so->so_rcv, m);
2848 so->so_rcv.sb_mb = m_free(m);
2849 m = so->so_rcv.sb_mb;
2850 sockbuf_pushsync(&so->so_rcv, nextrecord);
2882 SOCKBUF_UNLOCK(&so->so_rcv);
2899 sbfree(&so->so_rcv, m);
2900 so->so_rcv.sb_mb = m->m_next;
2904 m = so->so_rcv.sb_mb;
2908 sockbuf_pushsync(&so->so_rcv, nextrecord);
2913 SOCKBUF_UNLOCK(&so->so_rcv);
2914 VNET_SO_ASSERT(so);
2917 SOCKBUF_LOCK(&so->so_rcv);
2929 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
2931 nextrecord = so->so_rcv.sb_mb;
2939 KASSERT(so->so_rcv.sb_mb == m,
2941 KASSERT(so->so_rcv.sb_lastrecord == m,
2950 KASSERT(so->so_rcv.sb_mb == nextrecord,
2952 if (so->so_rcv.sb_mb == NULL) {
2953 KASSERT(so->so_rcv.sb_lastrecord == NULL,
2958 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2959 SBLASTRECORDCHK(&so->so_rcv);
2960 SBLASTMBUFCHK(&so->so_rcv);
2965 * used to store the type of any mbuf reads that have happened so far
2978 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2987 so->so_rcv.sb_state &= ~SBS_RCVATMARK;
2989 if (so->so_oobmark && len > so->so_oobmark - offset)
2990 len = so->so_oobmark - offset;
3001 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3002 SBLASTRECORDCHK(&so->so_rcv);
3003 SBLASTMBUFCHK(&so->so_rcv);
3004 SOCKBUF_UNLOCK(&so->so_rcv);
3011 SOCKBUF_LOCK(&so->so_rcv);
3015 * from the record, so it is necessary to
3023 (void)sbdroprecord_locked(&so->so_rcv);
3024 SOCKBUF_UNLOCK(&so->so_rcv);
3029 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3038 sbfree(&so->so_rcv, m);
3043 so->so_rcv.sb_mb = m = m->m_next;
3046 so->so_rcv.sb_mb = m_free(m);
3047 m = so->so_rcv.sb_mb;
3049 sockbuf_pushsync(&so->so_rcv, nextrecord);
3050 SBLASTRECORDCHK(&so->so_rcv);
3051 SBLASTMBUFCHK(&so->so_rcv);
3075 SOCKBUF_UNLOCK(&so->so_rcv);
3078 SOCKBUF_LOCK(&so->so_rcv);
3081 sbcut_locked(&so->so_rcv, len);
3084 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3085 if (so->so_oobmark) {
3087 so->so_oobmark -= len;
3088 if (so->so_oobmark == 0) {
3089 so->so_rcv.sb_state |= SBS_RCVATMARK;
3094 if (offset == so->so_oobmark)
3108 !sosendallatonce(so) && nextrecord == NULL) {
3109 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3110 if (so->so_error || so->so_rerror ||
3111 so->so_rcv.sb_state & SBS_CANTRCVMORE)
3118 SOCKBUF_UNLOCK(&so->so_rcv);
3119 VNET_SO_ASSERT(so);
3120 pr->pr_rcvd(so, flags);
3121 SOCKBUF_LOCK(&so->so_rcv);
3122 if (__predict_false(so->so_rcv.sb_mb == NULL &&
3123 (so->so_error || so->so_rerror ||
3124 so->so_rcv.sb_state & SBS_CANTRCVMORE)))
3127 SBLASTRECORDCHK(&so->so_rcv);
3128 SBLASTMBUFCHK(&so->so_rcv);
3133 if (so->so_rcv.sb_mb == NULL) {
3134 error = sbwait(so, SO_RCV);
3136 SOCKBUF_UNLOCK(&so->so_rcv);
3140 m = so->so_rcv.sb_mb;
3146 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3152 (void) sbdroprecord_locked(&so->so_rcv);
3161 so->so_rcv.sb_mb = nextrecord;
3162 if (so->so_rcv.sb_mb == NULL) {
3163 so->so_rcv.sb_mbtail = NULL;
3164 so->so_rcv.sb_lastrecord = NULL;
3166 so->so_rcv.sb_lastrecord = nextrecord;
3168 SBLASTRECORDCHK(&so->so_rcv);
3169 SBLASTMBUFCHK(&so->so_rcv);
3177 SOCKBUF_UNLOCK(&so->so_rcv);
3178 VNET_SO_ASSERT(so);
3179 pr->pr_rcvd(so, flags);
3180 SOCKBUF_LOCK(&so->so_rcv);
3183 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3185 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
3186 SOCKBUF_UNLOCK(&so->so_rcv);
3189 SOCKBUF_UNLOCK(&so->so_rcv);
3198 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3210 return (soreceive_rcvoob(so, uio, flags));
3217 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
3220 error = soreceive_generic_locked(so, psa, uio, mp, controlp, flagsp);
3221 SOCK_IO_RECV_UNLOCK(so);
3229 soreceive_stream_locked(struct socket *so, struct sockbuf *sb, argument
3236 SOCK_IO_RECV_ASSERT_LOCKED(so);
3245 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
3251 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3254 if (so->so_error) {
3259 error = so->so_error;
3261 so->so_error = 0;
3275 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) {
3282 ((so->so_state & SS_NBIO) ||
3299 error = sbwait(so, SO_RCV);
3305 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3377 if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
3381 VNET_SO_ASSERT(so);
3382 so->so_proto->pr_rcvd(so, flags);
3401 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3407 sb = &so->so_rcv;
3410 if (so->so_type != SOCK_STREAM)
3421 return (soreceive_rcvoob(so, uio, flags));
3434 return (soreceive_generic(so, psa, uio, mp0, controlp,
3442 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
3447 SOCK_IO_RECV_UNLOCK(so);
3448 return (soreceive_generic(so, psa, uio, mp0, controlp,
3452 error = soreceive_stream_locked(so, sb, psa, uio, mp0, controlp, flags);
3453 SOCK_IO_RECV_UNLOCK(so);
3464 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3470 struct protosw *pr = so->so_proto;
3487 return (soreceive_generic(so, psa, uio, mp0, controlp,
3496 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0,
3498 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0,
3504 SOCKBUF_LOCK(&so->so_rcv);
3505 while ((m = so->so_rcv.sb_mb) == NULL) {
3506 KASSERT(sbavail(&so->so_rcv) == 0,
3508 sbavail(&so->so_rcv)));
3509 if (so->so_error) {
3510 error = so->so_error;
3511 so->so_error = 0;
3512 SOCKBUF_UNLOCK(&so->so_rcv);
3515 if (so->so_rcv.sb_state & SBS_CANTRCVMORE ||
3517 SOCKBUF_UNLOCK(&so->so_rcv);
3520 if ((so->so_state & SS_NBIO) ||
3522 SOCKBUF_UNLOCK(&so->so_rcv);
3525 SBLASTRECORDCHK(&so->so_rcv);
3526 SBLASTMBUFCHK(&so->so_rcv);
3527 error = sbwait(so, SO_RCV);
3529 SOCKBUF_UNLOCK(&so->so_rcv);
3533 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
3537 SBLASTRECORDCHK(&so->so_rcv);
3538 SBLASTMBUFCHK(&so->so_rcv);
3541 KASSERT(so->so_rcv.sb_lastrecord == m,
3545 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord,
3551 so->so_rcv.sb_mb = NULL;
3552 sockbuf_pushsync(&so->so_rcv, nextrecord);
3558 sbfree(&so->so_rcv, m2);
3563 SBLASTRECORDCHK(&so->so_rcv);
3564 SBLASTMBUFCHK(&so->so_rcv);
3565 SOCKBUF_UNLOCK(&so->so_rcv);
3643 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, argument
3648 CURVNET_SET(so->so_vnet);
3649 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp);
3655 soshutdown(struct socket *so, enum shutdown_how how) argument
3659 CURVNET_SET(so->so_vnet);
3660 error = so->so_proto->pr_shutdown(so, how);
3670 sorflush(struct socket *so) argument
3674 VNET_SO_ASSERT(so);
3688 socantrcvmore(so);
3690 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR);
3692 KASSERT(SOLISTENING(so),
3693 ("%s: soiolock(%p) failed", __func__, so));
3697 sbrelease(so, SO_RCV);
3698 SOCK_IO_RECV_UNLOCK(so);
3703 sosetfib(struct socket *so, int fibnum) argument
3708 SOCK_LOCK(so);
3709 so->so_fibnum = fibnum;
3710 SOCK_UNLOCK(so);
3721 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) argument
3724 .so = so,
3730 CURVNET_SET(so->so_vnet);
3731 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd);
3775 so_setsockopt(struct socket *so, int level, int optname, void *optval, argument
3786 return (sosetopt(so, &sopt));
3790 sosetopt(struct socket *so, struct sockopt *sopt) argument
3801 CURVNET_SET(so->so_vnet);
3804 if (so->so_proto->pr_ctloutput != NULL)
3805 error = (*so->so_proto->pr_ctloutput)(so, sopt);
3811 error = accept_filt_setopt(so, sopt);
3826 SOCK_LOCK(so);
3827 so->so_linger = l.l_linger;
3829 so->so_options |= SO_LINGER;
3831 so->so_options &= ~SO_LINGER;
3832 SOCK_UNLOCK(so);
3854 SOCK_LOCK(so);
3856 so->so_options |= sopt->sopt_name;
3858 so->so_options &= ~sopt->sopt_name;
3859 SOCK_UNLOCK(so);
3863 error = so->so_proto->pr_ctloutput(so, sopt);
3871 so->so_user_cookie = val32;
3878 error = so->so_proto->pr_setsbopt(so, sopt);
3908 SOCK_LOCK(so);
3910 (SOLISTENING(so) ? &so->sol_sbsnd_timeo :
3911 &so->so_snd.sb_timeo) :
3912 (SOLISTENING(so) ? &so->sol_sbrcv_timeo :
3913 &so->so_rcv.sb_timeo);
3915 SOCK_UNLOCK(so);
3925 so, &extmac);
3940 so->so_ts_clock = optval;
3948 so->so_max_pacing_rate = val32;
4000 error = so_splice(so, so2, &splice);
4003 error = so_unsplice(so, false);
4010 error = hhook_run_socket(so, sopt,
4017 if (error == 0 && so->so_proto->pr_ctloutput != NULL)
4018 (void)(*so->so_proto->pr_ctloutput)(so, sopt);
4056 sogetopt(struct socket *so, struct sockopt *sopt) argument
4065 CURVNET_SET(so->so_vnet);
4068 if (so->so_proto->pr_ctloutput != NULL)
4069 error = (*so->so_proto->pr_ctloutput)(so, sopt);
4077 error = accept_filt_getopt(so, sopt);
4081 SOCK_LOCK(so);
4082 l.l_onoff = so->so_options & SO_LINGER;
4083 l.l_linger = so->so_linger;
4084 SOCK_UNLOCK(so);
4104 optval = so->so_options & sopt->sopt_name;
4110 SOCK_LOCK(so);
4111 optval = so->so_fibnum;
4112 SOCK_UNLOCK(so);
4116 optval = so->so_proto->pr_domain->dom_family;
4120 optval = so->so_type;
4124 optval = so->so_proto->pr_protocol;
4128 SOCK_LOCK(so);
4129 if (so->so_error) {
4130 optval = so->so_error;
4131 so->so_error = 0;
4133 optval = so->so_rerror;
4134 so->so_rerror = 0;
4136 SOCK_UNLOCK(so);
4140 SOCK_LOCK(so);
4141 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat :
4142 so->so_snd.sb_hiwat;
4143 SOCK_UNLOCK(so);
4147 SOCK_LOCK(so);
4148 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat :
4149 so->so_rcv.sb_hiwat;
4150 SOCK_UNLOCK(so);
4154 SOCK_LOCK(so);
4155 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat :
4156 so->so_snd.sb_lowat;
4157 SOCK_UNLOCK(so);
4161 SOCK_LOCK(so);
4162 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat :
4163 so->so_rcv.sb_lowat;
4164 SOCK_UNLOCK(so);
4169 SOCK_LOCK(so);
4171 (SOLISTENING(so) ? so->sol_sbsnd_timeo :
4172 so->so_snd.sb_timeo) :
4173 (SOLISTENING(so) ? so->sol_sbrcv_timeo :
4174 so->so_rcv.sb_timeo));
4175 SOCK_UNLOCK(so);
4195 so, &extmac);
4211 sopt->sopt_td->td_ucred, so, &extmac);
4221 SOCK_LOCK(so);
4222 optval = SOLISTENING(so) ? so->sol_qlimit : 0;
4223 SOCK_UNLOCK(so);
4227 SOCK_LOCK(so);
4228 optval = SOLISTENING(so) ? so->sol_qlen : 0;
4229 SOCK_UNLOCK(so);
4233 SOCK_LOCK(so);
4234 optval = SOLISTENING(so) ? so->sol_incqlen : 0;
4235 SOCK_UNLOCK(so);
4239 optval = so->so_ts_clock;
4243 optval = so->so_max_pacing_rate;
4258 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT);
4261 SOCK_LOCK(so);
4262 if (SOLISTENING(so)) {
4265 n = so->so_splice_sent;
4267 SOCK_UNLOCK(so);
4268 SOCK_IO_RECV_UNLOCK(so);
4276 error = hhook_run_socket(so, sopt,
4404 sohasoutofband(struct socket *so) argument
4407 if (so->so_sigio != NULL)
4408 pgsigio(&so->so_sigio, SIGURG, 0);
4409 selwakeuppri(&so->so_rdsel, PSOCK);
4413 sopoll_generic(struct socket *so, int events, struct thread *td) argument
4417 SOCK_LOCK(so);
4418 if (SOLISTENING(so)) {
4421 else if (!TAILQ_EMPTY(&so->sol_comp))
4423 else if ((events & POLLINIGNEOF) == 0 && so->so_error)
4426 selrecord(td, &so->so_rdsel);
4431 SOCK_SENDBUF_LOCK(so);
4432 SOCK_RECVBUF_LOCK(so);
4434 if (soreadabledata(so) && !isspliced(so))
4437 if (sowriteable(so) && !issplicedback(so))
4440 if (so->so_oobmark ||
4441 (so->so_rcv.sb_state & SBS_RCVATMARK))
4444 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
4446 if (so->so_snd.sb_state & SBS_CANTSENDMORE)
4450 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
4455 selrecord(td, &so->so_rdsel);
4456 so->so_rcv.sb_flags |= SB_SEL;
4459 selrecord(td, &so->so_wrsel);
4460 so->so_snd.sb_flags |= SB_SEL;
4463 SOCK_RECVBUF_UNLOCK(so);
4464 SOCK_SENDBUF_UNLOCK(so);
4466 SOCK_UNLOCK(so);
4473 struct socket *so = kn->kn_fp->f_data; local
4481 knl = &so->so_rdsel.si_note;
4482 sb = &so->so_rcv;
4487 knl = &so->so_wrsel.si_note;
4488 sb = &so->so_snd;
4493 knl = &so->so_wrsel.si_note;
4494 sb = &so->so_snd;
4501 SOCK_LOCK(so);
4502 if (SOLISTENING(so)) {
4505 SOCK_BUF_LOCK(so, which);
4508 SOCK_BUF_UNLOCK(so, which);
4510 SOCK_UNLOCK(so);
4517 struct socket *so = kn->kn_fp->f_data; local
4519 so_rdknl_lock(so);
4520 knlist_remove(&so->so_rdsel.si_note, kn, 1);
4521 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note))
4522 so->so_rcv.sb_flags &= ~SB_KNOTE;
4523 so_rdknl_unlock(so);
4530 struct socket *so; local
4532 so = kn->kn_fp->f_data;
4534 if (SOLISTENING(so)) {
4535 SOCK_LOCK_ASSERT(so);
4536 kn->kn_data = so->sol_qlen;
4537 if (so->so_error) {
4539 kn->kn_fflags = so->so_error;
4542 return (!TAILQ_EMPTY(&so->sol_comp));
4545 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0)
4548 SOCK_RECVBUF_LOCK_ASSERT(so);
4550 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl;
4551 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
4553 kn->kn_fflags = so->so_error;
4555 } else if (so->so_error || so->so_rerror)
4561 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat)
4566 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD));
4575 struct socket *so = kn->kn_fp->f_data; local
4577 so_wrknl_lock(so);
4578 knlist_remove(&so->so_wrsel.si_note, kn, 1);
4579 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note))
4580 so->so_snd.sb_flags &= ~SB_KNOTE;
4581 so_wrknl_unlock(so);
4588 struct socket *so; local
4590 so = kn->kn_fp->f_data;
4592 if (SOLISTENING(so))
4595 SOCK_SENDBUF_LOCK_ASSERT(so);
4596 kn->kn_data = sbspace(&so->so_snd);
4599 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE);
4602 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
4604 kn->kn_fflags = so->so_error;
4606 } else if (so->so_error) /* temporary udp error */
4608 else if (((so->so_state & SS_ISCONNECTED) == 0) &&
4609 (so->so_proto->pr_flags & PR_CONNREQUIRED))
4614 return (kn->kn_data >= so->so_snd.sb_lowat);
4620 struct socket *so; local
4622 so = kn->kn_fp->f_data;
4624 if (SOLISTENING(so))
4627 SOCK_SENDBUF_LOCK_ASSERT(so);
4628 kn->kn_data = sbused(&so->so_snd);
4637 socheckuid(struct socket *so, uid_t uid) argument
4640 if (so == NULL)
4642 if (so->so_cred->cr_uid != uid)
4680 soisconnecting(struct socket *so) argument
4683 SOCK_LOCK(so);
4684 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
4685 so->so_state |= SS_ISCONNECTING;
4686 SOCK_UNLOCK(so);
4690 soisconnected(struct socket *so) argument
4694 SOCK_LOCK(so);
4695 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING);
4696 so->so_state |= SS_ISCONNECTED;
4698 if (so->so_qstate == SQ_INCOMP) {
4699 struct socket *head = so->so_listen;
4702 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so));
4712 SOCK_UNLOCK(so);
4714 SOCK_LOCK(so);
4715 if (__predict_false(head != so->so_listen)) {
4721 SOCK_UNLOCK(so);
4730 if ((so->so_options & SO_ACCEPTFILTER) == 0) {
4731 TAILQ_REMOVE(&head->sol_incomp, so, so_list);
4733 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
4735 so->so_qstate = SQ_COMP;
4736 SOCK_UNLOCK(so);
4739 SOCK_RECVBUF_LOCK(so);
4740 soupcall_set(so, SO_RCV,
4743 so->so_options &= ~SO_ACCEPTFILTER;
4744 ret = head->sol_accept_filter->accf_callback(so,
4747 soupcall_clear(so, SO_RCV);
4748 SOCK_RECVBUF_UNLOCK(so);
4751 SOCK_RECVBUF_UNLOCK(so);
4752 SOCK_UNLOCK(so);
4757 SOCK_UNLOCK(so);
4758 wakeup(&so->so_timeo);
4759 sorwakeup(so);
4760 sowwakeup(so);
4764 soisdisconnecting(struct socket *so) argument
4767 SOCK_LOCK(so);
4768 so->so_state &= ~SS_ISCONNECTING;
4769 so->so_state |= SS_ISDISCONNECTING;
4771 if (!SOLISTENING(so)) {
4772 SOCK_RECVBUF_LOCK(so);
4773 socantrcvmore_locked(so);
4774 SOCK_SENDBUF_LOCK(so);
4775 socantsendmore_locked(so);
4777 SOCK_UNLOCK(so);
4778 wakeup(&so->so_timeo);
4782 soisdisconnected(struct socket *so) argument
4785 SOCK_LOCK(so);
4794 so->so_state |= SS_ISDISCONNECTED;
4796 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
4798 if (!SOLISTENING(so)) {
4799 SOCK_UNLOCK(so);
4800 SOCK_RECVBUF_LOCK(so);
4801 socantrcvmore_locked(so);
4802 SOCK_SENDBUF_LOCK(so);
4803 sbdrop_locked(&so->so_snd, sbused(&so->so_snd));
4804 socantsendmore_locked(so);
4806 SOCK_UNLOCK(so);
4807 wakeup(&so->so_timeo);
4811 soiolock(struct socket *so, struct sx *sx, int flags) argument
4830 if (__predict_false(SOLISTENING(so))) {
4861 sodtor_set(struct socket *so, so_dtor_t *func) argument
4864 SOCK_LOCK_ASSERT(so);
4865 so->so_dtor = func;
4872 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg) argument
4876 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4880 sb = &so->so_rcv;
4883 sb = &so->so_snd;
4886 SOCK_BUF_LOCK_ASSERT(so, which);
4893 soupcall_clear(struct socket *so, sb_which which) argument
4897 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4901 sb = &so->so_rcv;
4904 sb = &so->so_snd;
4907 SOCK_BUF_LOCK_ASSERT(so, which);
4909 ("%s: so %p no upcall to clear", __func__, so));
4916 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) argument
4919 SOLISTEN_LOCK_ASSERT(so);
4920 so->sol_upcall = func;
4921 so->sol_upcallarg = arg;
4927 struct socket *so = arg; local
4930 if (SOLISTENING(so)) {
4931 SOLISTEN_LOCK(so);
4933 SOCK_RECVBUF_LOCK(so);
4934 if (__predict_false(SOLISTENING(so))) {
4935 SOCK_RECVBUF_UNLOCK(so);
4944 struct socket *so = arg; local
4946 if (SOLISTENING(so))
4947 SOLISTEN_UNLOCK(so);
4949 SOCK_RECVBUF_UNLOCK(so);
4955 struct socket *so = arg; local
4958 if (SOLISTENING(so))
4959 SOLISTEN_LOCK_ASSERT(so);
4961 SOCK_RECVBUF_LOCK_ASSERT(so);
4963 if (SOLISTENING(so))
4964 SOLISTEN_UNLOCK_ASSERT(so);
4966 SOCK_RECVBUF_UNLOCK_ASSERT(so);
4973 struct socket *so = arg; local
4976 if (SOLISTENING(so)) {
4977 SOLISTEN_LOCK(so);
4979 SOCK_SENDBUF_LOCK(so);
4980 if (__predict_false(SOLISTENING(so))) {
4981 SOCK_SENDBUF_UNLOCK(so);
4990 struct socket *so = arg; local
4992 if (SOLISTENING(so))
4993 SOLISTEN_UNLOCK(so);
4995 SOCK_SENDBUF_UNLOCK(so);
5001 struct socket *so = arg; local
5004 if (SOLISTENING(so))
5005 SOLISTEN_LOCK_ASSERT(so);
5007 SOCK_SENDBUF_LOCK_ASSERT(so);
5009 if (SOLISTENING(so))
5010 SOLISTEN_UNLOCK_ASSERT(so);
5012 SOCK_SENDBUF_UNLOCK_ASSERT(so);
5018 * the kernel-format socket structure pointed to by so. This is done to
5025 sotoxsocket(struct socket *so, struct xsocket *xso) argument
5030 xso->xso_so = (uintptr_t)so;
5031 xso->so_type = so->so_type;
5032 xso->so_options = so->so_options;
5033 xso->so_linger = so->so_linger;
5034 xso->so_state = so->so_state;
5035 xso->so_pcb = (uintptr_t)so->so_pcb;
5036 xso->xso_protocol = so->so_proto->pr_protocol;
5037 xso->xso_family = so->so_proto->pr_domain->dom_family;
5038 xso->so_timeo = so->so_timeo;
5039 xso->so_error = so->so_error;
5040 xso->so_uid = so->so_cred->cr_uid;
5041 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0;
5042 SOCK_LOCK(so);
5043 xso->so_fibnum = so->so_fibnum;
5044 if (SOLISTENING(so)) {
5045 xso->so_qlen = so->sol_qlen;
5046 xso->so_incqlen = so->sol_incqlen;
5047 xso->so_qlimit = so->sol_qlimit;
5050 xso->so_state |= so->so_qstate;
5052 xso->so_oobmark = so->so_oobmark;
5053 sbtoxsockbuf(&so->so_snd, &xso->so_snd);
5054 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv);
5055 if ((so->so_rcv.sb_flags & SB_SPLICED) != 0)
5056 xso->so_splice_so = (uintptr_t)so->so_splice->dst;
5058 SOCK_UNLOCK(so);
5062 so_options_get(const struct socket *so) argument
5065 return (so->so_options);
5069 so_options_set(struct socket *so, int val) argument
5072 so->so_options = val;
5076 so_error_get(const struct socket *so) argument
5079 return (so->so_error);
5083 so_error_set(struct socket *so, int val) argument
5086 so->so_error = val;