/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved. * Copyright 2019 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SOCK_TEST extern int do_useracc; extern clock_t sock_test_timelimit; #endif /* SOCK_TEST */ #define MBLK_PULL_LEN 64 uint32_t so_mblk_pull_len = MBLK_PULL_LEN; #ifdef DEBUG boolean_t so_debug_length = B_FALSE; static boolean_t so_check_length(sonode_t *so); #endif static int so_acceptq_dequeue_locked(struct sonode *so, boolean_t dontblock, struct sonode **nsop) { struct sonode *nso = NULL; *nsop = NULL; ASSERT(MUTEX_HELD(&so->so_acceptq_lock)); while ((nso = list_remove_head(&so->so_acceptq_list)) == NULL) { /* * No need to check so_error here, because it is not * possible for a listening socket to be reset or otherwise * disconnected. * * So now we just need check if it's ok to wait. */ if (dontblock) return (EWOULDBLOCK); if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) return (EINTR); if (cv_wait_sig_swap(&so->so_acceptq_cv, &so->so_acceptq_lock) == 0) return (EINTR); } ASSERT(nso != NULL); ASSERT(so->so_acceptq_len > 0); so->so_acceptq_len--; nso->so_listener = NULL; *nsop = nso; return (0); } /* * int so_acceptq_dequeue(struct sonode *, boolean_t, struct sonode **) * * Pulls a connection off of the accept queue. * * Arguments: * so - listening socket * dontblock - indicate whether it's ok to sleep if there are no * connections on the queue * nsop - Value-return argument * * Return values: * 0 when a connection is successfully dequeued, in which case nsop * is set to point to the new connection. Upon failure a non-zero * value is returned, and the value of nsop is set to NULL. * * Note: * so_acceptq_dequeue() may return prematurly if the socket is falling * back to TPI. */ int so_acceptq_dequeue(struct sonode *so, boolean_t dontblock, struct sonode **nsop) { int error; mutex_enter(&so->so_acceptq_lock); error = so_acceptq_dequeue_locked(so, dontblock, nsop); mutex_exit(&so->so_acceptq_lock); return (error); } static void so_acceptq_flush_impl(struct sonode *so, list_t *list, boolean_t doclose) { struct sonode *nso; while ((nso = list_remove_head(list)) != NULL) { nso->so_listener = NULL; if (doclose) { (void) socket_close(nso, 0, CRED()); } else { /* * Only used for fallback - not possible when filters * are present. */ ASSERT(so->so_filter_active == 0); /* * Since the socket is on the accept queue, there can * only be one reference. We drop the reference and * just blow off the socket. */ ASSERT(nso->so_count == 1); nso->so_count--; /* drop the proto ref */ VN_RELE(SOTOV(nso)); } socket_destroy(nso); } } /* * void so_acceptq_flush(struct sonode *so) * * Removes all pending connections from a listening socket, and * frees the associated resources. * * Arguments * so - listening socket * doclose - make a close downcall for each socket on the accept queue * * Return values: * None. * * Note: * The caller has to ensure that no calls to so_acceptq_enqueue() or * so_acceptq_dequeue() occur while the accept queue is being flushed. * So either the socket needs to be in a state where no operations * would come in, or so_lock needs to be obtained. */ void so_acceptq_flush(struct sonode *so, boolean_t doclose) { so_acceptq_flush_impl(so, &so->so_acceptq_list, doclose); so_acceptq_flush_impl(so, &so->so_acceptq_defer, doclose); so->so_acceptq_len = 0; } int so_wait_connected_locked(struct sonode *so, boolean_t nonblock, sock_connid_t id) { ASSERT(MUTEX_HELD(&so->so_lock)); /* * The protocol has notified us that a connection attempt is being * made, so before we wait for a notification to arrive we must * clear out any errors associated with earlier connection attempts. */ if (so->so_error != 0 && SOCK_CONNID_LT(so->so_proto_connid, id)) so->so_error = 0; while (SOCK_CONNID_LT(so->so_proto_connid, id)) { if (nonblock) return (EINPROGRESS); if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) return (EINTR); if (cv_wait_sig_swap(&so->so_state_cv, &so->so_lock) == 0) return (EINTR); } if (so->so_error != 0) return (sogeterr(so, B_TRUE)); /* * Under normal circumstances, so_error should contain an error * in case the connect failed. However, it is possible for another * thread to come in a consume the error, so generate a sensible * error in that case. */ if ((so->so_state & SS_ISCONNECTED) == 0) return (ECONNREFUSED); return (0); } /* * int so_wait_connected(struct sonode *so, boolean_t nonblock, * sock_connid_t id) * * Wait until the socket is connected or an error has occured. * * Arguments: * so - socket * nonblock - indicate whether it's ok to sleep if the connection has * not yet been established * gen - generation number that was returned by the protocol * when the operation was started * * Returns: * 0 if the connection attempt was successful, or an error indicating why * the connection attempt failed. */ int so_wait_connected(struct sonode *so, boolean_t nonblock, sock_connid_t id) { int error; mutex_enter(&so->so_lock); error = so_wait_connected_locked(so, nonblock, id); mutex_exit(&so->so_lock); return (error); } int so_snd_wait_qnotfull_locked(struct sonode *so, boolean_t dontblock) { int error; ASSERT(MUTEX_HELD(&so->so_lock)); while (SO_SND_FLOWCTRLD(so)) { if (so->so_state & SS_CANTSENDMORE) return (EPIPE); if (dontblock) return (EWOULDBLOCK); if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) return (EINTR); if (so->so_sndtimeo == 0) { /* * Zero means disable timeout. */ error = cv_wait_sig(&so->so_snd_cv, &so->so_lock); } else { error = cv_reltimedwait_sig(&so->so_snd_cv, &so->so_lock, so->so_sndtimeo, TR_CLOCK_TICK); } if (error == 0) return (EINTR); else if (error == -1) return (EAGAIN); } return (0); } /* * int so_wait_sendbuf(struct sonode *so, boolean_t dontblock) * * Wait for the transport to notify us about send buffers becoming * available. */ int so_snd_wait_qnotfull(struct sonode *so, boolean_t dontblock) { int error = 0; mutex_enter(&so->so_lock); so->so_snd_wakeup = B_TRUE; error = so_snd_wait_qnotfull_locked(so, dontblock); so->so_snd_wakeup = B_FALSE; mutex_exit(&so->so_lock); return (error); } void so_snd_qfull(struct sonode *so) { mutex_enter(&so->so_lock); so->so_snd_qfull = B_TRUE; mutex_exit(&so->so_lock); } void so_snd_qnotfull(struct sonode *so) { mutex_enter(&so->so_lock); so->so_snd_qfull = B_FALSE; /* wake up everyone waiting for buffers */ cv_broadcast(&so->so_snd_cv); mutex_exit(&so->so_lock); } /* * Change the process/process group to which SIGIO is sent. */ int socket_chgpgrp(struct sonode *so, pid_t pid) { int error; ASSERT(MUTEX_HELD(&so->so_lock)); if (pid != 0) { /* * Permissions check by sending signal 0. * Note that when kill fails it does a * set_errno causing the system call to fail. */ error = kill(pid, 0); if (error != 0) { return (error); } } so->so_pgrp = pid; return (0); } /* * Generate a SIGIO, for 'writable' events include siginfo structure, * for read events just send the signal. */ /*ARGSUSED*/ static void socket_sigproc(proc_t *proc, int event) { k_siginfo_t info; ASSERT(event & (SOCKETSIG_WRITE | SOCKETSIG_READ | SOCKETSIG_URG)); if (event & SOCKETSIG_WRITE) { info.si_signo = SIGPOLL; info.si_code = POLL_OUT; info.si_errno = 0; info.si_fd = 0; info.si_band = 0; sigaddq(proc, NULL, &info, KM_NOSLEEP); } if (event & SOCKETSIG_READ) { sigtoproc(proc, NULL, SIGPOLL); } if (event & SOCKETSIG_URG) { sigtoproc(proc, NULL, SIGURG); } } void socket_sendsig(struct sonode *so, int event) { proc_t *proc; ASSERT(MUTEX_HELD(&so->so_lock)); if (so->so_pgrp == 0 || (!(so->so_state & SS_ASYNC) && event != SOCKETSIG_URG)) { return; } dprint(3, ("sending sig %d to %d\n", event, so->so_pgrp)); if (so->so_pgrp > 0) { /* * XXX This unfortunately still generates * a signal when a fd is closed but * the proc is active. */ mutex_enter(&pidlock); /* * Even if the thread started in another zone, we're receiving * on behalf of this socket's zone, so find the proc using the * socket's zone ID. */ proc = prfind_zone(so->so_pgrp, so->so_zoneid); if (proc == NULL) { mutex_exit(&pidlock); return; } mutex_enter(&proc->p_lock); mutex_exit(&pidlock); socket_sigproc(proc, event); mutex_exit(&proc->p_lock); } else { /* * Send to process group. Hold pidlock across * calls to socket_sigproc(). */ pid_t pgrp = -so->so_pgrp; mutex_enter(&pidlock); /* * Even if the thread started in another zone, we're receiving * on behalf of this socket's zone, so find the pgrp using the * socket's zone ID. */ proc = pgfind_zone(pgrp, so->so_zoneid); while (proc != NULL) { mutex_enter(&proc->p_lock); socket_sigproc(proc, event); mutex_exit(&proc->p_lock); proc = proc->p_pglink; } mutex_exit(&pidlock); } } #define MIN(a, b) ((a) < (b) ? (a) : (b)) /* Copy userdata into a new mblk_t */ mblk_t * socopyinuio(uio_t *uiop, ssize_t iosize, size_t wroff, ssize_t maxblk, size_t tail_len, int *errorp) { mblk_t *head = NULL, **tail = &head; ASSERT(iosize == INFPSZ || iosize > 0); if (iosize == INFPSZ || iosize > uiop->uio_resid) iosize = uiop->uio_resid; if (maxblk == INFPSZ) maxblk = iosize; /* Nothing to do in these cases, so we're done */ if (iosize < 0 || maxblk < 0 || (maxblk == 0 && iosize > 0)) goto done; /* * We will enter the loop below if iosize is 0; it will allocate an * empty message block and call uiomove(9F) which will just return. * We could avoid that with an extra check but would only slow * down the much more likely case where iosize is larger than 0. */ do { ssize_t blocksize; mblk_t *mp; blocksize = MIN(iosize, maxblk); ASSERT(blocksize >= 0); mp = allocb(wroff + blocksize + tail_len, BPRI_MED); if (mp == NULL) { *errorp = ENOMEM; return (head); } mp->b_rptr += wroff; mp->b_wptr = mp->b_rptr + blocksize; *tail = mp; tail = &mp->b_cont; /* uiomove(9F) either returns 0 or EFAULT */ if ((*errorp = uiomove(mp->b_rptr, (size_t)blocksize, UIO_WRITE, uiop)) != 0) { ASSERT(*errorp != ENOMEM); freemsg(head); return (NULL); } iosize -= blocksize; } while (iosize > 0); done: *errorp = 0; return (head); } mblk_t * socopyoutuio(mblk_t *mp, struct uio *uiop, ssize_t max_read, int *errorp) { int error; ptrdiff_t n; mblk_t *nmp; ASSERT(mp->b_wptr >= mp->b_rptr); /* * max_read is the offset of the oobmark and read can not go pass * the oobmark. */ if (max_read == INFPSZ || max_read > uiop->uio_resid) max_read = uiop->uio_resid; do { if ((n = MIN(max_read, MBLKL(mp))) != 0) { ASSERT(n > 0); error = uiomove(mp->b_rptr, n, UIO_READ, uiop); if (error != 0) { freemsg(mp); *errorp = error; return (NULL); } } mp->b_rptr += n; max_read -= n; while (mp != NULL && (mp->b_rptr >= mp->b_wptr)) { /* * get rid of zero length mblks */ nmp = mp; mp = mp->b_cont; freeb(nmp); } } while (mp != NULL && max_read > 0); *errorp = 0; return (mp); } static void so_prepend_msg(struct sonode *so, mblk_t *mp, mblk_t *last_tail) { ASSERT(last_tail != NULL); mp->b_next = so->so_rcv_q_head; mp->b_prev = last_tail; ASSERT(!(DB_FLAGS(mp) & DBLK_UIOA)); if (so->so_rcv_q_head == NULL) { ASSERT(so->so_rcv_q_last_head == NULL); so->so_rcv_q_last_head = mp; #ifdef DEBUG } else { ASSERT(!(DB_FLAGS(so->so_rcv_q_head) & DBLK_UIOA)); #endif } so->so_rcv_q_head = mp; #ifdef DEBUG if (so_debug_length) { mutex_enter(&so->so_lock); ASSERT(so_check_length(so)); mutex_exit(&so->so_lock); } #endif } /* * Move a mblk chain (mp_head, mp_last_head) to the sonode's rcv queue so it * can be processed by so_dequeue_msg(). */ void so_process_new_message(struct sonode *so, mblk_t *mp_head, mblk_t *mp_last_head) { if (so->so_filter_active > 0 && (mp_head = sof_filter_data_in_proc(so, mp_head, &mp_last_head)) == NULL) return; ASSERT(mp_head->b_prev != NULL); if (so->so_rcv_q_head == NULL) { so->so_rcv_q_head = mp_head; so->so_rcv_q_last_head = mp_last_head; ASSERT(so->so_rcv_q_last_head->b_prev != NULL); } else { boolean_t flag_equal = ((DB_FLAGS(mp_head) & DBLK_UIOA) == (DB_FLAGS(so->so_rcv_q_last_head) & DBLK_UIOA)); if (mp_head->b_next == NULL && DB_TYPE(mp_head) == M_DATA && DB_TYPE(so->so_rcv_q_last_head) == M_DATA && flag_equal) { so->so_rcv_q_last_head->b_prev->b_cont = mp_head; so->so_rcv_q_last_head->b_prev = mp_head->b_prev; mp_head->b_prev = NULL; } else if (flag_equal && (DB_FLAGS(mp_head) & DBLK_UIOA)) { /* * Append to last_head if more than one mblks, and both * mp_head and last_head are I/OAT mblks. */ ASSERT(mp_head->b_next != NULL); so->so_rcv_q_last_head->b_prev->b_cont = mp_head; so->so_rcv_q_last_head->b_prev = mp_head->b_prev; mp_head->b_prev = NULL; so->so_rcv_q_last_head->b_next = mp_head->b_next; mp_head->b_next = NULL; so->so_rcv_q_last_head = mp_last_head; } else { #ifdef DEBUG { mblk_t *tmp_mblk; tmp_mblk = mp_head; while (tmp_mblk != NULL) { ASSERT(tmp_mblk->b_prev != NULL); tmp_mblk = tmp_mblk->b_next; } } #endif so->so_rcv_q_last_head->b_next = mp_head; so->so_rcv_q_last_head = mp_last_head; } } } /* * Check flow control on a given sonode. Must have so_lock held, and * this function will release the hold. Return true if flow control * is cleared. */ boolean_t so_check_flow_control(struct sonode *so) { ASSERT(MUTEX_HELD(&so->so_lock)); if (so->so_flowctrld && (so->so_rcv_queued < so->so_rcvlowat && !(so->so_state & SS_FIL_RCV_FLOWCTRL))) { so->so_flowctrld = B_FALSE; mutex_exit(&so->so_lock); /* * Open up flow control. SCTP does not have any downcalls, and * it will clr flow ctrl in sosctp_recvmsg(). */ if (so->so_downcalls != NULL && so->so_downcalls->sd_clr_flowctrl != NULL) { (*so->so_downcalls->sd_clr_flowctrl) (so->so_proto_handle); } /* filters can start injecting data */ sof_sonode_notify_filters(so, SOF_EV_INJECT_DATA_IN_OK, 0); return (B_TRUE); } else { mutex_exit(&so->so_lock); return (B_FALSE); } } int so_dequeue_msg(struct sonode *so, mblk_t **mctlp, struct uio *uiop, rval_t *rvalp, int flags) { mblk_t *mp, *nmp; mblk_t *savemp, *savemptail; mblk_t *new_msg_head; mblk_t *new_msg_last_head; mblk_t *last_tail; boolean_t partial_read; boolean_t reset_atmark = B_FALSE; int more = 0; int error; ssize_t oobmark; sodirect_t *sodp = so->so_direct; partial_read = B_FALSE; *mctlp = NULL; again: mutex_enter(&so->so_lock); again1: #ifdef DEBUG if (so_debug_length) { ASSERT(so_check_length(so)); } #endif if (so->so_state & SS_RCVATMARK) { /* Check whether the caller is OK to read past the mark */ if (flags & MSG_NOMARK) { mutex_exit(&so->so_lock); return (EWOULDBLOCK); } reset_atmark = B_TRUE; } /* * First move messages from the dump area to processing area */ if (sodp != NULL) { if (sodp->sod_enabled) { if (sodp->sod_uioa.uioa_state & UIOA_ALLOC) { /* nothing to uioamove */ sodp = NULL; } else if (sodp->sod_uioa.uioa_state & UIOA_INIT) { sodp->sod_uioa.uioa_state &= UIOA_CLR; sodp->sod_uioa.uioa_state |= UIOA_ENABLED; /* * try to uioamove() the data that * has already queued. */ sod_uioa_so_init(so, sodp, uiop); } } else { sodp = NULL; } } new_msg_head = so->so_rcv_head; new_msg_last_head = so->so_rcv_last_head; so->so_rcv_head = NULL; so->so_rcv_last_head = NULL; oobmark = so->so_oobmark; /* * We can release the lock as there can only be one reader */ mutex_exit(&so->so_lock); if (new_msg_head != NULL) { so_process_new_message(so, new_msg_head, new_msg_last_head); } savemp = savemptail = NULL; rvalp->r_vals = 0; error = 0; mp = so->so_rcv_q_head; if (mp != NULL && (so->so_rcv_timer_tid == 0 || so->so_rcv_queued >= so->so_rcv_thresh)) { partial_read = B_FALSE; if (flags & MSG_PEEK) { if ((nmp = dupmsg(mp)) == NULL && (nmp = copymsg(mp)) == NULL) { size_t size = msgsize(mp); error = strwaitbuf(size, BPRI_HI); if (error) { return (error); } goto again; } mp = nmp; } else { ASSERT(mp->b_prev != NULL); last_tail = mp->b_prev; mp->b_prev = NULL; so->so_rcv_q_head = mp->b_next; if (so->so_rcv_q_head == NULL) { so->so_rcv_q_last_head = NULL; } mp->b_next = NULL; } ASSERT(mctlp != NULL); /* * First process PROTO or PCPROTO blocks, if any. */ if (DB_TYPE(mp) != M_DATA) { *mctlp = mp; savemp = mp; savemptail = mp; ASSERT(DB_TYPE(mp) == M_PROTO || DB_TYPE(mp) == M_PCPROTO); while (mp->b_cont != NULL && DB_TYPE(mp->b_cont) != M_DATA) { ASSERT(DB_TYPE(mp->b_cont) == M_PROTO || DB_TYPE(mp->b_cont) == M_PCPROTO); mp = mp->b_cont; savemptail = mp; } mp = savemptail->b_cont; savemptail->b_cont = NULL; } ASSERT(DB_TYPE(mp) == M_DATA); /* * Now process DATA blocks, if any. Note that for sodirect * enabled socket, uio_resid can be 0. */ if (uiop->uio_resid >= 0) { ssize_t copied = 0; if (sodp != NULL && (DB_FLAGS(mp) & DBLK_UIOA)) { mutex_enter(&so->so_lock); ASSERT(uiop == (uio_t *)&sodp->sod_uioa); copied = sod_uioa_mblk(so, mp); if (copied > 0) partial_read = B_TRUE; mutex_exit(&so->so_lock); /* mark this mblk as processed */ mp = NULL; } else { ssize_t oldresid = uiop->uio_resid; if (MBLKL(mp) < so_mblk_pull_len) { if (pullupmsg(mp, -1) == 1) { last_tail = mp; } } /* * Can not read beyond the oobmark */ mp = socopyoutuio(mp, uiop, oobmark == 0 ? INFPSZ : oobmark, &error); if (error != 0) { freemsg(*mctlp); *mctlp = NULL; more = 0; goto done; } ASSERT(oldresid >= uiop->uio_resid); copied = oldresid - uiop->uio_resid; if (oldresid > uiop->uio_resid) partial_read = B_TRUE; } ASSERT(copied >= 0); if (copied > 0 && !(flags & MSG_PEEK)) { mutex_enter(&so->so_lock); so->so_rcv_queued -= copied; ASSERT(so->so_oobmark >= 0); if (so->so_oobmark > 0) { so->so_oobmark -= copied; ASSERT(so->so_oobmark >= 0); if (so->so_oobmark == 0) { ASSERT(so->so_state & SS_OOBPEND); so->so_oobmark = 0; so->so_state |= SS_RCVATMARK; } } /* * so_check_flow_control() will drop * so->so_lock. */ rvalp->r_val2 = so_check_flow_control(so); } } if (mp != NULL) { /* more data blocks in msg */ more |= MOREDATA; if ((flags & (MSG_PEEK|MSG_TRUNC))) { if (flags & MSG_PEEK) { freemsg(mp); } else { unsigned int msize = msgdsize(mp); freemsg(mp); mutex_enter(&so->so_lock); so->so_rcv_queued -= msize; /* * so_check_flow_control() will drop * so->so_lock. */ rvalp->r_val2 = so_check_flow_control(so); } } else if (partial_read && !somsghasdata(mp)) { /* * Avoid queuing a zero-length tail part of * a message. partial_read == 1 indicates that * we read some of the message. */ freemsg(mp); more &= ~MOREDATA; } else { if (savemp != NULL && (flags & MSG_DUPCTRL)) { mblk_t *nmp; /* * There should only be non data mblks */ ASSERT(DB_TYPE(savemp) != M_DATA && DB_TYPE(savemptail) != M_DATA); try_again: if ((nmp = dupmsg(savemp)) == NULL && (nmp = copymsg(savemp)) == NULL) { size_t size = msgsize(savemp); error = strwaitbuf(size, BPRI_HI); if (error != 0) { /* * In case we * cannot copy * control data * free the remaining * data. */ freemsg(mp); goto done; } goto try_again; } ASSERT(nmp != NULL); ASSERT(DB_TYPE(nmp) != M_DATA); savemptail->b_cont = mp; *mctlp = nmp; mp = savemp; } /* * putback mp */ so_prepend_msg(so, mp, last_tail); } } /* fast check so_rcv_head if there is more data */ if (partial_read && !(so->so_state & SS_RCVATMARK) && *mctlp == NULL && uiop->uio_resid > 0 && !(flags & MSG_PEEK) && so->so_rcv_head != NULL) { goto again; } } else if (!partial_read) { mutex_enter(&so->so_lock); if (so->so_error != 0) { error = sogeterr(so, !(flags & MSG_PEEK)); mutex_exit(&so->so_lock); return (error); } /* * No pending data. Return right away for nonblocking * socket, otherwise sleep waiting for data. */ if (!(so->so_state & SS_CANTRCVMORE) && uiop->uio_resid > 0) { if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || (flags & MSG_DONTWAIT)) { error = EWOULDBLOCK; } else { if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) { mutex_exit(&so->so_lock); error = EINTR; goto done; } if (so->so_rcv_head != NULL) { goto again1; } so->so_rcv_wakeup = B_TRUE; so->so_rcv_wanted = uiop->uio_resid; if (so->so_rcvtimeo == 0) { /* * Zero means disable timeout. */ error = cv_wait_sig(&so->so_rcv_cv, &so->so_lock); } else { error = cv_reltimedwait_sig( &so->so_rcv_cv, &so->so_lock, so->so_rcvtimeo, TR_CLOCK_TICK); } so->so_rcv_wakeup = B_FALSE; so->so_rcv_wanted = 0; if (error == 0) { error = EINTR; } else if (error == -1) { error = EAGAIN; } else { goto again1; } } } mutex_exit(&so->so_lock); } if (reset_atmark && partial_read && !(flags & MSG_PEEK)) { /* * We are passed the mark, update state * 4.3BSD and 4.4BSD clears the mark when peeking across it. * The draft Posix socket spec states that the mark should * not be cleared when peeking. We follow the latter. */ mutex_enter(&so->so_lock); ASSERT(so_verify_oobstate(so)); so->so_state &= ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_RCVATMARK); freemsg(so->so_oobmsg); so->so_oobmsg = NULL; ASSERT(so_verify_oobstate(so)); mutex_exit(&so->so_lock); } ASSERT(so->so_rcv_wakeup == B_FALSE); done: if (sodp != NULL) { mutex_enter(&so->so_lock); if (sodp->sod_enabled && (sodp->sod_uioa.uioa_state & UIOA_ENABLED)) { SOD_UIOAFINI(sodp); if (sodp->sod_uioa.uioa_mbytes > 0) { ASSERT(so->so_rcv_q_head != NULL || so->so_rcv_head != NULL); so->so_rcv_queued -= sod_uioa_mblk(so, NULL); if (error == EWOULDBLOCK) error = 0; } } mutex_exit(&so->so_lock); } #ifdef DEBUG if (so_debug_length) { mutex_enter(&so->so_lock); ASSERT(so_check_length(so)); mutex_exit(&so->so_lock); } #endif rvalp->r_val1 = more; ASSERT(MUTEX_NOT_HELD(&so->so_lock)); return (error); } /* * Enqueue data from the protocol on the socket's rcv queue. * * We try to hook new M_DATA mblks onto an existing chain, however, * that cannot be done if the existing chain has already been * processed by I/OAT. Non-M_DATA mblks are just linked together via * b_next. In all cases the b_prev of the enqueued mblk is set to * point to the last mblk in its b_cont chain. */ void so_enqueue_msg(struct sonode *so, mblk_t *mp, size_t msg_size) { ASSERT(MUTEX_HELD(&so->so_lock)); #ifdef DEBUG if (so_debug_length) { ASSERT(so_check_length(so)); } #endif so->so_rcv_queued += msg_size; if (so->so_rcv_head == NULL) { ASSERT(so->so_rcv_last_head == NULL); so->so_rcv_head = mp; so->so_rcv_last_head = mp; } else if ((DB_TYPE(mp) == M_DATA && DB_TYPE(so->so_rcv_last_head) == M_DATA) && ((DB_FLAGS(mp) & DBLK_UIOA) == (DB_FLAGS(so->so_rcv_last_head) & DBLK_UIOA))) { /* Added to the end */ ASSERT(so->so_rcv_last_head != NULL); ASSERT(so->so_rcv_last_head->b_prev != NULL); so->so_rcv_last_head->b_prev->b_cont = mp; } else { /* Start a new end */ so->so_rcv_last_head->b_next = mp; so->so_rcv_last_head = mp; } while (mp->b_cont != NULL) mp = mp->b_cont; so->so_rcv_last_head->b_prev = mp; #ifdef DEBUG if (so_debug_length) { ASSERT(so_check_length(so)); } #endif } /* * Return B_TRUE if there is data in the message, B_FALSE otherwise. */ boolean_t somsghasdata(mblk_t *mp) { for (; mp; mp = mp->b_cont) if (mp->b_datap->db_type == M_DATA) { ASSERT(mp->b_wptr >= mp->b_rptr); if (mp->b_wptr > mp->b_rptr) return (B_TRUE); } return (B_FALSE); } /* * Flush the read side of sockfs. * * The caller must be sure that a reader is not already active when the * buffer is being flushed. */ void so_rcv_flush(struct sonode *so) { mblk_t *mp; ASSERT(MUTEX_HELD(&so->so_lock)); if (so->so_oobmsg != NULL) { freemsg(so->so_oobmsg); so->so_oobmsg = NULL; so->so_oobmark = 0; so->so_state &= ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_HADOOBDATA|SS_RCVATMARK); } /* * Free messages sitting in the recv queues */ while (so->so_rcv_q_head != NULL) { mp = so->so_rcv_q_head; so->so_rcv_q_head = mp->b_next; mp->b_next = mp->b_prev = NULL; freemsg(mp); } while (so->so_rcv_head != NULL) { mp = so->so_rcv_head; so->so_rcv_head = mp->b_next; mp->b_next = mp->b_prev = NULL; freemsg(mp); } so->so_rcv_queued = 0; so->so_rcv_q_head = NULL; so->so_rcv_q_last_head = NULL; so->so_rcv_head = NULL; so->so_rcv_last_head = NULL; } /* * Handle recv* calls that set MSG_OOB or MSG_OOB together with MSG_PEEK. */ int sorecvoob(struct sonode *so, struct nmsghdr *msg, struct uio *uiop, int flags, boolean_t oob_inline) { mblk_t *mp, *nmp; int error; dprintso(so, 1, ("sorecvoob(%p, %p, 0x%x)\n", (void *)so, (void *)msg, flags)); if (msg != NULL) { /* * There is never any oob data with addresses or control since * the T_EXDATA_IND does not carry any options. */ msg->msg_controllen = 0; msg->msg_namelen = 0; msg->msg_flags = 0; } mutex_enter(&so->so_lock); ASSERT(so_verify_oobstate(so)); if (oob_inline || (so->so_state & (SS_OOBPEND|SS_HADOOBDATA)) != SS_OOBPEND) { dprintso(so, 1, ("sorecvoob: inline or data consumed\n")); mutex_exit(&so->so_lock); return (EINVAL); } if (!(so->so_state & SS_HAVEOOBDATA)) { dprintso(so, 1, ("sorecvoob: no data yet\n")); mutex_exit(&so->so_lock); return (EWOULDBLOCK); } ASSERT(so->so_oobmsg != NULL); mp = so->so_oobmsg; if (flags & MSG_PEEK) { /* * Since recv* can not return ENOBUFS we can not use dupmsg. * Instead we revert to the consolidation private * allocb_wait plus bcopy. */ mblk_t *mp1; mp1 = allocb_wait(msgdsize(mp), BPRI_MED, STR_NOSIG, NULL); ASSERT(mp1); while (mp != NULL) { ssize_t size; size = MBLKL(mp); bcopy(mp->b_rptr, mp1->b_wptr, size); mp1->b_wptr += size; ASSERT(mp1->b_wptr <= mp1->b_datap->db_lim); mp = mp->b_cont; } mp = mp1; } else { /* * Update the state indicating that the data has been consumed. * Keep SS_OOBPEND set until data is consumed past the mark. */ so->so_oobmsg = NULL; so->so_state ^= SS_HAVEOOBDATA|SS_HADOOBDATA; } ASSERT(so_verify_oobstate(so)); mutex_exit(&so->so_lock); error = 0; nmp = mp; while (nmp != NULL && uiop->uio_resid > 0) { ssize_t n = MBLKL(nmp); n = MIN(n, uiop->uio_resid); if (n > 0) error = uiomove(nmp->b_rptr, n, UIO_READ, uiop); if (error) break; nmp = nmp->b_cont; } ASSERT(mp->b_next == NULL && mp->b_prev == NULL); freemsg(mp); return (error); } /* * Allocate and initializ sonode */ /* ARGSUSED */ struct sonode * socket_sonode_create(struct sockparams *sp, int family, int type, int protocol, int version, int sflags, int *errorp, struct cred *cr) { sonode_t *so; int kmflags; /* * Choose the right set of sonodeops based on the upcall and * down call version that the protocol has provided */ if (SOCK_UC_VERSION != sp->sp_smod_info->smod_uc_version || SOCK_DC_VERSION != sp->sp_smod_info->smod_dc_version) { /* * mismatch */ #ifdef DEBUG cmn_err(CE_CONT, "protocol and socket module version mismatch"); #endif *errorp = EINVAL; return (NULL); } kmflags = (sflags & SOCKET_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; so = kmem_cache_alloc(socket_cache, kmflags); if (so == NULL) { *errorp = ENOMEM; return (NULL); } sonode_init(so, sp, family, type, protocol, &so_sonodeops); if (version == SOV_DEFAULT) version = so_default_version; so->so_version = (short)version; /* * set the default values to be INFPSZ * if a protocol desires it can change the value later */ so->so_proto_props.sopp_rxhiwat = SOCKET_RECVHIWATER; so->so_proto_props.sopp_rxlowat = SOCKET_RECVLOWATER; so->so_proto_props.sopp_maxpsz = INFPSZ; so->so_proto_props.sopp_maxblk = INFPSZ; return (so); } int socket_init_common(struct sonode *so, struct sonode *pso, int flags, cred_t *cr) { int error = 0; if (pso != NULL) { /* * We have a passive open, so inherit basic state from * the parent (listener). * * No need to grab the new sonode's lock, since there is no * one that can have a reference to it. */ mutex_enter(&pso->so_lock); so->so_state |= SS_ISCONNECTED | (pso->so_state & SS_ASYNC); so->so_pgrp = pso->so_pgrp; so->so_rcvtimeo = pso->so_rcvtimeo; so->so_sndtimeo = pso->so_sndtimeo; so->so_xpg_rcvbuf = pso->so_xpg_rcvbuf; /* * Make note of the socket level options. TCP and IP level * options are already inherited. We could do all this after * accept is successful but doing it here simplifies code and * no harm done for error case. */ so->so_options = pso->so_options & (SO_DEBUG|SO_REUSEADDR| SO_KEEPALIVE|SO_DONTROUTE|SO_BROADCAST|SO_USELOOPBACK| SO_OOBINLINE|SO_DGRAM_ERRIND|SO_LINGER); so->so_proto_props = pso->so_proto_props; so->so_mode = pso->so_mode; so->so_pollev = pso->so_pollev & SO_POLLEV_ALWAYS; mutex_exit(&pso->so_lock); /* * If the parent has any filters, try to inherit them. */ if (pso->so_filter_active > 0 && (error = sof_sonode_inherit_filters(so, pso)) != 0) return (error); } else { struct sockparams *sp = so->so_sockparams; sock_upcalls_t *upcalls_to_use; /* * Attach automatic filters, if there are any. */ if (!list_is_empty(&sp->sp_auto_filters) && (error = sof_sonode_autoattach_filters(so, cr)) != 0) return (error); /* OK to attach filters */ so->so_state |= SS_FILOP_OK; /* * Based on the version number select the right upcalls to * pass down. Currently we only have one version so choose * default */ upcalls_to_use = &so_upcalls; /* active open, so create a lower handle */ so->so_proto_handle = sp->sp_smod_info->smod_proto_create_func(so->so_family, so->so_type, so->so_protocol, &so->so_downcalls, &so->so_mode, &error, flags, cr); if (so->so_proto_handle == NULL) { ASSERT(error != 0); /* * To be safe; if a lower handle cannot be created, and * the proto does not give a reason why, assume there * was a lack of memory. */ return ((error == 0) ? ENOMEM : error); } ASSERT(so->so_downcalls != NULL); ASSERT(so->so_downcalls->sd_send != NULL || so->so_downcalls->sd_send_uio != NULL); if (so->so_downcalls->sd_recv_uio != NULL) { ASSERT(so->so_downcalls->sd_poll != NULL); so->so_pollev |= SO_POLLEV_ALWAYS; } (*so->so_downcalls->sd_activate)(so->so_proto_handle, (sock_upper_handle_t)so, upcalls_to_use, 0, cr); /* Wildcard */ /* * FIXME No need for this, the protocol can deal with it in * sd_create(). Should update ICMP. */ if (so->so_protocol != so->so_sockparams->sp_protocol) { int protocol = so->so_protocol; int error; /* * Issue SO_PROTOTYPE setsockopt. */ error = socket_setsockopt(so, SOL_SOCKET, SO_PROTOTYPE, &protocol, (t_uscalar_t)sizeof (protocol), cr); if (error) { (void) (*so->so_downcalls->sd_close) (so->so_proto_handle, 0, cr); mutex_enter(&so->so_lock); so_rcv_flush(so); mutex_exit(&so->so_lock); /* * Setsockopt often fails with ENOPROTOOPT but * socket() should fail with * EPROTONOSUPPORT/EPROTOTYPE. */ return (EPROTONOSUPPORT); } } } if (uioasync.enabled) sod_sock_init(so); /* put an extra reference on the socket for the protocol */ VN_HOLD(SOTOV(so)); return (0); } /* * int socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode, * struct cred *cr, int32_t *rvalp) * * Handle ioctls that manipulate basic socket state; non-blocking, * async, etc. * * Returns: * < 0 - ioctl was not handle * >= 0 - ioctl was handled, if > 0, then it is an errno * * Notes: * Assumes the standard receive buffer is used to obtain info for * NREAD. */ /* ARGSUSED */ int socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode, struct cred *cr, int32_t *rvalp) { switch (cmd) { case SIOCSQPTR: /* * SIOCSQPTR is valid only when helper stream is created * by the protocol. */ return (EOPNOTSUPP); case FIONBIO: { int32_t value; if (so_copyin((void *)arg, &value, sizeof (int32_t), (mode & (int)FKIOCTL))) return (EFAULT); mutex_enter(&so->so_lock); if (value) { so->so_state |= SS_NDELAY; } else { so->so_state &= ~SS_NDELAY; } mutex_exit(&so->so_lock); return (0); } case FIOASYNC: { int32_t value; if (so_copyin((void *)arg, &value, sizeof (int32_t), (mode & (int)FKIOCTL))) return (EFAULT); mutex_enter(&so->so_lock); if (value) { /* Turn on SIGIO */ so->so_state |= SS_ASYNC; } else { /* Turn off SIGIO */ so->so_state &= ~SS_ASYNC; } mutex_exit(&so->so_lock); return (0); } case SIOCSPGRP: case FIOSETOWN: { int error; pid_t pid; if (so_copyin((void *)arg, &pid, sizeof (pid_t), (mode & (int)FKIOCTL))) return (EFAULT); mutex_enter(&so->so_lock); error = (pid != so->so_pgrp) ? socket_chgpgrp(so, pid) : 0; mutex_exit(&so->so_lock); return (error); } case SIOCGPGRP: case FIOGETOWN: if (so_copyout(&so->so_pgrp, (void *)arg, sizeof (pid_t), (mode & (int)FKIOCTL))) return (EFAULT); return (0); case SIOCATMARK: { int retval; /* * Only protocols that support urgent data can handle ATMARK. */ if ((so->so_mode & SM_EXDATA) == 0) return (EINVAL); /* * If the protocol is maintaining its own buffer, then the * request must be passed down. */ if (so->so_downcalls->sd_recv_uio != NULL) return (-1); retval = (so->so_state & SS_RCVATMARK) != 0; if (so_copyout(&retval, (void *)arg, sizeof (int), (mode & (int)FKIOCTL))) { return (EFAULT); } return (0); } case FIONREAD: { int retval; /* * If the protocol is maintaining its own buffer, then the * request must be passed down. */ if (so->so_downcalls->sd_recv_uio != NULL) return (-1); retval = MIN(so->so_rcv_queued, INT_MAX); if (so_copyout(&retval, (void *)arg, sizeof (retval), (mode & (int)FKIOCTL))) { return (EFAULT); } return (0); } case _I_GETPEERCRED: { int error = 0; if ((mode & FKIOCTL) == 0) return (EINVAL); mutex_enter(&so->so_lock); if ((so->so_mode & SM_CONNREQUIRED) == 0) { error = ENOTSUP; } else if ((so->so_state & SS_ISCONNECTED) == 0) { error = ENOTCONN; } else if (so->so_peercred != NULL) { k_peercred_t *kp = (k_peercred_t *)arg; kp->pc_cr = so->so_peercred; kp->pc_cpid = so->so_cpid; crhold(so->so_peercred); } else { error = EINVAL; } mutex_exit(&so->so_lock); return (error); } default: return (-1); } } /* * Handle the I_NREAD STREAM ioctl. */ static int so_strioc_nread(struct sonode *so, intptr_t arg, int mode, int32_t *rvalp) { size_t size = 0; int retval; int count = 0; mblk_t *mp; clock_t wakeup = drv_usectohz(10); if (so->so_downcalls == NULL || so->so_downcalls->sd_recv_uio != NULL) return (EINVAL); mutex_enter(&so->so_lock); /* Wait for reader to get out of the way. */ while (so->so_flag & SOREADLOCKED) { /* * If reader is waiting for data, then there should be nothing * on the rcv queue. */ if (so->so_rcv_wakeup) goto out; /* Do a timed sleep, in case the reader goes to sleep. */ (void) cv_reltimedwait(&so->so_read_cv, &so->so_lock, wakeup, TR_CLOCK_TICK); } /* * Since we are holding so_lock no new reader will come in, and the * protocol will not be able to enqueue data. So it's safe to walk * both rcv queues. */ mp = so->so_rcv_q_head; if (mp != NULL) { size = msgdsize(so->so_rcv_q_head); for (; mp != NULL; mp = mp->b_next) count++; } else { /* * In case the processing list was empty, get the size of the * next msg in line. */ size = msgdsize(so->so_rcv_head); } for (mp = so->so_rcv_head; mp != NULL; mp = mp->b_next) count++; out: mutex_exit(&so->so_lock); /* * Drop down from size_t to the "int" required by the * interface. Cap at INT_MAX. */ retval = MIN(size, INT_MAX); if (so_copyout(&retval, (void *)arg, sizeof (retval), (mode & (int)FKIOCTL))) { return (EFAULT); } else { *rvalp = count; return (0); } } /* * Process STREAM ioctls. * * Returns: * < 0 - ioctl was not handle * >= 0 - ioctl was handled, if > 0, then it is an errno */ int socket_strioc_common(struct sonode *so, int cmd, intptr_t arg, int mode, struct cred *cr, int32_t *rvalp) { int retval; /* Only STREAM iotcls are handled here */ if ((cmd & 0xffffff00U) != STR) return (-1); switch (cmd) { case I_CANPUT: /* * We return an error for I_CANPUT so that isastream(3C) will * not report the socket as being a STREAM. */ return (EOPNOTSUPP); case I_NREAD: /* Avoid doing a fallback for I_NREAD. */ return (so_strioc_nread(so, arg, mode, rvalp)); case I_LOOK: /* Avoid doing a fallback for I_LOOK. */ if (so_copyout("sockmod", (void *)arg, strlen("sockmod") + 1, (mode & (int)FKIOCTL))) { return (EFAULT); } return (0); default: break; } /* * Try to fall back to TPI, and if successful, reissue the ioctl. */ if ((retval = so_tpi_fallback(so, cr)) == 0) { /* Reissue the ioctl */ ASSERT(so->so_rcv_q_head == NULL); return (SOP_IOCTL(so, cmd, arg, mode, cr, rvalp)); } else { return (retval); } } /* * This is called for all socket types to verify that the buffer size is large * enough for the option, and if we can, handle the request as well. Most * options will be forwarded to the protocol. */ int socket_getopt_common(struct sonode *so, int level, int option_name, void *optval, socklen_t *optlenp, int flags) { if (level != SOL_SOCKET) return (-1); switch (option_name) { case SO_ERROR: case SO_DOMAIN: case SO_TYPE: case SO_ACCEPTCONN: { int32_t value; socklen_t optlen = *optlenp; if (optlen < (t_uscalar_t)sizeof (int32_t)) { return (EINVAL); } switch (option_name) { case SO_ERROR: mutex_enter(&so->so_lock); value = sogeterr(so, B_TRUE); mutex_exit(&so->so_lock); break; case SO_DOMAIN: value = so->so_family; break; case SO_TYPE: value = so->so_type; break; case SO_ACCEPTCONN: if (so->so_state & SS_ACCEPTCONN) value = SO_ACCEPTCONN; else value = 0; break; } bcopy(&value, optval, sizeof (value)); *optlenp = sizeof (value); return (0); } case SO_SNDTIMEO: case SO_RCVTIMEO: { clock_t value; socklen_t optlen = *optlenp; if (get_udatamodel() == DATAMODEL_NONE || get_udatamodel() == DATAMODEL_NATIVE) { if (optlen < sizeof (struct timeval)) return (EINVAL); } else { if (optlen < sizeof (struct timeval32)) return (EINVAL); } if (option_name == SO_RCVTIMEO) value = drv_hztousec(so->so_rcvtimeo); else value = drv_hztousec(so->so_sndtimeo); if (get_udatamodel() == DATAMODEL_NONE || get_udatamodel() == DATAMODEL_NATIVE) { ((struct timeval *)(optval))->tv_sec = value / (1000 * 1000); ((struct timeval *)(optval))->tv_usec = value % (1000 * 1000); *optlenp = sizeof (struct timeval); } else { ((struct timeval32 *)(optval))->tv_sec = value / (1000 * 1000); ((struct timeval32 *)(optval))->tv_usec = value % (1000 * 1000); *optlenp = sizeof (struct timeval32); } return (0); } case SO_DEBUG: case SO_REUSEADDR: case SO_KEEPALIVE: case SO_DONTROUTE: case SO_BROADCAST: case SO_USELOOPBACK: case SO_OOBINLINE: case SO_SNDBUF: #ifdef notyet case SO_SNDLOWAT: case SO_RCVLOWAT: #endif /* notyet */ case SO_DGRAM_ERRIND: { socklen_t optlen = *optlenp; if (optlen < (t_uscalar_t)sizeof (int32_t)) return (EINVAL); break; } case SO_RCVBUF: { socklen_t optlen = *optlenp; if (optlen < (t_uscalar_t)sizeof (int32_t)) return (EINVAL); if ((flags & _SOGETSOCKOPT_XPG4_2) && so->so_xpg_rcvbuf != 0) { /* * XXX If SO_RCVBUF has been set and this is an * XPG 4.2 application then do not ask the transport * since the transport might adjust the value and not * return exactly what was set by the application. * For non-XPG 4.2 application we return the value * that the transport is actually using. */ *(int32_t *)optval = so->so_xpg_rcvbuf; *optlenp = sizeof (so->so_xpg_rcvbuf); return (0); } /* * If the option has not been set then get a default * value from the transport. */ break; } case SO_LINGER: { socklen_t optlen = *optlenp; if (optlen < (t_uscalar_t)sizeof (struct linger)) return (EINVAL); break; } case SO_SND_BUFINFO: { socklen_t optlen = *optlenp; if (optlen < (t_uscalar_t)sizeof (struct so_snd_bufinfo)) return (EINVAL); ((struct so_snd_bufinfo *)(optval))->sbi_wroff = (so->so_proto_props).sopp_wroff; ((struct so_snd_bufinfo *)(optval))->sbi_maxblk = (so->so_proto_props).sopp_maxblk; ((struct so_snd_bufinfo *)(optval))->sbi_maxpsz = (so->so_proto_props).sopp_maxpsz; ((struct so_snd_bufinfo *)(optval))->sbi_tail = (so->so_proto_props).sopp_tail; *optlenp = sizeof (struct so_snd_bufinfo); return (0); } case SO_SND_COPYAVOID: { sof_instance_t *inst; /* * Avoid zero-copy if there is a filter with a data_out * callback. We could let the operation succeed, but then * the filter would have to copy the data anyway. */ for (inst = so->so_filter_top; inst != NULL; inst = inst->sofi_next) { if (SOF_INTERESTED(inst, data_out)) return (EOPNOTSUPP); } break; } default: break; } /* Unknown Option */ return (-1); } void socket_sonode_destroy(struct sonode *so) { sonode_fini(so); kmem_cache_free(socket_cache, so); } int so_zcopy_wait(struct sonode *so) { int error = 0; mutex_enter(&so->so_lock); while (!(so->so_copyflag & STZCNOTIFY)) { if (so->so_state & SS_CLOSING) { mutex_exit(&so->so_lock); return (EINTR); } if (cv_wait_sig(&so->so_copy_cv, &so->so_lock) == 0) { error = EINTR; break; } } so->so_copyflag &= ~STZCNOTIFY; mutex_exit(&so->so_lock); return (error); } void so_timer_callback(void *arg) { struct sonode *so = (struct sonode *)arg; mutex_enter(&so->so_lock); so->so_rcv_timer_tid = 0; if (so->so_rcv_queued > 0) { so_notify_data(so, so->so_rcv_queued); } else { mutex_exit(&so->so_lock); } } #ifdef DEBUG /* * Verify that the length stored in so_rcv_queued and the length of data blocks * queued is same. */ static boolean_t so_check_length(sonode_t *so) { mblk_t *mp = so->so_rcv_q_head; int len = 0; ASSERT(MUTEX_HELD(&so->so_lock)); if (mp != NULL) { len = msgdsize(mp); while ((mp = mp->b_next) != NULL) len += msgdsize(mp); } mp = so->so_rcv_head; if (mp != NULL) { len += msgdsize(mp); while ((mp = mp->b_next) != NULL) len += msgdsize(mp); } return ((len == so->so_rcv_queued) ? B_TRUE : B_FALSE); } #endif int so_get_mod_version(struct sockparams *sp) { ASSERT(sp != NULL && sp->sp_smod_info != NULL); return (sp->sp_smod_info->smod_version); } /* * so_start_fallback() * * Block new socket operations from coming in, and wait for active operations * to complete. Threads that are sleeping will be woken up so they can get * out of the way. * * The caller must be a reader on so_fallback_rwlock. */ static boolean_t so_start_fallback(struct sonode *so) { ASSERT(RW_READ_HELD(&so->so_fallback_rwlock)); mutex_enter(&so->so_lock); if (so->so_state & SS_FALLBACK_PENDING) { mutex_exit(&so->so_lock); return (B_FALSE); } so->so_state |= SS_FALLBACK_PENDING; /* * Poke all threads that might be sleeping. Any operation that comes * in after the cv_broadcast will observe the fallback pending flag * which cause the call to return where it would normally sleep. */ cv_broadcast(&so->so_state_cv); /* threads in connect() */ cv_broadcast(&so->so_rcv_cv); /* threads in recvmsg() */ cv_broadcast(&so->so_snd_cv); /* threads in sendmsg() */ mutex_enter(&so->so_acceptq_lock); cv_broadcast(&so->so_acceptq_cv); /* threads in accept() */ mutex_exit(&so->so_acceptq_lock); mutex_exit(&so->so_lock); /* * The main reason for the rw_tryupgrade call is to provide * observability during the fallback process. We want to * be able to see if there are pending operations. */ if (rw_tryupgrade(&so->so_fallback_rwlock) == 0) { /* * It is safe to drop and reaquire the fallback lock, because * we are guaranteed that another fallback cannot take place. */ rw_exit(&so->so_fallback_rwlock); DTRACE_PROBE1(pending__ops__wait, (struct sonode *), so); rw_enter(&so->so_fallback_rwlock, RW_WRITER); DTRACE_PROBE1(pending__ops__complete, (struct sonode *), so); } return (B_TRUE); } /* * so_end_fallback() * * Allow socket opertions back in. * * The caller must be a writer on so_fallback_rwlock. */ static void so_end_fallback(struct sonode *so) { ASSERT(RW_ISWRITER(&so->so_fallback_rwlock)); mutex_enter(&so->so_lock); so->so_state &= ~(SS_FALLBACK_PENDING|SS_FALLBACK_DRAIN); mutex_exit(&so->so_lock); rw_downgrade(&so->so_fallback_rwlock); } /* * so_quiesced_cb() * * Callback passed to the protocol during fallback. It is called once * the endpoint is quiescent. * * No requests from the user, no notifications from the protocol, so it * is safe to synchronize the state. Data can also be moved without * risk for reordering. * * We do not need to hold so_lock, since there can be only one thread * operating on the sonode. */ static mblk_t * so_quiesced_cb(sock_upper_handle_t sock_handle, sock_quiesce_arg_t *arg, struct T_capability_ack *tcap, struct sockaddr *laddr, socklen_t laddrlen, struct sockaddr *faddr, socklen_t faddrlen, short opts) { struct sonode *so = (struct sonode *)sock_handle; boolean_t atmark; mblk_t *retmp = NULL, **tailmpp = &retmp; if (tcap != NULL) sotpi_update_state(so, tcap, laddr, laddrlen, faddr, faddrlen, opts); /* * Some protocols do not quiece the data path during fallback. Once * we set the SS_FALLBACK_DRAIN flag any attempt to queue data will * fail and the protocol is responsible for saving the data for later * delivery (i.e., once the fallback has completed). */ mutex_enter(&so->so_lock); so->so_state |= SS_FALLBACK_DRAIN; SOCKET_TIMER_CANCEL(so); mutex_exit(&so->so_lock); if (so->so_rcv_head != NULL) { if (so->so_rcv_q_last_head == NULL) so->so_rcv_q_head = so->so_rcv_head; else so->so_rcv_q_last_head->b_next = so->so_rcv_head; so->so_rcv_q_last_head = so->so_rcv_last_head; } atmark = (so->so_state & SS_RCVATMARK) != 0; /* * Clear any OOB state having to do with pending data. The TPI * code path will set the appropriate oob state when we move the * oob data to the STREAM head. We leave SS_HADOOBDATA since the oob * data has already been consumed. */ so->so_state &= ~(SS_RCVATMARK|SS_OOBPEND|SS_HAVEOOBDATA); ASSERT(so->so_oobmsg != NULL || so->so_oobmark <= so->so_rcv_queued); /* * Move data to the STREAM head. */ while (so->so_rcv_q_head != NULL) { mblk_t *mp = so->so_rcv_q_head; size_t mlen = msgdsize(mp); so->so_rcv_q_head = mp->b_next; mp->b_next = NULL; mp->b_prev = NULL; /* * Send T_EXDATA_IND if we are at the oob mark. */ if (atmark) { struct T_exdata_ind *tei; mblk_t *mp1 = arg->soqa_exdata_mp; arg->soqa_exdata_mp = NULL; ASSERT(mp1 != NULL); mp1->b_datap->db_type = M_PROTO; tei = (struct T_exdata_ind *)mp1->b_rptr; tei->PRIM_type = T_EXDATA_IND; tei->MORE_flag = 0; mp1->b_wptr = (uchar_t *)&tei[1]; if (IS_SO_OOB_INLINE(so)) { mp1->b_cont = mp; } else { ASSERT(so->so_oobmsg != NULL); mp1->b_cont = so->so_oobmsg; so->so_oobmsg = NULL; /* process current mp next time around */ mp->b_next = so->so_rcv_q_head; so->so_rcv_q_head = mp; mlen = 0; } mp = mp1; /* we have consumed the oob mark */ atmark = B_FALSE; } else if (so->so_oobmark > 0) { /* * Check if the OOB mark is within the current * mblk chain. In that case we have to split it up. */ if (so->so_oobmark < mlen) { mblk_t *urg_mp = mp; atmark = B_TRUE; mp = NULL; mlen = so->so_oobmark; /* * It is assumed that the OOB mark does * not land within a mblk. */ do { so->so_oobmark -= MBLKL(urg_mp); mp = urg_mp; urg_mp = urg_mp->b_cont; } while (so->so_oobmark > 0); mp->b_cont = NULL; if (urg_mp != NULL) { urg_mp->b_next = so->so_rcv_q_head; so->so_rcv_q_head = urg_mp; } } else { so->so_oobmark -= mlen; if (so->so_oobmark == 0) atmark = B_TRUE; } } /* * Queue data on the STREAM head. */ so->so_rcv_queued -= mlen; *tailmpp = mp; tailmpp = &mp->b_next; } so->so_rcv_head = NULL; so->so_rcv_last_head = NULL; so->so_rcv_q_head = NULL; so->so_rcv_q_last_head = NULL; /* * Check if the oob byte is at the end of the data stream, or if the * oob byte has not yet arrived. In the latter case we have to send a * SIGURG and a mark indicator to the STREAM head. The mark indicator * is needed to guarantee correct behavior for SIOCATMARK. See block * comment in socktpi.h for more details. */ if (atmark || so->so_oobmark > 0) { mblk_t *mp; if (atmark && so->so_oobmsg != NULL) { struct T_exdata_ind *tei; mp = arg->soqa_exdata_mp; arg->soqa_exdata_mp = NULL; ASSERT(mp != NULL); mp->b_datap->db_type = M_PROTO; tei = (struct T_exdata_ind *)mp->b_rptr; tei->PRIM_type = T_EXDATA_IND; tei->MORE_flag = 0; mp->b_wptr = (uchar_t *)&tei[1]; mp->b_cont = so->so_oobmsg; so->so_oobmsg = NULL; *tailmpp = mp; tailmpp = &mp->b_next; } else { /* Send up the signal */ mp = arg->soqa_exdata_mp; arg->soqa_exdata_mp = NULL; ASSERT(mp != NULL); DB_TYPE(mp) = M_PCSIG; *mp->b_wptr++ = (uchar_t)SIGURG; *tailmpp = mp; tailmpp = &mp->b_next; /* Send up the mark indicator */ mp = arg->soqa_urgmark_mp; arg->soqa_urgmark_mp = NULL; mp->b_flag = atmark ? MSGMARKNEXT : MSGNOTMARKNEXT; *tailmpp = mp; tailmpp = &mp->b_next; so->so_oobmark = 0; } } ASSERT(so->so_oobmark == 0); ASSERT(so->so_rcv_queued == 0); return (retmp); } #ifdef DEBUG /* * Do an integrity check of the sonode. This should be done if a * fallback fails after sonode has initially been converted to use * TPI and subsequently have to be reverted. * * Failure to pass the integrity check will panic the system. */ void so_integrity_check(struct sonode *cur, struct sonode *orig) { VERIFY(cur->so_vnode == orig->so_vnode); VERIFY(cur->so_ops == orig->so_ops); /* * For so_state we can only VERIFY the state flags in CHECK_STATE. * The other state flags might be affected by a notification from the * protocol. */ #define CHECK_STATE (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_NDELAY|SS_NONBLOCK| \ SS_ASYNC|SS_ACCEPTCONN|SS_SAVEDEOR|SS_RCVATMARK|SS_OOBPEND| \ SS_HAVEOOBDATA|SS_HADOOBDATA|SS_SENTLASTREADSIG|SS_SENTLASTWRITESIG) VERIFY((cur->so_state & (orig->so_state & CHECK_STATE)) == (orig->so_state & CHECK_STATE)); VERIFY(cur->so_mode == orig->so_mode); VERIFY(cur->so_flag == orig->so_flag); VERIFY(cur->so_count == orig->so_count); /* Cannot VERIFY so_proto_connid; proto can update it */ VERIFY(cur->so_sockparams == orig->so_sockparams); /* an error might have been recorded, but it can not be lost */ VERIFY(cur->so_error != 0 || orig->so_error == 0); VERIFY(cur->so_family == orig->so_family); VERIFY(cur->so_type == orig->so_type); VERIFY(cur->so_protocol == orig->so_protocol); VERIFY(cur->so_version == orig->so_version); /* New conns might have arrived, but none should have been lost */ VERIFY(cur->so_acceptq_len >= orig->so_acceptq_len); VERIFY(list_head(&cur->so_acceptq_list) == list_head(&orig->so_acceptq_list)); VERIFY(cur->so_backlog == orig->so_backlog); /* New OOB migth have arrived, but mark should not have been lost */ VERIFY(cur->so_oobmark >= orig->so_oobmark); /* Cannot VERIFY so_oobmsg; the proto might have sent up a new one */ VERIFY(cur->so_pgrp == orig->so_pgrp); VERIFY(cur->so_peercred == orig->so_peercred); VERIFY(cur->so_cpid == orig->so_cpid); VERIFY(cur->so_zoneid == orig->so_zoneid); /* New data migth have arrived, but none should have been lost */ VERIFY(cur->so_rcv_queued >= orig->so_rcv_queued); VERIFY(cur->so_rcv_q_head == orig->so_rcv_q_head); VERIFY(cur->so_rcv_head == orig->so_rcv_head); VERIFY(cur->so_proto_handle == orig->so_proto_handle); VERIFY(cur->so_downcalls == orig->so_downcalls); /* Cannot VERIFY so_proto_props; they can be updated by proto */ } #endif /* * so_tpi_fallback() * * This is the fallback initation routine; things start here. * * Basic strategy: * o Block new socket operations from coming in * o Allocate/initate info needed by TPI * o Quiesce the connection, at which point we sync * state and move data * o Change operations (sonodeops) associated with the socket * o Unblock threads waiting for the fallback to finish */ int so_tpi_fallback(struct sonode *so, struct cred *cr) { int error; queue_t *q; struct sockparams *sp; struct sockparams *newsp = NULL; so_proto_fallback_func_t fbfunc; const char *devpath; boolean_t direct; struct sonode *nso; sock_quiesce_arg_t arg = { NULL, NULL }; #ifdef DEBUG struct sonode origso; #endif error = 0; sp = so->so_sockparams; fbfunc = sp->sp_smod_info->smod_proto_fallback_func; /* * Cannot fallback if the socket has active filters or a krecv callback. */ if (so->so_filter_active > 0 || so->so_krecv_cb != NULL) return (EINVAL); switch (so->so_family) { case AF_INET: devpath = sp->sp_smod_info->smod_fallback_devpath_v4; break; case AF_INET6: devpath = sp->sp_smod_info->smod_fallback_devpath_v6; break; default: return (EINVAL); } /* * Fallback can only happen if the socket module has a TPI device * and fallback function. */ if (devpath == NULL || fbfunc == NULL) return (EINVAL); /* * Initiate fallback; upon success we know that no new requests * will come in from the user. */ if (!so_start_fallback(so)) return (EAGAIN); #ifdef DEBUG /* * Make a copy of the sonode in case we need to make an integrity * check later on. */ bcopy(so, &origso, sizeof (*so)); #endif sp->sp_stats.sps_nfallback.value.ui64++; newsp = sockparams_hold_ephemeral_bydev(so->so_family, so->so_type, so->so_protocol, devpath, KM_SLEEP, &error); if (error != 0) goto out; if (so->so_direct != NULL) { sodirect_t *sodp = so->so_direct; mutex_enter(&so->so_lock); so->so_direct->sod_enabled = B_FALSE; so->so_state &= ~SS_SODIRECT; ASSERT(sodp->sod_uioafh == NULL); mutex_exit(&so->so_lock); } /* Turn sonode into a TPI socket */ error = sotpi_convert_sonode(so, newsp, &direct, &q, cr); if (error != 0) goto out; /* * When it comes to urgent data we have two cases to deal with; * (1) The oob byte has already arrived, or (2) the protocol has * notified that oob data is pending, but it has not yet arrived. * * For (1) all we need to do is send a T_EXDATA_IND to indicate were * in the byte stream the oob byte is. For (2) we have to send a * SIGURG (M_PCSIG), followed by a zero-length mblk indicating whether * the oob byte will be the next byte from the protocol. * * So in the worst case we need two mblks, one for the signal, another * for mark indication. In that case we use the exdata_mp for the sig. */ arg.soqa_exdata_mp = allocb_wait(sizeof (struct T_exdata_ind), BPRI_MED, STR_NOSIG, NULL); arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED, STR_NOSIG, NULL); /* * Now tell the protocol to start using TPI. so_quiesced_cb be * called once it's safe to synchronize state. */ DTRACE_PROBE1(proto__fallback__begin, struct sonode *, so); error = (*fbfunc)(so->so_proto_handle, q, direct, so_quiesced_cb, &arg); DTRACE_PROBE1(proto__fallback__end, struct sonode *, so); if (error != 0) { /* protocol was unable to do a fallback, revert the sonode */ sotpi_revert_sonode(so, cr); goto out; } /* * Walk the accept queue and notify the proto that they should * fall back to TPI. The protocol will send up the T_CONN_IND. */ nso = list_head(&so->so_acceptq_list); while (nso != NULL) { int rval; struct sonode *next; if (arg.soqa_exdata_mp == NULL) { arg.soqa_exdata_mp = allocb_wait(sizeof (struct T_exdata_ind), BPRI_MED, STR_NOSIG, NULL); } if (arg.soqa_urgmark_mp == NULL) { arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED, STR_NOSIG, NULL); } DTRACE_PROBE1(proto__fallback__begin, struct sonode *, nso); rval = (*fbfunc)(nso->so_proto_handle, NULL, direct, so_quiesced_cb, &arg); DTRACE_PROBE1(proto__fallback__end, struct sonode *, nso); if (rval != 0) { /* Abort the connection */ zcmn_err(getzoneid(), CE_WARN, "Failed to convert socket in accept queue to TPI. " "Pid = %d\n", curproc->p_pid); next = list_next(&so->so_acceptq_list, nso); list_remove(&so->so_acceptq_list, nso); so->so_acceptq_len--; (void) socket_close(nso, 0, CRED()); socket_destroy(nso); nso = next; } else { nso = list_next(&so->so_acceptq_list, nso); } } /* * Now flush the acceptq, this will destroy all sockets. They will * be recreated in sotpi_accept(). */ so_acceptq_flush(so, B_FALSE); mutex_enter(&so->so_lock); so->so_state |= SS_FALLBACK_COMP; mutex_exit(&so->so_lock); /* * Swap the sonode ops. Socket opertations that come in once this * is done will proceed without blocking. */ so->so_ops = &sotpi_sonodeops; /* * Wake up any threads stuck in poll. This is needed since the poll * head changes when the fallback happens (moves from the sonode to * the STREAMS head). */ pollwakeup(&so->so_poll_list, POLLERR); /* * When this non-STREAM socket was created we placed an extra ref on * the associated vnode to support asynchronous close. Drop that ref * here. */ ASSERT(SOTOV(so)->v_count >= 2); VN_RELE(SOTOV(so)); out: so_end_fallback(so); if (error != 0) { #ifdef DEBUG so_integrity_check(so, &origso); #endif zcmn_err(getzoneid(), CE_WARN, "Failed to convert socket to TPI (err=%d). Pid = %d\n", error, curproc->p_pid); if (newsp != NULL) SOCKPARAMS_DEC_REF(newsp); } if (arg.soqa_exdata_mp != NULL) freemsg(arg.soqa_exdata_mp); if (arg.soqa_urgmark_mp != NULL) freemsg(arg.soqa_urgmark_mp); return (error); } int so_krecv_set(sonode_t *so, so_krecv_f cb, void *arg) { int ret; if (cb == NULL && arg != NULL) return (EINVAL); SO_BLOCK_FALLBACK(so, so_krecv_set(so, cb, arg)); mutex_enter(&so->so_lock); if (so->so_state & SS_FALLBACK_COMP) { mutex_exit(&so->so_lock); SO_UNBLOCK_FALLBACK(so); return (ENOTSUP); } ret = so_lock_read(so, 0); VERIFY(ret == 0); /* * Other consumers may actually care about getting extant data delivered * to them, when they come along, they should figure out the best API * for that. */ so_rcv_flush(so); so->so_krecv_cb = cb; so->so_krecv_arg = arg; so_unlock_read(so); mutex_exit(&so->so_lock); SO_UNBLOCK_FALLBACK(so); return (0); } void so_krecv_unblock(sonode_t *so) { mutex_enter(&so->so_lock); VERIFY(so->so_krecv_cb != NULL); so->so_rcv_queued = 0; (void) so_check_flow_control(so); /* * so_check_flow_control() always drops so->so_lock, so we won't * need to drop it ourselves. */ }