/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SOCK_TEST int do_useracc = 1; /* Controlled by setting SO_DEBUG to 4 */ #else #define do_useracc 1 #endif /* SOCK_TEST */ extern int xnet_truncate_print; /* * Note: DEF_IOV_MAX is defined and used as it is in "fs/vncalls.c" * as there isn't a formal definition of IOV_MAX ??? */ #define MSG_MAXIOVLEN 16 /* * Kernel component of socket creation. * * The socket library determines which version number to use. * First the library calls this with a NULL devpath. If this fails * to find a transport (using solookup) the library will look in /etc/netconfig * for the appropriate transport. If one is found it will pass in the * devpath for the kernel to use. */ int so_socket(int domain, int type, int protocol, char *devpath, int version) { vnode_t *accessvp; struct sonode *so; vnode_t *vp; struct file *fp; int fd; int error; boolean_t wildcard = B_FALSE; int saved_error = 0; int sdomain = domain; dprint(1, ("so_socket(%d,%d,%d,%p,%d)\n", domain, type, protocol, devpath, version)); if (domain == AF_NCA) { /* * The request is for an NCA socket so for NL7C use the * INET domain instead and mark NL7C_AF_NCA below. */ domain = AF_INET; /* * NL7C is not supported in non-global zones, * we enforce this restriction here. */ if (getzoneid() != GLOBAL_ZONEID) { return (set_errno(ENOTSUP)); } } accessvp = solookup(domain, type, protocol, devpath, &error); if (accessvp == NULL) { /* * If there is either an EPROTONOSUPPORT or EPROTOTYPE error * it makes sense doing the wildcard lookup since the * protocol might not be in the table. */ if (devpath != NULL || protocol == 0 || !(error == EPROTONOSUPPORT || error == EPROTOTYPE)) return (set_errno(error)); saved_error = error; /* * Try wildcard lookup. Never use devpath for wildcards. */ accessvp = solookup(domain, type, 0, NULL, &error); if (accessvp == NULL) { /* * Can't find in kernel table - have library * fall back to /etc/netconfig and tell us * the devpath (The library will do this if it didn't * already pass in a devpath). */ if (saved_error != 0) error = saved_error; return (set_errno(error)); } wildcard = B_TRUE; } /* Check the device policy */ if ((error = secpolicy_spec_open(CRED(), accessvp, FREAD|FWRITE)) != 0) { return (set_errno(error)); } if (protocol == IPPROTO_SCTP) { so = sosctp_create(accessvp, domain, type, protocol, version, NULL, &error); } else if (protocol == PROTO_SDP) { so = sosdp_create(accessvp, domain, type, protocol, version, NULL, &error); } else { so = sotpi_create(accessvp, domain, type, protocol, version, NULL, &error); } if (so == NULL) { return (set_errno(error)); } if (sdomain == AF_NCA && domain == AF_INET) { so->so_nl7c_flags = NL7C_AF_NCA; } vp = SOTOV(so); if (wildcard) { /* * Issue SO_PROTOTYPE setsockopt. */ error = SOP_SETSOCKOPT(so, SOL_SOCKET, SO_PROTOTYPE, &protocol, (t_uscalar_t)sizeof (protocol)); if (error) { (void) VOP_CLOSE(vp, 0, 1, 0, CRED()); VN_RELE(vp); /* * Setsockopt often fails with ENOPROTOOPT but socket() * should fail with EPROTONOSUPPORT/EPROTOTYPE. */ if (saved_error != 0 && error == ENOPROTOOPT) error = saved_error; else error = EPROTONOSUPPORT; return (set_errno(error)); } } if (error = falloc(vp, FWRITE|FREAD, &fp, &fd)) { (void) VOP_CLOSE(vp, 0, 1, 0, CRED()); VN_RELE(vp); return (set_errno(error)); } /* * Now fill in the entries that falloc reserved */ mutex_exit(&fp->f_tlock); setf(fd, fp); return (fd); } /* * Map from a file descriptor to a socket node. * Returns with the file descriptor held i.e. the caller has to * use releasef when done with the file descriptor. */ static struct sonode * getsonode(int sock, int *errorp, file_t **fpp) { file_t *fp; vnode_t *vp; struct sonode *so; if ((fp = getf(sock)) == NULL) { *errorp = EBADF; eprintline(*errorp); return (NULL); } vp = fp->f_vnode; /* Check if it is a socket */ if (vp->v_type != VSOCK) { releasef(sock); *errorp = ENOTSOCK; eprintline(*errorp); return (NULL); } /* * Use the stream head to find the real socket vnode. * This is needed when namefs sits above sockfs. */ if (vp->v_stream) { ASSERT(vp->v_stream->sd_vnode); vp = vp->v_stream->sd_vnode; so = VTOSO(vp); if (so->so_version == SOV_STREAM) { releasef(sock); *errorp = ENOTSOCK; eprintsoline(so, *errorp); return (NULL); } } else { so = VTOSO(vp); } if (fpp) *fpp = fp; return (so); } /* * Allocate and copyin a sockaddr. * Ensures NULL termination for AF_UNIX addresses by extending them * with one NULL byte if need be. Verifies that the length is not * excessive to prevent an application from consuming all of kernel * memory. Returns NULL when an error occurred. */ static struct sockaddr * copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp, int *errorp) { char *faddr; size_t namelen = (size_t)*namelenp; ASSERT(namelen != 0); if (namelen > SO_MAXARGSIZE) { *errorp = EINVAL; eprintsoline(so, *errorp); return (NULL); } faddr = (char *)kmem_alloc(namelen, KM_SLEEP); if (copyin(name, faddr, namelen)) { kmem_free(faddr, namelen); *errorp = EFAULT; eprintsoline(so, *errorp); return (NULL); } /* * Add space for NULL termination if needed. * Do a quick check if the last byte is NUL. */ if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') { /* Check if there is any NULL termination */ size_t i; int foundnull = 0; for (i = sizeof (name->sa_family); i < namelen; i++) { if (faddr[i] == '\0') { foundnull = 1; break; } } if (!foundnull) { /* Add extra byte for NUL padding */ char *nfaddr; nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP); bcopy(faddr, nfaddr, namelen); kmem_free(faddr, namelen); /* NUL terminate */ nfaddr[namelen] = '\0'; namelen++; ASSERT((socklen_t)namelen == namelen); *namelenp = (socklen_t)namelen; faddr = nfaddr; } } return ((struct sockaddr *)faddr); } /* * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL. */ static int copyout_arg(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr, socklen_t klen) { if (uaddr != NULL) { if (ulen > klen) ulen = klen; if (ulen != 0) { if (copyout(kaddr, uaddr, ulen)) return (EFAULT); } } else ulen = 0; if (ulenp != NULL) { if (copyout(&ulen, ulenp, sizeof (ulen))) return (EFAULT); } return (0); } /* * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL. * If klen is greater than ulen it still uses the non-truncated * klen to update ulenp. */ static int copyout_name(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr, socklen_t klen) { if (uaddr != NULL) { if (ulen >= klen) ulen = klen; else if (ulen != 0 && xnet_truncate_print) { printf("sockfs: truncating copyout of address using " "XNET semantics for pid = %d. Lengths %d, %d\n", curproc->p_pid, klen, ulen); } if (ulen != 0) { if (copyout(kaddr, uaddr, ulen)) return (EFAULT); } else klen = 0; } else klen = 0; if (ulenp != NULL) { if (copyout(&klen, ulenp, sizeof (klen))) return (EFAULT); } return (0); } /* * The socketpair() code in libsocket creates two sockets (using * the /etc/netconfig fallback if needed) before calling this routine * to connect the two sockets together. * * For a SOCK_STREAM socketpair a listener is needed - in that case this * routine will create a new file descriptor as part of accepting the * connection. The library socketpair() will check if svs[2] has changed * in which case it will close the changed fd. * * Note that this code could use the TPI feature of accepting the connection * on the listening endpoint. However, that would require significant changes * to soaccept. */ int so_socketpair(int sv[2]) { int svs[2]; struct sonode *so1, *so2; int error; struct sockaddr_ux *name; size_t namelen; dprint(1, ("so_socketpair(%p)\n", sv)); error = useracc(sv, sizeof (svs), B_WRITE); if (error && do_useracc) return (set_errno(EFAULT)); if (copyin(sv, svs, sizeof (svs))) return (set_errno(EFAULT)); if ((so1 = getsonode(svs[0], &error, NULL)) == NULL) return (set_errno(error)); if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) { releasef(svs[0]); return (set_errno(error)); } if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) { error = EOPNOTSUPP; goto done; } /* * The code below makes assumptions about the "sockfs" implementation. * So make sure that the correct implementation is really used. */ ASSERT(so1->so_ops == &sotpi_sonodeops); ASSERT(so2->so_ops == &sotpi_sonodeops); if (so1->so_type == SOCK_DGRAM) { /* * Bind both sockets and connect them with each other. * Need to allocate name/namelen for soconnect. */ error = SOP_BIND(so1, NULL, 0, _SOBIND_UNSPEC); if (error) { eprintsoline(so1, error); goto done; } error = SOP_BIND(so2, NULL, 0, _SOBIND_UNSPEC); if (error) { eprintsoline(so2, error); goto done; } namelen = sizeof (struct sockaddr_ux); name = kmem_alloc(namelen, KM_SLEEP); name->sou_family = AF_UNIX; name->sou_addr = so2->so_ux_laddr; error = SOP_CONNECT(so1, (struct sockaddr *)name, (socklen_t)namelen, 0, _SOCONNECT_NOXLATE); if (error) { kmem_free(name, namelen); eprintsoline(so1, error); goto done; } name->sou_addr = so1->so_ux_laddr; error = SOP_CONNECT(so2, (struct sockaddr *)name, (socklen_t)namelen, 0, _SOCONNECT_NOXLATE); kmem_free(name, namelen); if (error) { eprintsoline(so2, error); goto done; } releasef(svs[0]); releasef(svs[1]); } else { /* * Bind both sockets, with so1 being a listener. * Connect so2 to so1 - nonblocking to avoid waiting for * soaccept to complete. * Accept a connection on so1. Pass out the new fd as sv[0]. * The library will detect the changed fd and close * the original one. */ struct sonode *nso; struct vnode *nvp; struct file *nfp; int nfd; /* * We could simply call SOP_LISTEN() here (which would do the * binding automatically) if the code didn't rely on passing * _SOBIND_NOXLATE to the TPI implementation of SOP_BIND(). */ error = SOP_BIND(so1, NULL, 0, _SOBIND_UNSPEC|_SOBIND_NOXLATE| _SOBIND_LISTEN|_SOBIND_SOCKETPAIR); if (error) { eprintsoline(so1, error); goto done; } error = SOP_BIND(so2, NULL, 0, _SOBIND_UNSPEC); if (error) { eprintsoline(so2, error); goto done; } namelen = sizeof (struct sockaddr_ux); name = kmem_alloc(namelen, KM_SLEEP); name->sou_family = AF_UNIX; name->sou_addr = so1->so_ux_laddr; error = SOP_CONNECT(so2, (struct sockaddr *)name, (socklen_t)namelen, FNONBLOCK, _SOCONNECT_NOXLATE); kmem_free(name, namelen); if (error) { if (error != EINPROGRESS) { eprintsoline(so2, error); goto done; } } error = SOP_ACCEPT(so1, 0, &nso); if (error) { eprintsoline(so1, error); goto done; } /* wait for so2 being SS_CONNECTED ignoring signals */ mutex_enter(&so2->so_lock); error = sowaitconnected(so2, 0, 1); mutex_exit(&so2->so_lock); nvp = SOTOV(nso); if (error != 0) { (void) VOP_CLOSE(nvp, 0, 1, 0, CRED()); VN_RELE(nvp); eprintsoline(so2, error); goto done; } if (error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd)) { (void) VOP_CLOSE(nvp, 0, 1, 0, CRED()); VN_RELE(nvp); eprintsoline(nso, error); goto done; } /* * fill in the entries that falloc reserved */ mutex_exit(&nfp->f_tlock); setf(nfd, nfp); releasef(svs[0]); releasef(svs[1]); svs[0] = nfd; /* * The socketpair library routine will close the original * svs[0] when this code passes out a different file * descriptor. */ if (copyout(svs, sv, sizeof (svs))) { (void) closeandsetf(nfd, NULL); eprintline(EFAULT); return (set_errno(EFAULT)); } } return (0); done: releasef(svs[0]); releasef(svs[1]); return (set_errno(error)); } int bind(int sock, struct sockaddr *name, socklen_t namelen, int version) { struct sonode *so; int error; dprint(1, ("bind(%d, %p, %d)\n", sock, name, namelen)); if ((so = getsonode(sock, &error, NULL)) == NULL) return (set_errno(error)); /* Allocate and copyin name */ /* * X/Open test does not expect EFAULT with NULL name and non-zero * namelen. */ if (name != NULL && namelen != 0) { ASSERT(MUTEX_NOT_HELD(&so->so_lock)); name = copyin_name(so, name, &namelen, &error); if (name == NULL) { releasef(sock); return (set_errno(error)); } } else { name = NULL; namelen = 0; } switch (version) { default: error = SOP_BIND(so, name, namelen, 0); break; case SOV_XPG4_2: error = SOP_BIND(so, name, namelen, _SOBIND_XPG4_2); break; case SOV_SOCKBSD: error = SOP_BIND(so, name, namelen, _SOBIND_SOCKBSD); break; } done: releasef(sock); if (name != NULL) kmem_free(name, (size_t)namelen); if (error) return (set_errno(error)); return (0); } /* ARGSUSED2 */ int listen(int sock, int backlog, int version) { struct sonode *so; int error; dprint(1, ("listen(%d, %d)\n", sock, backlog)); if ((so = getsonode(sock, &error, NULL)) == NULL) return (set_errno(error)); error = SOP_LISTEN(so, backlog); releasef(sock); if (error) return (set_errno(error)); return (0); } /*ARGSUSED3*/ int accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version) { struct sonode *so; file_t *fp; int error; socklen_t namelen; struct sonode *nso; struct vnode *nvp; struct file *nfp; int nfd; dprint(1, ("accept(%d, %p, %p)\n", sock, name, namelenp)); if ((so = getsonode(sock, &error, &fp)) == NULL) return (set_errno(error)); if (name != NULL) { ASSERT(MUTEX_NOT_HELD(&so->so_lock)); if (copyin(namelenp, &namelen, sizeof (namelen))) { releasef(sock); return (set_errno(EFAULT)); } if (namelen != 0) { error = useracc(name, (size_t)namelen, B_WRITE); if (error && do_useracc) { releasef(sock); return (set_errno(EFAULT)); } } else name = NULL; } else { namelen = 0; } /* * Allocate the user fd before SOP_ACCEPT() in order to * catch EMFILE errors before calling SOP_ACCEPT(). */ if ((nfd = ufalloc(0)) == -1) { eprintsoline(so, EMFILE); releasef(sock); return (set_errno(EMFILE)); } error = SOP_ACCEPT(so, fp->f_flag, &nso); releasef(sock); if (error) { setf(nfd, NULL); return (set_errno(error)); } nvp = SOTOV(nso); /* * so_faddr_sa can not go away even though we are not holding so_lock. * However, in theory its content could change from underneath us. * But this is not possible in practice since it can only * change due to either some socket system call * or due to a T_CONN_CON being received from the stream head. * Since the falloc/setf have not yet been done no thread * can do any system call on nso and T_CONN_CON can not arrive * on a socket that is already connected. * Thus there is no reason to hold so_lock here. * * SOP_ACCEPT() is required to have set the valid bit for the faddr, * but it could be instantly cleared by a disconnect from the transport. * For that reason we ignore it here. */ ASSERT(MUTEX_NOT_HELD(&nso->so_lock)); error = copyout_name(name, namelen, namelenp, nso->so_faddr_sa, (socklen_t)nso->so_faddr_len); if (error) { setf(nfd, NULL); (void) VOP_CLOSE(nvp, 0, 1, 0, CRED()); VN_RELE(nvp); return (set_errno(error)); } if (error = falloc(NULL, FWRITE|FREAD, &nfp, NULL)) { setf(nfd, NULL); (void) VOP_CLOSE(nvp, 0, 1, 0, CRED()); VN_RELE(nvp); eprintsoline(so, error); return (set_errno(error)); } /* * fill in the entries that falloc reserved */ nfp->f_vnode = nvp; mutex_exit(&nfp->f_tlock); setf(nfd, nfp); /* * Copy FNDELAY and FNONBLOCK from listener to acceptor */ if (so->so_state & (SS_NDELAY|SS_NONBLOCK)) { uint_t oflag = nfp->f_flag; int arg = 0; if (so->so_state & SS_NONBLOCK) arg |= FNONBLOCK; else if (so->so_state & SS_NDELAY) arg |= FNDELAY; /* * This code is a simplification of the F_SETFL code in fcntl() * Ignore any errors from VOP_SETFL. */ if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred)) != 0) { eprintsoline(so, error); error = 0; } else { mutex_enter(&nfp->f_tlock); nfp->f_flag &= ~FMASK | (FREAD|FWRITE); nfp->f_flag |= arg; mutex_exit(&nfp->f_tlock); } } return (nfd); } int connect(int sock, struct sockaddr *name, socklen_t namelen, int version) { struct sonode *so; file_t *fp; int error; dprint(1, ("connect(%d, %p, %d)\n", sock, name, namelen)); if ((so = getsonode(sock, &error, &fp)) == NULL) return (set_errno(error)); /* Allocate and copyin name */ if (namelen != 0) { ASSERT(MUTEX_NOT_HELD(&so->so_lock)); name = copyin_name(so, name, &namelen, &error); if (name == NULL) { releasef(sock); return (set_errno(error)); } } else name = NULL; error = SOP_CONNECT(so, name, namelen, fp->f_flag, (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2); releasef(sock); if (name) kmem_free(name, (size_t)namelen); if (error) return (set_errno(error)); return (0); } /*ARGSUSED2*/ int shutdown(int sock, int how, int version) { struct sonode *so; int error; dprint(1, ("shutdown(%d, %d)\n", sock, how)); if ((so = getsonode(sock, &error, NULL)) == NULL) return (set_errno(error)); error = SOP_SHUTDOWN(so, how); releasef(sock); if (error) return (set_errno(error)); return (0); } /* * Common receive routine. */ static ssize_t recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags, socklen_t *namelenp, socklen_t *controllenp, int *flagsp) { struct sonode *so; file_t *fp; void *name; socklen_t namelen; void *control; socklen_t controllen; ssize_t len; int error; if ((so = getsonode(sock, &error, &fp)) == NULL) return (set_errno(error)); len = uiop->uio_resid; uiop->uio_fmode = fp->f_flag; uiop->uio_extflg = UIO_COPY_CACHED; name = msg->msg_name; namelen = msg->msg_namelen; control = msg->msg_control; controllen = msg->msg_controllen; msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL | MSG_DONTWAIT | MSG_XPG4_2); error = SOP_RECVMSG(so, msg, uiop); if (error) { releasef(sock); return (set_errno(error)); } lwp_stat_update(LWP_STAT_MSGRCV, 1); so_update_attrs(so, SOACC); releasef(sock); error = copyout_name(name, namelen, namelenp, msg->msg_name, msg->msg_namelen); if (error) goto err; if (flagsp != NULL) { /* * Clear internal flag. */ msg->msg_flags &= ~MSG_XPG4_2; /* * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only * when controllen is zero and there is control data to * copy out. */ if (controllen != 0 && (msg->msg_controllen > controllen || control == NULL)) { dprint(1, ("recvit: CTRUNC %d %d %p\n", msg->msg_controllen, controllen, control)); msg->msg_flags |= MSG_CTRUNC; } if (copyout(&msg->msg_flags, flagsp, sizeof (msg->msg_flags))) { error = EFAULT; goto err; } } /* * Note: This MUST be done last. There can be no "goto err" after this * point since it could make so_closefds run twice on some part * of the file descriptor array. */ if (controllen != 0) { if (!(flags & MSG_XPG4_2)) { /* * Good old msg_accrights can only return a multiple * of 4 bytes. */ controllen &= ~((int)sizeof (uint32_t) - 1); } error = copyout_arg(control, controllen, controllenp, msg->msg_control, msg->msg_controllen); if (error) goto err; if (msg->msg_controllen > controllen || control == NULL) { if (control == NULL) controllen = 0; so_closefds(msg->msg_control, msg->msg_controllen, !(flags & MSG_XPG4_2), controllen); } } if (msg->msg_namelen != 0) kmem_free(msg->msg_name, (size_t)msg->msg_namelen); if (msg->msg_controllen != 0) kmem_free(msg->msg_control, (size_t)msg->msg_controllen); return (len - uiop->uio_resid); err: /* * If we fail and the control part contains file descriptors * we have to close the fd's. */ if (msg->msg_controllen != 0) so_closefds(msg->msg_control, msg->msg_controllen, !(flags & MSG_XPG4_2), 0); if (msg->msg_namelen != 0) kmem_free(msg->msg_name, (size_t)msg->msg_namelen); if (msg->msg_controllen != 0) kmem_free(msg->msg_control, (size_t)msg->msg_controllen); return (set_errno(error)); } /* * Native system call */ ssize_t recv(int sock, void *buffer, size_t len, int flags) { struct nmsghdr lmsg; struct uio auio; struct iovec aiov[1]; dprint(1, ("recv(%d, %p, %ld, %d)\n", sock, buffer, len, flags)); if ((ssize_t)len < 0) { return (set_errno(EINVAL)); } aiov[0].iov_base = buffer; aiov[0].iov_len = len; auio.uio_loffset = 0; auio.uio_iov = aiov; auio.uio_iovcnt = 1; auio.uio_resid = len; auio.uio_segflg = UIO_USERSPACE; auio.uio_limit = 0; lmsg.msg_namelen = 0; lmsg.msg_controllen = 0; lmsg.msg_flags = 0; return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL)); } ssize_t recvfrom(int sock, void *buffer, size_t len, int flags, struct sockaddr *name, socklen_t *namelenp) { struct nmsghdr lmsg; struct uio auio; struct iovec aiov[1]; dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n", sock, buffer, len, flags, name, namelenp)); if ((ssize_t)len < 0) { return (set_errno(EINVAL)); } aiov[0].iov_base = buffer; aiov[0].iov_len = len; auio.uio_loffset = 0; auio.uio_iov = aiov; auio.uio_iovcnt = 1; auio.uio_resid = len; auio.uio_segflg = UIO_USERSPACE; auio.uio_limit = 0; lmsg.msg_name = (char *)name; if (namelenp != NULL) { if (copyin(namelenp, &lmsg.msg_namelen, sizeof (lmsg.msg_namelen))) return (set_errno(EFAULT)); } else { lmsg.msg_namelen = 0; } lmsg.msg_controllen = 0; lmsg.msg_flags = 0; return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL)); } /* * Uses the MSG_XPG4_2 flag to determine if the caller is using * struct omsghdr or struct nmsghdr. */ ssize_t recvmsg(int sock, struct nmsghdr *msg, int flags) { STRUCT_DECL(nmsghdr, u_lmsg); STRUCT_HANDLE(nmsghdr, umsgptr); struct nmsghdr lmsg; struct uio auio; struct iovec aiov[MSG_MAXIOVLEN]; int iovcnt; ssize_t len; int i; int *flagsp; model_t model; dprint(1, ("recvmsg(%d, %p, %d)\n", sock, msg, flags)); model = get_udatamodel(); STRUCT_INIT(u_lmsg, model); STRUCT_SET_HANDLE(umsgptr, model, msg); if (flags & MSG_XPG4_2) { if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg))) return (set_errno(EFAULT)); flagsp = STRUCT_FADDR(umsgptr, msg_flags); } else { /* * Assumes that nmsghdr and omsghdr are identically shaped * except for the added msg_flags field. */ if (copyin(msg, STRUCT_BUF(u_lmsg), SIZEOF_STRUCT(omsghdr, model))) return (set_errno(EFAULT)); STRUCT_FSET(u_lmsg, msg_flags, 0); flagsp = NULL; } /* * Code below us will kmem_alloc memory and hang it * off msg_control and msg_name fields. This forces * us to copy the structure to its native form. */ lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name); lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen); lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov); lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen); lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control); lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen); lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags); iovcnt = lmsg.msg_iovlen; if (iovcnt <= 0 || iovcnt > MSG_MAXIOVLEN) { return (set_errno(EMSGSIZE)); } #ifdef _SYSCALL32_IMPL /* * 32-bit callers need to have their iovec expanded, while ensuring * that they can't move more than 2Gbytes of data in a single call. */ if (model == DATAMODEL_ILP32) { struct iovec32 aiov32[MSG_MAXIOVLEN]; ssize32_t count32; if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iovcnt * sizeof (struct iovec32))) return (set_errno(EFAULT)); count32 = 0; for (i = 0; i < iovcnt; i++) { ssize32_t iovlen32; iovlen32 = aiov32[i].iov_len; count32 += iovlen32; if (iovlen32 < 0 || count32 < 0) return (set_errno(EINVAL)); aiov[i].iov_len = iovlen32; aiov[i].iov_base = (caddr_t)(uintptr_t)aiov32[i].iov_base; } } else #endif /* _SYSCALL32_IMPL */ if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) { return (set_errno(EFAULT)); } len = 0; for (i = 0; i < iovcnt; i++) { ssize_t iovlen = aiov[i].iov_len; len += iovlen; if (iovlen < 0 || len < 0) { return (set_errno(EINVAL)); } } auio.uio_loffset = 0; auio.uio_iov = aiov; auio.uio_iovcnt = iovcnt; auio.uio_resid = len; auio.uio_segflg = UIO_USERSPACE; auio.uio_limit = 0; if (lmsg.msg_control != NULL && (do_useracc == 0 || useracc(lmsg.msg_control, lmsg.msg_controllen, B_WRITE) != 0)) { return (set_errno(EFAULT)); } return (recvit(sock, &lmsg, &auio, flags, STRUCT_FADDR(umsgptr, msg_namelen), STRUCT_FADDR(umsgptr, msg_controllen), flagsp)); } /* * Common send function. */ static ssize_t sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags) { struct sonode *so; file_t *fp; void *name; socklen_t namelen; void *control; socklen_t controllen; ssize_t len; int error; if ((so = getsonode(sock, &error, &fp)) == NULL) return (set_errno(error)); uiop->uio_fmode = fp->f_flag; if (so->so_family == AF_UNIX) uiop->uio_extflg = UIO_COPY_CACHED; else uiop->uio_extflg = UIO_COPY_DEFAULT; /* Allocate and copyin name and control */ name = msg->msg_name; namelen = msg->msg_namelen; if (name != NULL && namelen != 0) { ASSERT(MUTEX_NOT_HELD(&so->so_lock)); name = copyin_name(so, (struct sockaddr *)name, &namelen, &error); if (name == NULL) goto done3; /* copyin_name null terminates addresses for AF_UNIX */ msg->msg_namelen = namelen; msg->msg_name = name; } else { msg->msg_name = name = NULL; msg->msg_namelen = namelen = 0; } control = msg->msg_control; controllen = msg->msg_controllen; if ((control != NULL) && (controllen != 0)) { /* * Verify that the length is not excessive to prevent * an application from consuming all of kernel memory. */ if (controllen > SO_MAXARGSIZE) { error = EINVAL; goto done2; } control = kmem_alloc(controllen, KM_SLEEP); ASSERT(MUTEX_NOT_HELD(&so->so_lock)); if (copyin(msg->msg_control, control, controllen)) { error = EFAULT; goto done1; } msg->msg_control = control; } else { msg->msg_control = control = NULL; msg->msg_controllen = controllen = 0; } len = uiop->uio_resid; msg->msg_flags = flags; error = SOP_SENDMSG(so, msg, uiop); done1: if (control != NULL) kmem_free(control, controllen); done2: if (name != NULL) kmem_free(name, namelen); done3: if (error != 0) { releasef(sock); return (set_errno(error)); } lwp_stat_update(LWP_STAT_MSGSND, 1); so_update_attrs(so, SOMOD); releasef(sock); return (len - uiop->uio_resid); } /* * Native system call */ ssize_t send(int sock, void *buffer, size_t len, int flags) { struct nmsghdr lmsg; struct uio auio; struct iovec aiov[1]; dprint(1, ("send(%d, %p, %ld, %d)\n", sock, buffer, len, flags)); if ((ssize_t)len < 0) { return (set_errno(EINVAL)); } aiov[0].iov_base = buffer; aiov[0].iov_len = len; auio.uio_loffset = 0; auio.uio_iov = aiov; auio.uio_iovcnt = 1; auio.uio_resid = len; auio.uio_segflg = UIO_USERSPACE; auio.uio_limit = 0; lmsg.msg_name = NULL; lmsg.msg_control = NULL; if (!(flags & MSG_XPG4_2)) { /* * In order to be compatible with the libsocket/sockmod * implementation we set EOR for all send* calls. */ flags |= MSG_EOR; } return (sendit(sock, &lmsg, &auio, flags)); } /* * Uses the MSG_XPG4_2 flag to determine if the caller is using * struct omsghdr or struct nmsghdr. */ ssize_t sendmsg(int sock, struct nmsghdr *msg, int flags) { struct nmsghdr lmsg; STRUCT_DECL(nmsghdr, u_lmsg); struct uio auio; struct iovec aiov[MSG_MAXIOVLEN]; int iovcnt; ssize_t len; int i; model_t model; dprint(1, ("sendmsg(%d, %p, %d)\n", sock, msg, flags)); model = get_udatamodel(); STRUCT_INIT(u_lmsg, model); if (flags & MSG_XPG4_2) { if (copyin(msg, (char *)STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg))) return (set_errno(EFAULT)); } else { /* * Assumes that nmsghdr and omsghdr are identically shaped * except for the added msg_flags field. */ if (copyin(msg, (char *)STRUCT_BUF(u_lmsg), SIZEOF_STRUCT(omsghdr, model))) return (set_errno(EFAULT)); /* * In order to be compatible with the libsocket/sockmod * implementation we set EOR for all send* calls. */ flags |= MSG_EOR; } /* * Code below us will kmem_alloc memory and hang it * off msg_control and msg_name fields. This forces * us to copy the structure to its native form. */ lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name); lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen); lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov); lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen); lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control); lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen); lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags); iovcnt = lmsg.msg_iovlen; if (iovcnt <= 0 || iovcnt > MSG_MAXIOVLEN) { /* * Unless this is XPG 4.2 we allow iovcnt == 0 to * be compatible with SunOS 4.X and 4.4BSD. */ if (iovcnt != 0 || (flags & MSG_XPG4_2)) return (set_errno(EMSGSIZE)); } #ifdef _SYSCALL32_IMPL /* * 32-bit callers need to have their iovec expanded, while ensuring * that they can't move more than 2Gbytes of data in a single call. */ if (model == DATAMODEL_ILP32) { struct iovec32 aiov32[MSG_MAXIOVLEN]; ssize32_t count32; if (iovcnt != 0 && copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iovcnt * sizeof (struct iovec32))) return (set_errno(EFAULT)); count32 = 0; for (i = 0; i < iovcnt; i++) { ssize32_t iovlen32; iovlen32 = aiov32[i].iov_len; count32 += iovlen32; if (iovlen32 < 0 || count32 < 0) return (set_errno(EINVAL)); aiov[i].iov_len = iovlen32; aiov[i].iov_base = (caddr_t)(uintptr_t)aiov32[i].iov_base; } } else #endif /* _SYSCALL32_IMPL */ if (iovcnt != 0 && copyin(lmsg.msg_iov, aiov, (unsigned)iovcnt * sizeof (struct iovec))) { return (set_errno(EFAULT)); } len = 0; for (i = 0; i < iovcnt; i++) { ssize_t iovlen = aiov[i].iov_len; len += iovlen; if (iovlen < 0 || len < 0) { return (set_errno(EINVAL)); } } auio.uio_loffset = 0; auio.uio_iov = aiov; auio.uio_iovcnt = iovcnt; auio.uio_resid = len; auio.uio_segflg = UIO_USERSPACE; auio.uio_limit = 0; return (sendit(sock, &lmsg, &auio, flags)); } ssize_t sendto(int sock, void *buffer, size_t len, int flags, struct sockaddr *name, socklen_t namelen) { struct nmsghdr lmsg; struct uio auio; struct iovec aiov[1]; dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n", sock, buffer, len, flags, name, namelen)); if ((ssize_t)len < 0) { return (set_errno(EINVAL)); } aiov[0].iov_base = buffer; aiov[0].iov_len = len; auio.uio_loffset = 0; auio.uio_iov = aiov; auio.uio_iovcnt = 1; auio.uio_resid = len; auio.uio_segflg = UIO_USERSPACE; auio.uio_limit = 0; lmsg.msg_name = (char *)name; lmsg.msg_namelen = namelen; lmsg.msg_control = NULL; if (!(flags & MSG_XPG4_2)) { /* * In order to be compatible with the libsocket/sockmod * implementation we set EOR for all send* calls. */ flags |= MSG_EOR; } return (sendit(sock, &lmsg, &auio, flags)); } /*ARGSUSED3*/ int getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version) { struct sonode *so; int error; socklen_t namelen; union { struct sockaddr_in sin; struct sockaddr_in6 sin6; } sin; /* Temporary buffer, common case */ void *addr; /* Temporary buffer, uncommon case */ socklen_t addrlen, size; dprint(1, ("getpeername(%d, %p, %p)\n", sock, name, namelenp)); if ((so = getsonode(sock, &error, NULL)) == NULL) goto bad; ASSERT(MUTEX_NOT_HELD(&so->so_lock)); if (copyin(namelenp, &namelen, sizeof (namelen)) || (name == NULL && namelen != 0)) { error = EFAULT; goto rel_out; } /* * If a connect or accept has been done, unless we're an Xnet socket, * the remote address has already been updated in so_faddr_sa. */ if (so->so_version != SOV_SOCKSTREAM && so->so_version != SOV_SOCKBSD || !(so->so_state & SS_FADDR_VALID)) { if ((error = SOP_GETPEERNAME(so)) != 0) goto rel_out; } if (so->so_faddr_maxlen <= sizeof (sin)) { size = 0; addr = &sin; } else { /* * Allocate temporary to avoid holding so_lock across * copyout */ size = so->so_faddr_maxlen; addr = kmem_alloc(size, KM_SLEEP); } /* Prevent so_faddr_sa/len from changing while accessed */ mutex_enter(&so->so_lock); if (!(so->so_state & SS_ISCONNECTED)) { mutex_exit(&so->so_lock); error = ENOTCONN; goto free_out; } addrlen = so->so_faddr_len; bcopy(so->so_faddr_sa, addr, addrlen); mutex_exit(&so->so_lock); ASSERT(MUTEX_NOT_HELD(&so->so_lock)); error = copyout_name(name, namelen, namelenp, addr, (so->so_state & SS_FADDR_NOXLATE) ? 0 : addrlen); free_out: if (size != 0) kmem_free(addr, size); rel_out: releasef(sock); bad: return (error != 0 ? set_errno(error) : 0); } /*ARGSUSED3*/ int getsockname(int sock, struct sockaddr *name, socklen_t *namelenp, int version) { struct sonode *so; int error; socklen_t namelen; union { struct sockaddr_in sin; struct sockaddr_in6 sin6; } sin; /* Temporary buffer, common case */ void *addr; /* Temporary buffer, uncommon case */ socklen_t addrlen, size; dprint(1, ("getsockname(%d, %p, %p)\n", sock, name, namelenp)); if ((so = getsonode(sock, &error, NULL)) == NULL) goto bad; ASSERT(MUTEX_NOT_HELD(&so->so_lock)); if (copyin(namelenp, &namelen, sizeof (namelen)) || (name == NULL && namelen != 0)) { error = EFAULT; goto rel_out; } /* * If a bind or accept has been done, unless we're an Xnet endpoint, * the local address has already been updated in so_laddr_sa. */ if ((so->so_version != SOV_SOCKSTREAM && so->so_version != SOV_SOCKBSD) || !(so->so_state & SS_LADDR_VALID)) { if ((error = SOP_GETSOCKNAME(so)) != 0) goto rel_out; } if (so->so_laddr_maxlen <= sizeof (sin)) { size = 0; addr = &sin; } else { /* * Allocate temporary to avoid holding so_lock across * copyout */ size = so->so_laddr_maxlen; addr = kmem_alloc(size, KM_SLEEP); } /* Prevent so_laddr_sa/len from changing while accessed */ mutex_enter(&so->so_lock); addrlen = so->so_laddr_len; bcopy(so->so_laddr_sa, addr, addrlen); mutex_exit(&so->so_lock); ASSERT(MUTEX_NOT_HELD(&so->so_lock)); error = copyout_name(name, namelen, namelenp, addr, addrlen); if (size != 0) kmem_free(addr, size); rel_out: releasef(sock); bad: return (error != 0 ? set_errno(error) : 0); } /*ARGSUSED5*/ int getsockopt(int sock, int level, int option_name, void *option_value, socklen_t *option_lenp, int version) { struct sonode *so; socklen_t optlen, optlen_res; void *optval; int error; dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n", sock, level, option_name, option_value, option_lenp)); if ((so = getsonode(sock, &error, NULL)) == NULL) return (set_errno(error)); ASSERT(MUTEX_NOT_HELD(&so->so_lock)); if (copyin(option_lenp, &optlen, sizeof (optlen))) { releasef(sock); return (set_errno(EFAULT)); } /* * Verify that the length is not excessive to prevent * an application from consuming all of kernel memory. */ if (optlen > SO_MAXARGSIZE) { error = EINVAL; releasef(sock); return (set_errno(error)); } optval = kmem_alloc(optlen, KM_SLEEP); optlen_res = optlen; error = SOP_GETSOCKOPT(so, level, option_name, optval, &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2); releasef(sock); if (error) { kmem_free(optval, optlen); return (set_errno(error)); } error = copyout_arg(option_value, optlen, option_lenp, optval, optlen_res); kmem_free(optval, optlen); if (error) return (set_errno(error)); return (0); } /*ARGSUSED5*/ int setsockopt(int sock, int level, int option_name, void *option_value, socklen_t option_len, int version) { struct sonode *so; intptr_t buffer[2]; void *optval = NULL; int error; dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n", sock, level, option_name, option_value, option_len)); if ((so = getsonode(sock, &error, NULL)) == NULL) return (set_errno(error)); if (option_value != NULL) { if (option_len != 0) { /* * Verify that the length is not excessive to prevent * an application from consuming all of kernel memory. */ if (option_len > SO_MAXARGSIZE) { error = EINVAL; goto done2; } optval = option_len <= sizeof (buffer) ? &buffer : kmem_alloc((size_t)option_len, KM_SLEEP); ASSERT(MUTEX_NOT_HELD(&so->so_lock)); if (copyin(option_value, optval, (size_t)option_len)) { error = EFAULT; goto done1; } } } else option_len = 0; error = SOP_SETSOCKOPT(so, level, option_name, optval, (t_uscalar_t)option_len); done1: if (optval != buffer) kmem_free(optval, (size_t)option_len); done2: releasef(sock); if (error) return (set_errno(error)); return (0); } /* * Add config info when devpath is non-NULL; delete info when devpath is NULL. * devpath is a user address. */ int sockconfig(int domain, int type, int protocol, char *devpath) { char *kdevpath; /* Copied in devpath string */ size_t kdevpathlen; int error = 0; dprint(1, ("sockconfig(%d, %d, %d, %p)\n", domain, type, protocol, devpath)); if (secpolicy_net_config(CRED(), B_FALSE) != 0) return (set_errno(EPERM)); if (devpath == NULL) { /* Deleting an entry */ kdevpath = NULL; kdevpathlen = 0; } else { /* * Adding an entry. * Copyin the devpath. * This also makes it possible to check for too long pathnames. * Compress the space needed for the devpath before passing it * to soconfig - soconfig will store the string until * the configuration is removed. */ char *buf; buf = kmem_alloc(MAXPATHLEN, KM_SLEEP); if ((error = copyinstr(devpath, buf, MAXPATHLEN, &kdevpathlen)) != 0) { kmem_free(buf, MAXPATHLEN); goto done; } kdevpath = kmem_alloc(kdevpathlen, KM_SLEEP); bcopy(buf, kdevpath, kdevpathlen); kdevpath[kdevpathlen - 1] = '\0'; kmem_free(buf, MAXPATHLEN); } error = soconfig(domain, type, protocol, kdevpath, (int)kdevpathlen); done: if (error) { eprintline(error); return (set_errno(error)); } return (0); } /* * Sendfile is implemented through two schemes, direct I/O or by * caching in the filesystem page cache. We cache the input file by * default and use direct I/O only if sendfile_max_size is set * appropriately as explained below. Note that this logic is consistent * with other filesystems where caching is turned on by default * unless explicitly turned off by using the DIRECTIO ioctl. * * We choose a slightly different scheme here. One can turn off * caching by setting sendfile_max_size to 0. One can also enable * caching of files <= sendfile_max_size by setting sendfile_max_size * to an appropriate value. By default sendfile_max_size is set to the * maximum value so that all files are cached. In future, we may provide * better interfaces for caching the file. * * Sendfile through Direct I/O (Zero copy) * -------------------------------------- * * As disks are normally slower than the network, we can't have a * single thread that reads the disk and writes to the network. We * need to have parallelism. This is done by having the sendfile * thread create another thread that reads from the filesystem * and queues it for network processing. In this scheme, the data * is never copied anywhere i.e it is zero copy unlike the other * scheme. * * We have a sendfile queue (snfq) where each sendfile * request (snf_req_t) is queued for processing by a thread. Number * of threads is dynamically allocated and they exit if they are idling * beyond a specified amount of time. When each request (snf_req_t) is * processed by a thread, it produces a number of mblk_t structures to * be consumed by the sendfile thread. snf_deque and snf_enque are * used for consuming and producing mblks. Size of the filesystem * read is determined by the tuneable (sendfile_read_size). A single * mblk holds sendfile_read_size worth of data (except the last * read of the file) which is sent down as a whole to the network. * sendfile_read_size is set to 1 MB as this seems to be the optimal * value for the UFS filesystem backed by a striped storage array. * * Synchronisation between read (producer) and write (consumer) threads. * -------------------------------------------------------------------- * * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while * adding and deleting items in this list. Error can happen anytime * during read or write. There could be unprocessed mblks in the * sr_ib_XXX list when a read or write error occurs. Whenever error * is encountered, we need two things to happen : * * a) One of the threads need to clean the mblks. * b) When one thread encounters an error, the other should stop. * * For (a), we don't want to penalise the reader thread as it could do * some useful work processing other requests. For (b), the error can * be detected by examining sr_read_error or sr_write_error. * sr_lock protects sr_read_error and sr_write_error. If both reader and * writer encounters error, we need to report the write error back to * the application as that's what would have happened if the operations * were done sequentially. With this in mind, following should work : * * - Check for errors before read or write. * - If the reader encounters error, set the error in sr_read_error. * Check sr_write_error, if it is set, send cv_signal as it is * waiting for reader to complete. If it is not set, the writer * is either running sinking data to the network or blocked * because of flow control. For handling the latter case, we * always send a signal. In any case, it will examine sr_read_error * and return. sr_read_error is marked with SR_READ_DONE to tell * the writer that the reader is done in all the cases. * - If the writer encounters error, set the error in sr_write_error. * The reader thread is either blocked because of flow control or * running reading data from the disk. For the former, we need to * wakeup the thread. Again to keep it simple, we always wake up * the reader thread. Then, wait for the read thread to complete * if it is not done yet. Cleanup and return. * * High and low water marks for the read thread. * -------------------------------------------- * * If sendfile() is used to send data over a slow network, we need to * make sure that the read thread does not produce data at a faster * rate than the network. This can happen if the disk is faster than * the network. In such a case, we don't want to build a very large queue. * But we would still like to get all of the network throughput possible. * This implies that network should never block waiting for data. * As there are lot of disk throughput/network throughput combinations * possible, it is difficult to come up with an accurate number. * A typical 10K RPM disk has a max seek latency 17ms and rotational * latency of 3ms for reading a disk block. Thus, the total latency to * initiate a new read, transfer data from the disk and queue for * transmission would take about a max of 25ms. Todays max transfer rate * for network is 100MB/sec. If the thread is blocked because of flow * control, it would take 25ms to get new data ready for transmission. * We have to make sure that network is not idling, while we are initiating * new transfers. So, at 100MB/sec, to keep network busy we would need * 2.5MB of data. Roundig off, we keep the low water mark to be 3MB of data. * We need to pick a high water mark so that the woken up thread would * do considerable work before blocking again to prevent thrashing. Currently, * we pick this to be 10 times that of the low water mark. * * Sendfile with segmap caching (One copy from page cache to mblks). * ---------------------------------------------------------------- * * We use the segmap cache for caching the file, if the size of file * is <= sendfile_max_size. In this case we don't use threads as VM * is reasonably fast enough to keep up with the network. If the underlying * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth * of data into segmap space, and use the virtual address from segmap * directly through desballoc() to avoid copy. Once the transport is done * with the data, the mapping will be released through segmap_release() * called by the call-back routine. * * If zero-copy is not allowed by the transport, we simply call VOP_READ() * to copy the data from the filesystem into our temporary network buffer. * * To disable caching, set sendfile_max_size to 0. */ uint_t sendfile_read_size = 1024 * 1024; #define SENDFILE_REQ_LOWAT 3 * 1024 * 1024 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT; uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT; struct sendfile_stats sf_stats; struct sendfile_queue *snfq; clock_t snfq_timeout; off64_t sendfile_max_size; static void snf_enque(snf_req_t *, mblk_t *); static mblk_t *snf_deque(snf_req_t *); void sendfile_init(void) { snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP); mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL); snfq->snfq_max_threads = max_ncpus; snfq_timeout = SNFQ_TIMEOUT; /* Cache all files by default. */ sendfile_max_size = MAXOFFSET_T; } /* * Queues a mblk_t for network processing. */ static void snf_enque(snf_req_t *sr, mblk_t *mp) { mp->b_next = NULL; mutex_enter(&sr->sr_lock); if (sr->sr_mp_head == NULL) { sr->sr_mp_head = sr->sr_mp_tail = mp; cv_signal(&sr->sr_cv); } else { sr->sr_mp_tail->b_next = mp; sr->sr_mp_tail = mp; } sr->sr_qlen += MBLKL(mp); while ((sr->sr_qlen > sr->sr_hiwat) && (sr->sr_write_error == 0)) { sf_stats.ss_full_waits++; cv_wait(&sr->sr_cv, &sr->sr_lock); } mutex_exit(&sr->sr_lock); } /* * De-queues a mblk_t for network processing. */ static mblk_t * snf_deque(snf_req_t *sr) { mblk_t *mp; mutex_enter(&sr->sr_lock); /* * If we have encountered an error on read or read is * completed and no more mblks, return NULL. * We need to check for NULL sr_mp_head also as * the reads could have completed and there is * nothing more to come. */ if (((sr->sr_read_error & ~SR_READ_DONE) != 0) || ((sr->sr_read_error & SR_READ_DONE) && sr->sr_mp_head == NULL)) { mutex_exit(&sr->sr_lock); return (NULL); } /* * To start with neither SR_READ_DONE is marked nor * the error is set. When we wake up from cv_wait, * following are the possibilities : * * a) sr_read_error is zero and mblks are queued. * b) sr_read_error is set to SR_READ_DONE * and mblks are queued. * c) sr_read_error is set to SR_READ_DONE * and no mblks. * d) sr_read_error is set to some error other * than SR_READ_DONE. */ while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) { sf_stats.ss_empty_waits++; cv_wait(&sr->sr_cv, &sr->sr_lock); } /* Handle (a) and (b) first - the normal case. */ if (((sr->sr_read_error & ~SR_READ_DONE) == 0) && (sr->sr_mp_head != NULL)) { mp = sr->sr_mp_head; sr->sr_mp_head = mp->b_next; sr->sr_qlen -= MBLKL(mp); if (sr->sr_qlen < sr->sr_lowat) cv_signal(&sr->sr_cv); mutex_exit(&sr->sr_lock); mp->b_next = NULL; return (mp); } /* Handle (c) and (d). */ mutex_exit(&sr->sr_lock); return (NULL); } /* * Reads data from the filesystem and queues it for network processing. */ void snf_async_read(snf_req_t *sr) { size_t iosize; u_offset_t fileoff; u_offset_t size; int ret_size; int error; file_t *fp; mblk_t *mp; fp = sr->sr_fp; size = sr->sr_file_size; fileoff = sr->sr_file_off; /* * Ignore the error for filesystems that doesn't support DIRECTIO. */ (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0, kcred, NULL); while ((size != 0) && (sr->sr_write_error == 0)) { iosize = (int)MIN(sr->sr_maxpsz, size); if ((mp = allocb(iosize, BPRI_MED)) == NULL) { error = EAGAIN; break; } ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize); /* Error or Reached EOF ? */ if ((error != 0) || (ret_size == 0)) { freeb(mp); break; } mp->b_wptr = mp->b_rptr + ret_size; snf_enque(sr, mp); size -= ret_size; fileoff += ret_size; } (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0, kcred, NULL); mutex_enter(&sr->sr_lock); sr->sr_read_error = error; sr->sr_read_error |= SR_READ_DONE; cv_signal(&sr->sr_cv); mutex_exit(&sr->sr_lock); } void snf_async_thread(void) { snf_req_t *sr; callb_cpr_t cprinfo; clock_t time_left = 1; clock_t now; CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq"); mutex_enter(&snfq->snfq_lock); for (;;) { /* * If we didn't find a entry, then block until woken up * again and then look through the queues again. */ while ((sr = snfq->snfq_req_head) == NULL) { CALLB_CPR_SAFE_BEGIN(&cprinfo); if (time_left <= 0) { snfq->snfq_svc_threads--; CALLB_CPR_EXIT(&cprinfo); thread_exit(); /* NOTREACHED */ } snfq->snfq_idle_cnt++; time_to_wait(&now, snfq_timeout); time_left = cv_timedwait(&snfq->snfq_cv, &snfq->snfq_lock, now); snfq->snfq_idle_cnt--; CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock); } snfq->snfq_req_head = sr->sr_next; snfq->snfq_req_cnt--; mutex_exit(&snfq->snfq_lock); snf_async_read(sr); mutex_enter(&snfq->snfq_lock); } } snf_req_t * create_thread(int operation, struct vnode *vp, file_t *fp, u_offset_t fileoff, u_offset_t size) { snf_req_t *sr; stdata_t *stp; sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP); sr->sr_vp = vp; sr->sr_fp = fp; stp = vp->v_stream; /* * store sd_qn_maxpsz into sr_maxpsz while we have stream head. * stream might be closed before thread returns from snf_async_read. */ if (stp->sd_qn_maxpsz > 0) { sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz); } else { sr->sr_maxpsz = MAXBSIZE; } sr->sr_operation = operation; sr->sr_file_off = fileoff; sr->sr_file_size = size; sr->sr_hiwat = sendfile_req_hiwat; sr->sr_lowat = sendfile_req_lowat; mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL); /* * See whether we need another thread for servicing this * request. If there are already enough requests queued * for the threads, create one if not exceeding * snfq_max_threads. */ mutex_enter(&snfq->snfq_lock); if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt && snfq->snfq_svc_threads < snfq->snfq_max_threads) { (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0, TS_RUN, minclsyspri); snfq->snfq_svc_threads++; } if (snfq->snfq_req_head == NULL) { snfq->snfq_req_head = snfq->snfq_req_tail = sr; cv_signal(&snfq->snfq_cv); } else { snfq->snfq_req_tail->sr_next = sr; snfq->snfq_req_tail = sr; } snfq->snfq_req_cnt++; mutex_exit(&snfq->snfq_lock); return (sr); } int snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size, ssize_t *count) { snf_req_t *sr; mblk_t *mp; int iosize; int error = 0; short fflag; struct vnode *vp; int ksize; ksize = 0; *count = 0; vp = fp->f_vnode; fflag = fp->f_flag; if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL) return (EAGAIN); /* * We check for read error in snf_deque. It has to check * for successful READ_DONE and return NULL, and we might * as well make an additional check there. */ while ((mp = snf_deque(sr)) != NULL) { if (ISSIG(curthread, JUSTLOOKING)) { freeb(mp); error = EINTR; break; } iosize = MBLKL(mp); if ((error = kstrwritemp(vp, mp, fflag)) != 0) { freeb(mp); break; } ksize += iosize; } *count = ksize; mutex_enter(&sr->sr_lock); sr->sr_write_error = error; /* Look at the big comments on why we cv_signal here. */ cv_signal(&sr->sr_cv); /* Wait for the reader to complete always. */ while (!(sr->sr_read_error & SR_READ_DONE)) { cv_wait(&sr->sr_cv, &sr->sr_lock); } /* If there is no write error, check for read error. */ if (error == 0) error = (sr->sr_read_error & ~SR_READ_DONE); if (error != 0) { mblk_t *next_mp; mp = sr->sr_mp_head; while (mp != NULL) { next_mp = mp->b_next; mp->b_next = NULL; freeb(mp); mp = next_mp; } } mutex_exit(&sr->sr_lock); kmem_free(sr, sizeof (snf_req_t)); return (error); } typedef struct { frtn_t snfi_frtn; caddr_t snfi_base; uint_t snfi_mapoff; size_t snfi_len; vnode_t *snfi_vp; } snf_smap_desbinfo; /* * The callback function when the last ref of the mblk is dropped, * normally occurs when TCP receives the ack. But it can be the driver * too due to lazy reclaim. */ void snf_smap_desbfree(snf_smap_desbinfo *snfi) { if (!segmap_kpm) { /* * We don't need to call segmap_fault(F_SOFTUNLOCK) for * segmap_kpm as long as the latter never falls back to * "use_segmap_range". (See segmap_getmapflt().) * * Using S_OTHER saves an redundant hat_setref() in * segmap_unlock() */ (void) segmap_fault(kas.a_hat, segkmap, (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base + snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTUNLOCK, S_OTHER); } (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED); VN_RELE(snfi->snfi_vp); kmem_free(snfi, sizeof (*snfi)); } /* * Use segmap instead of bcopy to send down a chain of desballoca'ed, mblks. * Each mblk contains a segmap slot of no more than MAXBSIZE. The total * length of a chain is no more than sd_qn_maxpsz. * * At the end of the whole sendfile() operation, we wait till the data from * the last mblk is ack'ed by the transport before returning so that the * caller of sendfile() can safely modify the file content. */ int snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size, uint_t maxpsz, ssize_t *count, boolean_t nowait) { caddr_t base; int mapoff; vnode_t *vp; mblk_t *mp, *mp1; int iosize, iosize1; int error; short fflag; int ksize; snf_smap_desbinfo *snfi; struct vattr va; boolean_t dowait = B_FALSE; vp = fp->f_vnode; fflag = fp->f_flag; ksize = 0; for (;;) { if (ISSIG(curthread, JUSTLOOKING)) { error = EINTR; break; } iosize = 0; mp = NULL; do { mapoff = fileoff & MAXBOFFSET; iosize1 = MAXBSIZE - mapoff; if (iosize1 > size) iosize1 = size; /* * we don't forcefault because we'll call * segmap_fault(F_SOFTLOCK) next. * * S_READ will get the ref bit set (by either * segmap_getmapflt() or segmap_fault()) and page * shared locked. */ base = segmap_getmapflt(segkmap, fvp, fileoff, iosize1, segmap_kpm ? SM_FAULT : 0, S_READ); snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP); snfi->snfi_len = (size_t)roundup(mapoff+iosize1, PAGESIZE)- (mapoff & PAGEMASK); /* * We must call segmap_fault() even for segmap_kpm * because that's how error gets returned. * (segmap_getmapflt() never fails but segmap_fault() * does.) */ if (segmap_fault(kas.a_hat, segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base + mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK, S_READ) != 0) { (void) segmap_release(segkmap, base, 0); kmem_free(snfi, sizeof (*snfi)); freemsg(mp); error = EIO; goto out; } snfi->snfi_frtn.free_func = snf_smap_desbfree; snfi->snfi_frtn.free_arg = (caddr_t)snfi; snfi->snfi_base = base; snfi->snfi_mapoff = mapoff; mp1 = esballoca((uchar_t *)base + mapoff, iosize1, BPRI_HI, &snfi->snfi_frtn); if (mp1 == NULL) { (void) segmap_fault(kas.a_hat, segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base + mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTUNLOCK, S_OTHER); (void) segmap_release(segkmap, base, 0); kmem_free(snfi, sizeof (*snfi)); freemsg(mp); error = EAGAIN; goto out; } VN_HOLD(fvp); snfi->snfi_vp = fvp; mp1->b_wptr += iosize1; /* Mark this dblk with the zero-copy flag */ mp1->b_datap->db_struioflag |= STRUIO_ZC; if (mp == NULL) mp = mp1; else linkb(mp, mp1); iosize += iosize1; fileoff += iosize1; size -= iosize1; } while (iosize < maxpsz && size != 0); if (size == 0 && !nowait) { ASSERT(!dowait); dowait = B_TRUE; mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY; } VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); if ((error = kstrwritemp(vp, mp, fflag)) != 0) { *count = ksize; freemsg(mp); return (error); } ksize += iosize; if (size == 0) goto done; (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL); va.va_mask = AT_SIZE; error = VOP_GETATTR(fvp, &va, 0, kcred); if (error) break; /* Read as much as possible. */ if (fileoff >= va.va_size) break; if (size + fileoff > va.va_size) size = va.va_size - fileoff; } out: VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); done: *count = ksize; if (dowait) { stdata_t *stp; stp = vp->v_stream; mutex_enter(&stp->sd_lock); while (!(stp->sd_flag & STZCNOTIFY)) { if (cv_wait_sig(&stp->sd_zcopy_wait, &stp->sd_lock) == 0) { error = EINTR; break; } } stp->sd_flag &= ~STZCNOTIFY; mutex_exit(&stp->sd_lock); } return (error); } int snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size, uint_t maxpsz, ssize_t *count) { struct vnode *vp; mblk_t *mp; int iosize; int error; short fflag; int ksize; int ioflag; struct uio auio; struct iovec aiov; struct vattr va; vp = fp->f_vnode; fflag = fp->f_flag; ksize = 0; auio.uio_iov = &aiov; auio.uio_iovcnt = 1; auio.uio_segflg = UIO_SYSSPACE; auio.uio_llimit = MAXOFFSET_T; auio.uio_fmode = fflag; auio.uio_extflg = UIO_COPY_CACHED; ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC); /* If read sync is not asked for, filter sync flags */ if ((ioflag & FRSYNC) == 0) ioflag &= ~(FSYNC|FDSYNC); for (;;) { if (ISSIG(curthread, JUSTLOOKING)) { error = EINTR; break; } iosize = (int)MIN(maxpsz, size); if ((mp = allocb(iosize, BPRI_MED)) == NULL) { error = EAGAIN; break; } aiov.iov_base = (caddr_t)mp->b_rptr; aiov.iov_len = iosize; auio.uio_loffset = fileoff; auio.uio_resid = iosize; error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL); iosize -= auio.uio_resid; if (error == EINTR && iosize != 0) error = 0; if (error != 0 || iosize == 0) { freeb(mp); break; } mp->b_wptr = mp->b_rptr + iosize; VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); if ((error = kstrwritemp(vp, mp, fflag)) != 0) { *count = ksize; freeb(mp); return (error); } ksize += iosize; size -= iosize; if (size == 0) goto done; fileoff += iosize; (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL); va.va_mask = AT_SIZE; error = VOP_GETATTR(fvp, &va, 0, kcred); if (error) break; /* Read as much as possible. */ if (fileoff >= va.va_size) size = 0; else if (size + fileoff > va.va_size) size = va.va_size - fileoff; } VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); done: *count = ksize; return (error); } #if defined(_SYSCALL32_IMPL) || defined(_ILP32) /* * Largefile support for 32 bit applications only. */ int sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv, ssize32_t *count32) { ssize32_t sfv_len; u_offset_t sfv_off, va_size; struct vnode *vp, *fvp, *realvp; struct vattr va; stdata_t *stp; ssize_t count = 0; int error = 0; boolean_t dozcopy = B_FALSE; uint_t maxpsz; sfv_len = (ssize32_t)sfv->sfv_len; if (sfv_len < 0) { error = EINVAL; goto out; } if (sfv_len == 0) goto out; sfv_off = (u_offset_t)sfv->sfv_off; /* Same checks as in pread */ if (sfv_off > MAXOFFSET_T) { error = EINVAL; goto out; } if (sfv_off + sfv_len > MAXOFFSET_T) sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off); /* * There are no more checks on sfv_len. So, we cast it to * u_offset_t and share the snf_direct_io/snf_cache code between * 32 bit and 64 bit. * * TODO: should do nbl_need_check() like read()? */ if (sfv_len > sendfile_max_size) { sf_stats.ss_file_not_cached++; error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len, &count); goto out; } fvp = rfp->f_vnode; if (VOP_REALVP(fvp, &realvp) == 0) fvp = realvp; /* * Grab the lock as a reader to prevent the file size * from changing underneath. */ (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL); va.va_mask = AT_SIZE; error = VOP_GETATTR(fvp, &va, 0, kcred); va_size = va.va_size; if (error == 0 && sfv_off >= va_size) error = EINVAL; if (error != 0) { VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); goto out; } /* Read as much as possible. */ if (sfv_off + sfv_len > va_size) sfv_len = va_size - sfv_off; vp = fp->f_vnode; stp = vp->v_stream; if (stp->sd_qn_maxpsz == INFPSZ) maxpsz = maxphys; else maxpsz = roundup(stp->sd_qn_maxpsz, MAXBSIZE); /* * When the NOWAIT flag is not set, we enable zero-copy only if the * transfer size is large enough. This prevents performance loss * when the caller sends the file piece by piece. */ if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) || (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) && !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) { if ((stp->sd_copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) { int on = 1; if (SOP_SETSOCKOPT(VTOSO(vp), SOL_SOCKET, SO_SND_COPYAVOID, &on, sizeof (on)) == 0) dozcopy = B_TRUE; } else { dozcopy = (stp->sd_copyflag & STZCVMSAFE); } } if (dozcopy) { sf_stats.ss_file_segmap++; error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len, maxpsz, &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0)); } else { sf_stats.ss_file_cached++; error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len, maxpsz, &count); } out: releasef(sfv->sfv_fd); *count32 = (ssize32_t)count; return (error); } #endif #ifdef _SYSCALL32_IMPL /* * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a * ssize_t rather than ssize32_t; see the comments above read32 for details. */ ssize_t recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags) { return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags)); } ssize_t recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags, caddr32_t name, caddr32_t namelenp) { return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags, (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp)); } ssize_t send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags) { return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags)); } ssize_t sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags, caddr32_t name, socklen_t namelen) { return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags, (void *)(uintptr_t)name, namelen)); } #endif /* _SYSCALL32_IMPL */ /* * Function wrappers (mostly arround the sonode switch) for * backward compatibility. */ int soaccept(struct sonode *so, int fflag, struct sonode **nsop) { return (SOP_ACCEPT(so, fflag, nsop)); } int sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen, int backlog, int flags) { int error; error = SOP_BIND(so, name, namelen, flags); if (error == 0 && backlog != 0) return (SOP_LISTEN(so, backlog)); return (error); } int solisten(struct sonode *so, int backlog) { return (SOP_LISTEN(so, backlog)); } int soconnect(struct sonode *so, const struct sockaddr *name, socklen_t namelen, int fflag, int flags) { return (SOP_CONNECT(so, name, namelen, fflag, flags)); } int sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop) { return (SOP_RECVMSG(so, msg, uiop)); } int sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop) { return (SOP_SENDMSG(so, msg, uiop)); } int sogetpeername(struct sonode *so) { return (SOP_GETPEERNAME(so)); } int sogetsockname(struct sonode *so) { return (SOP_GETSOCKNAME(so)); } int soshutdown(struct sonode *so, int how) { return (SOP_SHUTDOWN(so, how)); } int sogetsockopt(struct sonode *so, int level, int option_name, void *optval, socklen_t *optlenp, int flags) { return (SOP_GETSOCKOPT(so, level, option_name, optval, optlenp, flags)); } int sosetsockopt(struct sonode *so, int level, int option_name, const void *optval, t_uscalar_t optlen) { return (SOP_SETSOCKOPT(so, level, option_name, optval, optlen)); } /* * Because this is backward compatibility interface it only needs to be * able to handle the creation of TPI sockfs sockets. */ struct sonode * socreate(vnode_t *accessvp, int domain, int type, int protocol, int version, struct sonode *tso, int *errorp) { return (sotpi_create(accessvp, domain, type, protocol, version, tso, errorp)); }