1321b17ecSEdward Tomasz Napierala /*- 2321b17ecSEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 3321b17ecSEdward Tomasz Napierala * All rights reserved. 4321b17ecSEdward Tomasz Napierala * 5321b17ecSEdward Tomasz Napierala * This software was developed by Edward Tomasz Napierala under sponsorship 6321b17ecSEdward Tomasz Napierala * from the FreeBSD Foundation. 7321b17ecSEdward Tomasz Napierala * 8321b17ecSEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 9321b17ecSEdward Tomasz Napierala * modification, are permitted provided that the following conditions 10321b17ecSEdward Tomasz Napierala * are met: 11321b17ecSEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 12321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 13321b17ecSEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 14321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 15321b17ecSEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 16321b17ecSEdward Tomasz Napierala * 17321b17ecSEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18321b17ecSEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19321b17ecSEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20321b17ecSEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21321b17ecSEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22321b17ecSEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23321b17ecSEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24321b17ecSEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25321b17ecSEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26321b17ecSEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27321b17ecSEdward Tomasz Napierala * SUCH DAMAGE. 28321b17ecSEdward Tomasz Napierala * 29321b17ecSEdward Tomasz Napierala */ 30321b17ecSEdward Tomasz Napierala 31321b17ecSEdward Tomasz Napierala /* 325aabcd7cSEdward Tomasz Napierala * Software implementation of iSCSI Common Layer kobj(9) interface. 33321b17ecSEdward Tomasz Napierala */ 34321b17ecSEdward Tomasz Napierala 35321b17ecSEdward Tomasz Napierala #include <sys/cdefs.h> 36321b17ecSEdward Tomasz Napierala __FBSDID("$FreeBSD$"); 37321b17ecSEdward Tomasz Napierala 38321b17ecSEdward Tomasz Napierala #include <sys/param.h> 39321b17ecSEdward Tomasz Napierala #include <sys/capsicum.h> 40321b17ecSEdward Tomasz Napierala #include <sys/condvar.h> 41321b17ecSEdward Tomasz Napierala #include <sys/conf.h> 42321b17ecSEdward Tomasz Napierala #include <sys/file.h> 43321b17ecSEdward Tomasz Napierala #include <sys/kernel.h> 44321b17ecSEdward Tomasz Napierala #include <sys/kthread.h> 45321b17ecSEdward Tomasz Napierala #include <sys/lock.h> 46321b17ecSEdward Tomasz Napierala #include <sys/mbuf.h> 47321b17ecSEdward Tomasz Napierala #include <sys/mutex.h> 48321b17ecSEdward Tomasz Napierala #include <sys/module.h> 49321b17ecSEdward Tomasz Napierala #include <sys/protosw.h> 50321b17ecSEdward Tomasz Napierala #include <sys/socket.h> 51321b17ecSEdward Tomasz Napierala #include <sys/socketvar.h> 52321b17ecSEdward Tomasz Napierala #include <sys/sysctl.h> 53321b17ecSEdward Tomasz Napierala #include <sys/systm.h> 54321b17ecSEdward Tomasz Napierala #include <sys/sx.h> 55321b17ecSEdward Tomasz Napierala #include <sys/uio.h> 56321b17ecSEdward Tomasz Napierala #include <vm/uma.h> 57321b17ecSEdward Tomasz Napierala #include <netinet/in.h> 58321b17ecSEdward Tomasz Napierala #include <netinet/tcp.h> 59321b17ecSEdward Tomasz Napierala 60321b17ecSEdward Tomasz Napierala #include <dev/iscsi/icl.h> 61321b17ecSEdward Tomasz Napierala #include <dev/iscsi/iscsi_proto.h> 62321b17ecSEdward Tomasz Napierala #include <icl_conn_if.h> 63321b17ecSEdward Tomasz Napierala 64321b17ecSEdward Tomasz Napierala static int coalesce = 1; 65321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN, 66321b17ecSEdward Tomasz Napierala &coalesce, 0, "Try to coalesce PDUs before sending"); 67321b17ecSEdward Tomasz Napierala static int partial_receive_len = 128 * 1024; 68321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 69321b17ecSEdward Tomasz Napierala &partial_receive_len, 0, "Minimum read size for partially received " 70321b17ecSEdward Tomasz Napierala "data segment"); 71321b17ecSEdward Tomasz Napierala static int sendspace = 1048576; 72321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN, 73321b17ecSEdward Tomasz Napierala &sendspace, 0, "Default send socket buffer size"); 74321b17ecSEdward Tomasz Napierala static int recvspace = 1048576; 75321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN, 76321b17ecSEdward Tomasz Napierala &recvspace, 0, "Default receive socket buffer size"); 77321b17ecSEdward Tomasz Napierala 78321b17ecSEdward Tomasz Napierala static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); 79321b17ecSEdward Tomasz Napierala static uma_zone_t icl_pdu_zone; 80321b17ecSEdward Tomasz Napierala 81321b17ecSEdward Tomasz Napierala static volatile u_int icl_ncons; 82321b17ecSEdward Tomasz Napierala 83321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) 84321b17ecSEdward Tomasz Napierala #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) 85321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) 86321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) 87321b17ecSEdward Tomasz Napierala 88321b17ecSEdward Tomasz Napierala STAILQ_HEAD(icl_pdu_stailq, icl_pdu); 89321b17ecSEdward Tomasz Napierala 90321b17ecSEdward Tomasz Napierala static icl_conn_new_pdu_t icl_soft_conn_new_pdu; 91321b17ecSEdward Tomasz Napierala static icl_conn_pdu_free_t icl_soft_conn_pdu_free; 92321b17ecSEdward Tomasz Napierala static icl_conn_pdu_data_segment_length_t 93321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length; 94321b17ecSEdward Tomasz Napierala static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; 95321b17ecSEdward Tomasz Napierala static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; 96321b17ecSEdward Tomasz Napierala static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; 97321b17ecSEdward Tomasz Napierala static icl_conn_handoff_t icl_soft_conn_handoff; 98321b17ecSEdward Tomasz Napierala static icl_conn_free_t icl_soft_conn_free; 99321b17ecSEdward Tomasz Napierala static icl_conn_close_t icl_soft_conn_close; 1007a03d007SEdward Tomasz Napierala static icl_conn_task_setup_t icl_soft_conn_task_setup; 1017a03d007SEdward Tomasz Napierala static icl_conn_task_done_t icl_soft_conn_task_done; 1027a03d007SEdward Tomasz Napierala static icl_conn_transfer_setup_t icl_soft_conn_transfer_setup; 1037a03d007SEdward Tomasz Napierala static icl_conn_transfer_done_t icl_soft_conn_transfer_done; 104f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 105f41492b0SEdward Tomasz Napierala static icl_conn_connect_t icl_soft_conn_connect; 106f41492b0SEdward Tomasz Napierala #endif 107321b17ecSEdward Tomasz Napierala 108321b17ecSEdward Tomasz Napierala static kobj_method_t icl_soft_methods[] = { 109321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), 110321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), 111321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_data_segment_length, 112321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length), 113321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), 114321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), 115321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), 116321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), 117321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_free, icl_soft_conn_free), 118321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_close, icl_soft_conn_close), 1197a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup), 1207a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done), 1217a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup), 1227a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done), 123f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 124f41492b0SEdward Tomasz Napierala KOBJMETHOD(icl_conn_connect, icl_soft_conn_connect), 125f41492b0SEdward Tomasz Napierala #endif 126321b17ecSEdward Tomasz Napierala { 0, 0 } 127321b17ecSEdward Tomasz Napierala }; 128321b17ecSEdward Tomasz Napierala 129321b17ecSEdward Tomasz Napierala DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn)); 130321b17ecSEdward Tomasz Napierala 131321b17ecSEdward Tomasz Napierala static void 132321b17ecSEdward Tomasz Napierala icl_conn_fail(struct icl_conn *ic) 133321b17ecSEdward Tomasz Napierala { 134321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) 135321b17ecSEdward Tomasz Napierala return; 136321b17ecSEdward Tomasz Napierala 137321b17ecSEdward Tomasz Napierala /* 138321b17ecSEdward Tomasz Napierala * XXX 139321b17ecSEdward Tomasz Napierala */ 140321b17ecSEdward Tomasz Napierala ic->ic_socket->so_error = EDOOFUS; 141321b17ecSEdward Tomasz Napierala (ic->ic_error)(ic); 142321b17ecSEdward Tomasz Napierala } 143321b17ecSEdward Tomasz Napierala 144321b17ecSEdward Tomasz Napierala static struct mbuf * 145321b17ecSEdward Tomasz Napierala icl_conn_receive(struct icl_conn *ic, size_t len) 146321b17ecSEdward Tomasz Napierala { 147321b17ecSEdward Tomasz Napierala struct uio uio; 148321b17ecSEdward Tomasz Napierala struct socket *so; 149321b17ecSEdward Tomasz Napierala struct mbuf *m; 150321b17ecSEdward Tomasz Napierala int error, flags; 151321b17ecSEdward Tomasz Napierala 152321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 153321b17ecSEdward Tomasz Napierala 154321b17ecSEdward Tomasz Napierala memset(&uio, 0, sizeof(uio)); 155321b17ecSEdward Tomasz Napierala uio.uio_resid = len; 156321b17ecSEdward Tomasz Napierala 157321b17ecSEdward Tomasz Napierala flags = MSG_DONTWAIT; 158321b17ecSEdward Tomasz Napierala error = soreceive(so, NULL, &uio, &m, NULL, &flags); 159321b17ecSEdward Tomasz Napierala if (error != 0) { 160321b17ecSEdward Tomasz Napierala ICL_DEBUG("soreceive error %d", error); 161321b17ecSEdward Tomasz Napierala return (NULL); 162321b17ecSEdward Tomasz Napierala } 163321b17ecSEdward Tomasz Napierala if (uio.uio_resid != 0) { 164321b17ecSEdward Tomasz Napierala m_freem(m); 165321b17ecSEdward Tomasz Napierala ICL_DEBUG("short read"); 166321b17ecSEdward Tomasz Napierala return (NULL); 167321b17ecSEdward Tomasz Napierala } 168321b17ecSEdward Tomasz Napierala 169321b17ecSEdward Tomasz Napierala return (m); 170321b17ecSEdward Tomasz Napierala } 171321b17ecSEdward Tomasz Napierala 172*d0d587c7SAlexander Motin static int 173*d0d587c7SAlexander Motin icl_conn_receive_buf(struct icl_conn *ic, void *buf, size_t len) 174*d0d587c7SAlexander Motin { 175*d0d587c7SAlexander Motin struct iovec iov[1]; 176*d0d587c7SAlexander Motin struct uio uio; 177*d0d587c7SAlexander Motin struct socket *so; 178*d0d587c7SAlexander Motin int error, flags; 179*d0d587c7SAlexander Motin 180*d0d587c7SAlexander Motin so = ic->ic_socket; 181*d0d587c7SAlexander Motin 182*d0d587c7SAlexander Motin memset(&uio, 0, sizeof(uio)); 183*d0d587c7SAlexander Motin iov[0].iov_base = buf; 184*d0d587c7SAlexander Motin iov[0].iov_len = len; 185*d0d587c7SAlexander Motin uio.uio_iov = iov; 186*d0d587c7SAlexander Motin uio.uio_iovcnt = 1; 187*d0d587c7SAlexander Motin uio.uio_offset = 0; 188*d0d587c7SAlexander Motin uio.uio_resid = len; 189*d0d587c7SAlexander Motin uio.uio_segflg = UIO_SYSSPACE; 190*d0d587c7SAlexander Motin uio.uio_rw = UIO_READ; 191*d0d587c7SAlexander Motin 192*d0d587c7SAlexander Motin flags = MSG_DONTWAIT; 193*d0d587c7SAlexander Motin error = soreceive(so, NULL, &uio, NULL, NULL, &flags); 194*d0d587c7SAlexander Motin if (error != 0) { 195*d0d587c7SAlexander Motin ICL_DEBUG("soreceive error %d", error); 196*d0d587c7SAlexander Motin return (-1); 197*d0d587c7SAlexander Motin } 198*d0d587c7SAlexander Motin if (uio.uio_resid != 0) { 199*d0d587c7SAlexander Motin ICL_DEBUG("short read"); 200*d0d587c7SAlexander Motin return (-1); 201*d0d587c7SAlexander Motin } 202*d0d587c7SAlexander Motin 203*d0d587c7SAlexander Motin return (0); 204*d0d587c7SAlexander Motin } 205*d0d587c7SAlexander Motin 206321b17ecSEdward Tomasz Napierala static struct icl_pdu * 207321b17ecSEdward Tomasz Napierala icl_pdu_new_empty(struct icl_conn *ic, int flags) 208321b17ecSEdward Tomasz Napierala { 209321b17ecSEdward Tomasz Napierala struct icl_pdu *ip; 210321b17ecSEdward Tomasz Napierala 211321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 212321b17ecSEdward Tomasz Napierala refcount_acquire(&ic->ic_outstanding_pdus); 213321b17ecSEdward Tomasz Napierala #endif 214321b17ecSEdward Tomasz Napierala ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO); 215321b17ecSEdward Tomasz Napierala if (ip == NULL) { 216321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); 217321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 218321b17ecSEdward Tomasz Napierala refcount_release(&ic->ic_outstanding_pdus); 219321b17ecSEdward Tomasz Napierala #endif 220321b17ecSEdward Tomasz Napierala return (NULL); 221321b17ecSEdward Tomasz Napierala } 222321b17ecSEdward Tomasz Napierala 223321b17ecSEdward Tomasz Napierala ip->ip_conn = ic; 224321b17ecSEdward Tomasz Napierala 225321b17ecSEdward Tomasz Napierala return (ip); 226321b17ecSEdward Tomasz Napierala } 227321b17ecSEdward Tomasz Napierala 228321b17ecSEdward Tomasz Napierala static void 229321b17ecSEdward Tomasz Napierala icl_pdu_free(struct icl_pdu *ip) 230321b17ecSEdward Tomasz Napierala { 231321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 232321b17ecSEdward Tomasz Napierala 233321b17ecSEdward Tomasz Napierala ic = ip->ip_conn; 234321b17ecSEdward Tomasz Napierala 235321b17ecSEdward Tomasz Napierala m_freem(ip->ip_bhs_mbuf); 236321b17ecSEdward Tomasz Napierala m_freem(ip->ip_ahs_mbuf); 237321b17ecSEdward Tomasz Napierala m_freem(ip->ip_data_mbuf); 238321b17ecSEdward Tomasz Napierala uma_zfree(icl_pdu_zone, ip); 239321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 240321b17ecSEdward Tomasz Napierala refcount_release(&ic->ic_outstanding_pdus); 241321b17ecSEdward Tomasz Napierala #endif 242321b17ecSEdward Tomasz Napierala } 243321b17ecSEdward Tomasz Napierala 244321b17ecSEdward Tomasz Napierala void 245321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 246321b17ecSEdward Tomasz Napierala { 2475aabcd7cSEdward Tomasz Napierala 248321b17ecSEdward Tomasz Napierala icl_pdu_free(ip); 249321b17ecSEdward Tomasz Napierala } 250321b17ecSEdward Tomasz Napierala 251321b17ecSEdward Tomasz Napierala /* 252321b17ecSEdward Tomasz Napierala * Allocate icl_pdu with empty BHS to fill up by the caller. 253321b17ecSEdward Tomasz Napierala */ 254321b17ecSEdward Tomasz Napierala struct icl_pdu * 255321b17ecSEdward Tomasz Napierala icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) 256321b17ecSEdward Tomasz Napierala { 257321b17ecSEdward Tomasz Napierala struct icl_pdu *ip; 258321b17ecSEdward Tomasz Napierala 259321b17ecSEdward Tomasz Napierala ip = icl_pdu_new_empty(ic, flags); 260321b17ecSEdward Tomasz Napierala if (ip == NULL) 261321b17ecSEdward Tomasz Napierala return (NULL); 262321b17ecSEdward Tomasz Napierala 263321b17ecSEdward Tomasz Napierala ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs), 264321b17ecSEdward Tomasz Napierala flags, MT_DATA, M_PKTHDR); 265321b17ecSEdward Tomasz Napierala if (ip->ip_bhs_mbuf == NULL) { 266*d0d587c7SAlexander Motin ICL_WARN("failed to allocate BHS mbuf"); 267321b17ecSEdward Tomasz Napierala icl_pdu_free(ip); 268321b17ecSEdward Tomasz Napierala return (NULL); 269321b17ecSEdward Tomasz Napierala } 270321b17ecSEdward Tomasz Napierala ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); 271321b17ecSEdward Tomasz Napierala memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); 272321b17ecSEdward Tomasz Napierala ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); 273321b17ecSEdward Tomasz Napierala 274321b17ecSEdward Tomasz Napierala return (ip); 275321b17ecSEdward Tomasz Napierala } 276321b17ecSEdward Tomasz Napierala 277321b17ecSEdward Tomasz Napierala static int 278321b17ecSEdward Tomasz Napierala icl_pdu_ahs_length(const struct icl_pdu *request) 279321b17ecSEdward Tomasz Napierala { 280321b17ecSEdward Tomasz Napierala 281321b17ecSEdward Tomasz Napierala return (request->ip_bhs->bhs_total_ahs_len * 4); 282321b17ecSEdward Tomasz Napierala } 283321b17ecSEdward Tomasz Napierala 284321b17ecSEdward Tomasz Napierala static size_t 285321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_length(const struct icl_pdu *request) 286321b17ecSEdward Tomasz Napierala { 287321b17ecSEdward Tomasz Napierala uint32_t len = 0; 288321b17ecSEdward Tomasz Napierala 289321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[0]; 290321b17ecSEdward Tomasz Napierala len <<= 8; 291321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[1]; 292321b17ecSEdward Tomasz Napierala len <<= 8; 293321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[2]; 294321b17ecSEdward Tomasz Napierala 295321b17ecSEdward Tomasz Napierala return (len); 296321b17ecSEdward Tomasz Napierala } 297321b17ecSEdward Tomasz Napierala 298321b17ecSEdward Tomasz Napierala size_t 299321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, 300321b17ecSEdward Tomasz Napierala const struct icl_pdu *request) 301321b17ecSEdward Tomasz Napierala { 302321b17ecSEdward Tomasz Napierala 303321b17ecSEdward Tomasz Napierala return (icl_pdu_data_segment_length(request)); 304321b17ecSEdward Tomasz Napierala } 305321b17ecSEdward Tomasz Napierala 306321b17ecSEdward Tomasz Napierala static void 307321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) 308321b17ecSEdward Tomasz Napierala { 309321b17ecSEdward Tomasz Napierala 310321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[2] = len; 311321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[1] = len >> 8; 312321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[0] = len >> 16; 313321b17ecSEdward Tomasz Napierala } 314321b17ecSEdward Tomasz Napierala 315321b17ecSEdward Tomasz Napierala static size_t 316321b17ecSEdward Tomasz Napierala icl_pdu_padding(const struct icl_pdu *ip) 317321b17ecSEdward Tomasz Napierala { 318321b17ecSEdward Tomasz Napierala 319321b17ecSEdward Tomasz Napierala if ((ip->ip_data_len % 4) != 0) 320321b17ecSEdward Tomasz Napierala return (4 - (ip->ip_data_len % 4)); 321321b17ecSEdward Tomasz Napierala 322321b17ecSEdward Tomasz Napierala return (0); 323321b17ecSEdward Tomasz Napierala } 324321b17ecSEdward Tomasz Napierala 325321b17ecSEdward Tomasz Napierala static size_t 326321b17ecSEdward Tomasz Napierala icl_pdu_size(const struct icl_pdu *response) 327321b17ecSEdward Tomasz Napierala { 328321b17ecSEdward Tomasz Napierala size_t len; 329321b17ecSEdward Tomasz Napierala 330321b17ecSEdward Tomasz Napierala KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); 331321b17ecSEdward Tomasz Napierala 332321b17ecSEdward Tomasz Napierala len = sizeof(struct iscsi_bhs) + response->ip_data_len + 333321b17ecSEdward Tomasz Napierala icl_pdu_padding(response); 334321b17ecSEdward Tomasz Napierala if (response->ip_conn->ic_header_crc32c) 335321b17ecSEdward Tomasz Napierala len += ISCSI_HEADER_DIGEST_SIZE; 336321b17ecSEdward Tomasz Napierala if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) 337321b17ecSEdward Tomasz Napierala len += ISCSI_DATA_DIGEST_SIZE; 338321b17ecSEdward Tomasz Napierala 339321b17ecSEdward Tomasz Napierala return (len); 340321b17ecSEdward Tomasz Napierala } 341321b17ecSEdward Tomasz Napierala 342321b17ecSEdward Tomasz Napierala static int 343321b17ecSEdward Tomasz Napierala icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep) 344321b17ecSEdward Tomasz Napierala { 345321b17ecSEdward Tomasz Napierala 346*d0d587c7SAlexander Motin if (icl_conn_receive_buf(request->ip_conn, 347*d0d587c7SAlexander Motin request->ip_bhs, sizeof(struct iscsi_bhs))) { 348321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive BHS"); 349321b17ecSEdward Tomasz Napierala return (-1); 350321b17ecSEdward Tomasz Napierala } 351321b17ecSEdward Tomasz Napierala 352321b17ecSEdward Tomasz Napierala *availablep -= sizeof(struct iscsi_bhs); 353321b17ecSEdward Tomasz Napierala return (0); 354321b17ecSEdward Tomasz Napierala } 355321b17ecSEdward Tomasz Napierala 356321b17ecSEdward Tomasz Napierala static int 357321b17ecSEdward Tomasz Napierala icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep) 358321b17ecSEdward Tomasz Napierala { 359321b17ecSEdward Tomasz Napierala 360321b17ecSEdward Tomasz Napierala request->ip_ahs_len = icl_pdu_ahs_length(request); 361321b17ecSEdward Tomasz Napierala if (request->ip_ahs_len == 0) 362321b17ecSEdward Tomasz Napierala return (0); 363321b17ecSEdward Tomasz Napierala 364321b17ecSEdward Tomasz Napierala request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn, 365321b17ecSEdward Tomasz Napierala request->ip_ahs_len); 366321b17ecSEdward Tomasz Napierala if (request->ip_ahs_mbuf == NULL) { 367321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive AHS"); 368321b17ecSEdward Tomasz Napierala return (-1); 369321b17ecSEdward Tomasz Napierala } 370321b17ecSEdward Tomasz Napierala 371321b17ecSEdward Tomasz Napierala *availablep -= request->ip_ahs_len; 372321b17ecSEdward Tomasz Napierala return (0); 373321b17ecSEdward Tomasz Napierala } 374321b17ecSEdward Tomasz Napierala 375321b17ecSEdward Tomasz Napierala static uint32_t 376321b17ecSEdward Tomasz Napierala icl_mbuf_to_crc32c(const struct mbuf *m0) 377321b17ecSEdward Tomasz Napierala { 378321b17ecSEdward Tomasz Napierala uint32_t digest = 0xffffffff; 379321b17ecSEdward Tomasz Napierala const struct mbuf *m; 380321b17ecSEdward Tomasz Napierala 381321b17ecSEdward Tomasz Napierala for (m = m0; m != NULL; m = m->m_next) 382321b17ecSEdward Tomasz Napierala digest = calculate_crc32c(digest, 383321b17ecSEdward Tomasz Napierala mtod(m, const void *), m->m_len); 384321b17ecSEdward Tomasz Napierala 385321b17ecSEdward Tomasz Napierala digest = digest ^ 0xffffffff; 386321b17ecSEdward Tomasz Napierala 387321b17ecSEdward Tomasz Napierala return (digest); 388321b17ecSEdward Tomasz Napierala } 389321b17ecSEdward Tomasz Napierala 390321b17ecSEdward Tomasz Napierala static int 391321b17ecSEdward Tomasz Napierala icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep) 392321b17ecSEdward Tomasz Napierala { 393321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 394321b17ecSEdward Tomasz Napierala 395321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_header_crc32c == false) 396321b17ecSEdward Tomasz Napierala return (0); 397321b17ecSEdward Tomasz Napierala 398*d0d587c7SAlexander Motin CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); 399*d0d587c7SAlexander Motin if (icl_conn_receive_buf(request->ip_conn, 400*d0d587c7SAlexander Motin &received_digest, ISCSI_HEADER_DIGEST_SIZE)) { 401321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive header digest"); 402321b17ecSEdward Tomasz Napierala return (-1); 403321b17ecSEdward Tomasz Napierala } 404321b17ecSEdward Tomasz Napierala *availablep -= ISCSI_HEADER_DIGEST_SIZE; 405321b17ecSEdward Tomasz Napierala 406321b17ecSEdward Tomasz Napierala /* 407321b17ecSEdward Tomasz Napierala * XXX: Handle AHS. 408321b17ecSEdward Tomasz Napierala */ 409321b17ecSEdward Tomasz Napierala valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 410321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 411321b17ecSEdward Tomasz Napierala ICL_WARN("header digest check failed; got 0x%x, " 412321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 413321b17ecSEdward Tomasz Napierala return (-1); 414321b17ecSEdward Tomasz Napierala } 415321b17ecSEdward Tomasz Napierala 416321b17ecSEdward Tomasz Napierala return (0); 417321b17ecSEdward Tomasz Napierala } 418321b17ecSEdward Tomasz Napierala 419321b17ecSEdward Tomasz Napierala /* 420321b17ecSEdward Tomasz Napierala * Return the number of bytes that should be waiting in the receive socket 421321b17ecSEdward Tomasz Napierala * before icl_pdu_receive_data_segment() gets called. 422321b17ecSEdward Tomasz Napierala */ 423321b17ecSEdward Tomasz Napierala static size_t 424321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(const struct icl_pdu *request) 425321b17ecSEdward Tomasz Napierala { 426321b17ecSEdward Tomasz Napierala size_t len; 427321b17ecSEdward Tomasz Napierala 428321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 429321b17ecSEdward Tomasz Napierala if (len == 0) 430321b17ecSEdward Tomasz Napierala return (0); 431321b17ecSEdward Tomasz Napierala 432321b17ecSEdward Tomasz Napierala /* 433321b17ecSEdward Tomasz Napierala * Account for the parts of data segment already read from 434321b17ecSEdward Tomasz Napierala * the socket buffer. 435321b17ecSEdward Tomasz Napierala */ 436321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 437321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 438321b17ecSEdward Tomasz Napierala 439321b17ecSEdward Tomasz Napierala /* 440321b17ecSEdward Tomasz Napierala * Don't always wait for the full data segment to be delivered 441321b17ecSEdward Tomasz Napierala * to the socket; this might badly affect performance due to 442321b17ecSEdward Tomasz Napierala * TCP window scaling. 443321b17ecSEdward Tomasz Napierala */ 444321b17ecSEdward Tomasz Napierala if (len > partial_receive_len) { 445321b17ecSEdward Tomasz Napierala #if 0 446321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data, limiting to %zd", 447321b17ecSEdward Tomasz Napierala len, partial_receive_len)); 448321b17ecSEdward Tomasz Napierala #endif 449321b17ecSEdward Tomasz Napierala len = partial_receive_len; 450321b17ecSEdward Tomasz Napierala 451321b17ecSEdward Tomasz Napierala return (len); 452321b17ecSEdward Tomasz Napierala } 453321b17ecSEdward Tomasz Napierala 454321b17ecSEdward Tomasz Napierala /* 455321b17ecSEdward Tomasz Napierala * Account for padding. Note that due to the way code is written, 456321b17ecSEdward Tomasz Napierala * the icl_pdu_receive_data_segment() must always receive padding 457321b17ecSEdward Tomasz Napierala * along with the last part of data segment, because it would be 458321b17ecSEdward Tomasz Napierala * impossible to tell whether we've already received the full data 459321b17ecSEdward Tomasz Napierala * segment including padding, or without it. 460321b17ecSEdward Tomasz Napierala */ 461321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 462321b17ecSEdward Tomasz Napierala len += 4 - (len % 4); 463321b17ecSEdward Tomasz Napierala 464321b17ecSEdward Tomasz Napierala #if 0 465321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data", len)); 466321b17ecSEdward Tomasz Napierala #endif 467321b17ecSEdward Tomasz Napierala 468321b17ecSEdward Tomasz Napierala return (len); 469321b17ecSEdward Tomasz Napierala } 470321b17ecSEdward Tomasz Napierala 471321b17ecSEdward Tomasz Napierala static int 472321b17ecSEdward Tomasz Napierala icl_pdu_receive_data_segment(struct icl_pdu *request, 473321b17ecSEdward Tomasz Napierala size_t *availablep, bool *more_neededp) 474321b17ecSEdward Tomasz Napierala { 475321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 476321b17ecSEdward Tomasz Napierala size_t len, padding = 0; 477321b17ecSEdward Tomasz Napierala struct mbuf *m; 478321b17ecSEdward Tomasz Napierala 479321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 480321b17ecSEdward Tomasz Napierala 481321b17ecSEdward Tomasz Napierala *more_neededp = false; 482321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 483321b17ecSEdward Tomasz Napierala 484321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 485321b17ecSEdward Tomasz Napierala if (len == 0) 486321b17ecSEdward Tomasz Napierala return (0); 487321b17ecSEdward Tomasz Napierala 488321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 489321b17ecSEdward Tomasz Napierala padding = 4 - (len % 4); 490321b17ecSEdward Tomasz Napierala 491321b17ecSEdward Tomasz Napierala /* 492321b17ecSEdward Tomasz Napierala * Account for already received parts of data segment. 493321b17ecSEdward Tomasz Napierala */ 494321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 495321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 496321b17ecSEdward Tomasz Napierala 497321b17ecSEdward Tomasz Napierala if (len + padding > *availablep) { 498321b17ecSEdward Tomasz Napierala /* 499321b17ecSEdward Tomasz Napierala * Not enough data in the socket buffer. Receive as much 500321b17ecSEdward Tomasz Napierala * as we can. Don't receive padding, since, obviously, it's 501321b17ecSEdward Tomasz Napierala * not the end of data segment yet. 502321b17ecSEdward Tomasz Napierala */ 503321b17ecSEdward Tomasz Napierala #if 0 504321b17ecSEdward Tomasz Napierala ICL_DEBUG("limited from %zd to %zd", 505321b17ecSEdward Tomasz Napierala len + padding, *availablep - padding)); 506321b17ecSEdward Tomasz Napierala #endif 507321b17ecSEdward Tomasz Napierala len = *availablep - padding; 508321b17ecSEdward Tomasz Napierala *more_neededp = true; 509321b17ecSEdward Tomasz Napierala padding = 0; 510321b17ecSEdward Tomasz Napierala } 511321b17ecSEdward Tomasz Napierala 512321b17ecSEdward Tomasz Napierala /* 513321b17ecSEdward Tomasz Napierala * Must not try to receive padding without at least one byte 514321b17ecSEdward Tomasz Napierala * of actual data segment. 515321b17ecSEdward Tomasz Napierala */ 516321b17ecSEdward Tomasz Napierala if (len > 0) { 517321b17ecSEdward Tomasz Napierala m = icl_conn_receive(request->ip_conn, len + padding); 518321b17ecSEdward Tomasz Napierala if (m == NULL) { 519321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data segment"); 520321b17ecSEdward Tomasz Napierala return (-1); 521321b17ecSEdward Tomasz Napierala } 522321b17ecSEdward Tomasz Napierala 523321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) 524321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = m; 525321b17ecSEdward Tomasz Napierala else 526321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, m); 527321b17ecSEdward Tomasz Napierala 528321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 529321b17ecSEdward Tomasz Napierala *availablep -= len + padding; 530321b17ecSEdward Tomasz Napierala } else 531321b17ecSEdward Tomasz Napierala ICL_DEBUG("len 0"); 532321b17ecSEdward Tomasz Napierala 533321b17ecSEdward Tomasz Napierala if (*more_neededp) 534321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 535321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(request); 536321b17ecSEdward Tomasz Napierala 537321b17ecSEdward Tomasz Napierala return (0); 538321b17ecSEdward Tomasz Napierala } 539321b17ecSEdward Tomasz Napierala 540321b17ecSEdward Tomasz Napierala static int 541321b17ecSEdward Tomasz Napierala icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep) 542321b17ecSEdward Tomasz Napierala { 543321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 544321b17ecSEdward Tomasz Napierala 545321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_data_crc32c == false) 546321b17ecSEdward Tomasz Napierala return (0); 547321b17ecSEdward Tomasz Napierala 548321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0) 549321b17ecSEdward Tomasz Napierala return (0); 550321b17ecSEdward Tomasz Napierala 551*d0d587c7SAlexander Motin CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); 552*d0d587c7SAlexander Motin if (icl_conn_receive_buf(request->ip_conn, 553*d0d587c7SAlexander Motin &received_digest, ISCSI_DATA_DIGEST_SIZE)) { 554321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data digest"); 555321b17ecSEdward Tomasz Napierala return (-1); 556321b17ecSEdward Tomasz Napierala } 557321b17ecSEdward Tomasz Napierala *availablep -= ISCSI_DATA_DIGEST_SIZE; 558321b17ecSEdward Tomasz Napierala 559321b17ecSEdward Tomasz Napierala /* 560321b17ecSEdward Tomasz Napierala * Note that ip_data_mbuf also contains padding; since digest 561321b17ecSEdward Tomasz Napierala * calculation is supposed to include that, we iterate over 562321b17ecSEdward Tomasz Napierala * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. 563321b17ecSEdward Tomasz Napierala */ 564321b17ecSEdward Tomasz Napierala valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 565321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 566321b17ecSEdward Tomasz Napierala ICL_WARN("data digest check failed; got 0x%x, " 567321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 568321b17ecSEdward Tomasz Napierala return (-1); 569321b17ecSEdward Tomasz Napierala } 570321b17ecSEdward Tomasz Napierala 571321b17ecSEdward Tomasz Napierala return (0); 572321b17ecSEdward Tomasz Napierala } 573321b17ecSEdward Tomasz Napierala 574321b17ecSEdward Tomasz Napierala /* 575321b17ecSEdward Tomasz Napierala * Somewhat contrary to the name, this attempts to receive only one 576321b17ecSEdward Tomasz Napierala * "part" of PDU at a time; call it repeatedly until it returns non-NULL. 577321b17ecSEdward Tomasz Napierala */ 578321b17ecSEdward Tomasz Napierala static struct icl_pdu * 579321b17ecSEdward Tomasz Napierala icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep) 580321b17ecSEdward Tomasz Napierala { 581321b17ecSEdward Tomasz Napierala struct icl_pdu *request; 582321b17ecSEdward Tomasz Napierala struct socket *so; 583321b17ecSEdward Tomasz Napierala size_t len; 584321b17ecSEdward Tomasz Napierala int error; 585321b17ecSEdward Tomasz Napierala bool more_needed; 586321b17ecSEdward Tomasz Napierala 587321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 588321b17ecSEdward Tomasz Napierala 589321b17ecSEdward Tomasz Napierala if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { 590321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_receive_pdu == NULL, 591321b17ecSEdward Tomasz Napierala ("ic->ic_receive_pdu != NULL")); 592*d0d587c7SAlexander Motin request = icl_soft_conn_new_pdu(ic, M_NOWAIT); 593321b17ecSEdward Tomasz Napierala if (request == NULL) { 594321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to allocate PDU; " 595321b17ecSEdward Tomasz Napierala "dropping connection"); 596321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 597321b17ecSEdward Tomasz Napierala return (NULL); 598321b17ecSEdward Tomasz Napierala } 599321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = request; 600321b17ecSEdward Tomasz Napierala } else { 601321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_receive_pdu != NULL, 602321b17ecSEdward Tomasz Napierala ("ic->ic_receive_pdu == NULL")); 603321b17ecSEdward Tomasz Napierala request = ic->ic_receive_pdu; 604321b17ecSEdward Tomasz Napierala } 605321b17ecSEdward Tomasz Napierala 606321b17ecSEdward Tomasz Napierala if (*availablep < ic->ic_receive_len) { 607321b17ecSEdward Tomasz Napierala #if 0 608321b17ecSEdward Tomasz Napierala ICL_DEBUG("not enough data; need %zd, " 609321b17ecSEdward Tomasz Napierala "have %zd", ic->ic_receive_len, *availablep); 610321b17ecSEdward Tomasz Napierala #endif 611321b17ecSEdward Tomasz Napierala return (NULL); 612321b17ecSEdward Tomasz Napierala } 613321b17ecSEdward Tomasz Napierala 614321b17ecSEdward Tomasz Napierala switch (ic->ic_receive_state) { 615321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_BHS: 616321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving BHS"); 617321b17ecSEdward Tomasz Napierala error = icl_pdu_receive_bhs(request, availablep); 618321b17ecSEdward Tomasz Napierala if (error != 0) { 619321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive BHS; " 620321b17ecSEdward Tomasz Napierala "dropping connection"); 621321b17ecSEdward Tomasz Napierala break; 622321b17ecSEdward Tomasz Napierala } 623321b17ecSEdward Tomasz Napierala 624321b17ecSEdward Tomasz Napierala /* 625321b17ecSEdward Tomasz Napierala * We don't enforce any limit for AHS length; 626321b17ecSEdward Tomasz Napierala * its length is stored in 8 bit field. 627321b17ecSEdward Tomasz Napierala */ 628321b17ecSEdward Tomasz Napierala 629321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 630321b17ecSEdward Tomasz Napierala if (len > ic->ic_max_data_segment_length) { 631321b17ecSEdward Tomasz Napierala ICL_WARN("received data segment " 632321b17ecSEdward Tomasz Napierala "length %zd is larger than negotiated " 633321b17ecSEdward Tomasz Napierala "MaxDataSegmentLength %zd; " 634321b17ecSEdward Tomasz Napierala "dropping connection", 635321b17ecSEdward Tomasz Napierala len, ic->ic_max_data_segment_length); 636321b17ecSEdward Tomasz Napierala error = EINVAL; 637321b17ecSEdward Tomasz Napierala break; 638321b17ecSEdward Tomasz Napierala } 639321b17ecSEdward Tomasz Napierala 640321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_AHS; 641321b17ecSEdward Tomasz Napierala ic->ic_receive_len = icl_pdu_ahs_length(request); 642321b17ecSEdward Tomasz Napierala break; 643321b17ecSEdward Tomasz Napierala 644321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_AHS: 645321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving AHS"); 646321b17ecSEdward Tomasz Napierala error = icl_pdu_receive_ahs(request, availablep); 647321b17ecSEdward Tomasz Napierala if (error != 0) { 648321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive AHS; " 649321b17ecSEdward Tomasz Napierala "dropping connection"); 650321b17ecSEdward Tomasz Napierala break; 651321b17ecSEdward Tomasz Napierala } 652321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; 653321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c == false) 654321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 655321b17ecSEdward Tomasz Napierala else 656321b17ecSEdward Tomasz Napierala ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; 657321b17ecSEdward Tomasz Napierala break; 658321b17ecSEdward Tomasz Napierala 659321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_HEADER_DIGEST: 660321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving header digest"); 661321b17ecSEdward Tomasz Napierala error = icl_pdu_check_header_digest(request, availablep); 662321b17ecSEdward Tomasz Napierala if (error != 0) { 663321b17ecSEdward Tomasz Napierala ICL_DEBUG("header digest failed; " 664321b17ecSEdward Tomasz Napierala "dropping connection"); 665321b17ecSEdward Tomasz Napierala break; 666321b17ecSEdward Tomasz Napierala } 667321b17ecSEdward Tomasz Napierala 668321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_DATA; 669321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 670321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(request); 671321b17ecSEdward Tomasz Napierala break; 672321b17ecSEdward Tomasz Napierala 673321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA: 674321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data segment"); 675321b17ecSEdward Tomasz Napierala error = icl_pdu_receive_data_segment(request, availablep, 676321b17ecSEdward Tomasz Napierala &more_needed); 677321b17ecSEdward Tomasz Napierala if (error != 0) { 678321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data segment;" 679321b17ecSEdward Tomasz Napierala "dropping connection"); 680321b17ecSEdward Tomasz Napierala break; 681321b17ecSEdward Tomasz Napierala } 682321b17ecSEdward Tomasz Napierala 683321b17ecSEdward Tomasz Napierala if (more_needed) 684321b17ecSEdward Tomasz Napierala break; 685321b17ecSEdward Tomasz Napierala 686321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; 687321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) 688321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 689321b17ecSEdward Tomasz Napierala else 690321b17ecSEdward Tomasz Napierala ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; 691321b17ecSEdward Tomasz Napierala break; 692321b17ecSEdward Tomasz Napierala 693321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA_DIGEST: 694321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data digest"); 695321b17ecSEdward Tomasz Napierala error = icl_pdu_check_data_digest(request, availablep); 696321b17ecSEdward Tomasz Napierala if (error != 0) { 697321b17ecSEdward Tomasz Napierala ICL_DEBUG("data digest failed; " 698321b17ecSEdward Tomasz Napierala "dropping connection"); 699321b17ecSEdward Tomasz Napierala break; 700321b17ecSEdward Tomasz Napierala } 701321b17ecSEdward Tomasz Napierala 702321b17ecSEdward Tomasz Napierala /* 703321b17ecSEdward Tomasz Napierala * We've received complete PDU; reset the receive state machine 704321b17ecSEdward Tomasz Napierala * and return the PDU. 705321b17ecSEdward Tomasz Napierala */ 706321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_BHS; 707321b17ecSEdward Tomasz Napierala ic->ic_receive_len = sizeof(struct iscsi_bhs); 708321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = NULL; 709321b17ecSEdward Tomasz Napierala return (request); 710321b17ecSEdward Tomasz Napierala 711321b17ecSEdward Tomasz Napierala default: 712321b17ecSEdward Tomasz Napierala panic("invalid ic_receive_state %d\n", ic->ic_receive_state); 713321b17ecSEdward Tomasz Napierala } 714321b17ecSEdward Tomasz Napierala 715321b17ecSEdward Tomasz Napierala if (error != 0) { 716321b17ecSEdward Tomasz Napierala /* 717321b17ecSEdward Tomasz Napierala * Don't free the PDU; it's pointed to by ic->ic_receive_pdu 7185aabcd7cSEdward Tomasz Napierala * and will get freed in icl_soft_conn_close(). 719321b17ecSEdward Tomasz Napierala */ 720321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 721321b17ecSEdward Tomasz Napierala } 722321b17ecSEdward Tomasz Napierala 723321b17ecSEdward Tomasz Napierala return (NULL); 724321b17ecSEdward Tomasz Napierala } 725321b17ecSEdward Tomasz Napierala 726321b17ecSEdward Tomasz Napierala static void 727321b17ecSEdward Tomasz Napierala icl_conn_receive_pdus(struct icl_conn *ic, size_t available) 728321b17ecSEdward Tomasz Napierala { 729321b17ecSEdward Tomasz Napierala struct icl_pdu *response; 730321b17ecSEdward Tomasz Napierala struct socket *so; 731321b17ecSEdward Tomasz Napierala 732321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 733321b17ecSEdward Tomasz Napierala 734321b17ecSEdward Tomasz Napierala /* 735321b17ecSEdward Tomasz Napierala * This can never happen; we're careful to only mess with ic->ic_socket 736321b17ecSEdward Tomasz Napierala * pointer when the send/receive threads are not running. 737321b17ecSEdward Tomasz Napierala */ 738321b17ecSEdward Tomasz Napierala KASSERT(so != NULL, ("NULL socket")); 739321b17ecSEdward Tomasz Napierala 740321b17ecSEdward Tomasz Napierala for (;;) { 741321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) 742321b17ecSEdward Tomasz Napierala return; 743321b17ecSEdward Tomasz Napierala 744321b17ecSEdward Tomasz Napierala if (so->so_error != 0) { 745321b17ecSEdward Tomasz Napierala ICL_DEBUG("connection error %d; " 746321b17ecSEdward Tomasz Napierala "dropping connection", so->so_error); 747321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 748321b17ecSEdward Tomasz Napierala return; 749321b17ecSEdward Tomasz Napierala } 750321b17ecSEdward Tomasz Napierala 751321b17ecSEdward Tomasz Napierala /* 752321b17ecSEdward Tomasz Napierala * Loop until we have a complete PDU or there is not enough 753321b17ecSEdward Tomasz Napierala * data in the socket buffer. 754321b17ecSEdward Tomasz Napierala */ 755321b17ecSEdward Tomasz Napierala if (available < ic->ic_receive_len) { 756321b17ecSEdward Tomasz Napierala #if 0 757321b17ecSEdward Tomasz Napierala ICL_DEBUG("not enough data; have %zd, " 758321b17ecSEdward Tomasz Napierala "need %zd", available, 759321b17ecSEdward Tomasz Napierala ic->ic_receive_len); 760321b17ecSEdward Tomasz Napierala #endif 761321b17ecSEdward Tomasz Napierala return; 762321b17ecSEdward Tomasz Napierala } 763321b17ecSEdward Tomasz Napierala 764321b17ecSEdward Tomasz Napierala response = icl_conn_receive_pdu(ic, &available); 765321b17ecSEdward Tomasz Napierala if (response == NULL) 766321b17ecSEdward Tomasz Napierala continue; 767321b17ecSEdward Tomasz Napierala 768321b17ecSEdward Tomasz Napierala if (response->ip_ahs_len > 0) { 769321b17ecSEdward Tomasz Napierala ICL_WARN("received PDU with unsupported " 770321b17ecSEdward Tomasz Napierala "AHS; opcode 0x%x; dropping connection", 771321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_opcode); 772321b17ecSEdward Tomasz Napierala icl_pdu_free(response); 773321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 774321b17ecSEdward Tomasz Napierala return; 775321b17ecSEdward Tomasz Napierala } 776321b17ecSEdward Tomasz Napierala 777321b17ecSEdward Tomasz Napierala (ic->ic_receive)(response); 778321b17ecSEdward Tomasz Napierala } 779321b17ecSEdward Tomasz Napierala } 780321b17ecSEdward Tomasz Napierala 781321b17ecSEdward Tomasz Napierala static void 782321b17ecSEdward Tomasz Napierala icl_receive_thread(void *arg) 783321b17ecSEdward Tomasz Napierala { 784321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 785321b17ecSEdward Tomasz Napierala size_t available; 786321b17ecSEdward Tomasz Napierala struct socket *so; 787321b17ecSEdward Tomasz Napierala 788321b17ecSEdward Tomasz Napierala ic = arg; 789321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 790321b17ecSEdward Tomasz Napierala 791321b17ecSEdward Tomasz Napierala for (;;) { 792321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 793321b17ecSEdward Tomasz Napierala //ICL_DEBUG("terminating"); 794321b17ecSEdward Tomasz Napierala break; 795321b17ecSEdward Tomasz Napierala } 796321b17ecSEdward Tomasz Napierala 797321b17ecSEdward Tomasz Napierala /* 798321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 799321b17ecSEdward Tomasz Napierala * soreadable() in icl_soupcall_receive() 800266078c6SPedro F. Giffuni * to avoid unnecessary wakeups until there 801321b17ecSEdward Tomasz Napierala * is enough data received to read the PDU. 802321b17ecSEdward Tomasz Napierala */ 803321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_rcv); 804321b17ecSEdward Tomasz Napierala available = sbavail(&so->so_rcv); 805321b17ecSEdward Tomasz Napierala if (available < ic->ic_receive_len) { 806321b17ecSEdward Tomasz Napierala so->so_rcv.sb_lowat = ic->ic_receive_len; 807321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); 808321b17ecSEdward Tomasz Napierala } else 809321b17ecSEdward Tomasz Napierala so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; 810321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_rcv); 811321b17ecSEdward Tomasz Napierala 812321b17ecSEdward Tomasz Napierala icl_conn_receive_pdus(ic, available); 813321b17ecSEdward Tomasz Napierala } 814321b17ecSEdward Tomasz Napierala 815321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 816321b17ecSEdward Tomasz Napierala ic->ic_receive_running = false; 817321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 818321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 819321b17ecSEdward Tomasz Napierala kthread_exit(); 820321b17ecSEdward Tomasz Napierala } 821321b17ecSEdward Tomasz Napierala 822321b17ecSEdward Tomasz Napierala static int 823321b17ecSEdward Tomasz Napierala icl_soupcall_receive(struct socket *so, void *arg, int waitflag) 824321b17ecSEdward Tomasz Napierala { 825321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 826321b17ecSEdward Tomasz Napierala 827321b17ecSEdward Tomasz Napierala if (!soreadable(so)) 828321b17ecSEdward Tomasz Napierala return (SU_OK); 829321b17ecSEdward Tomasz Napierala 830321b17ecSEdward Tomasz Napierala ic = arg; 831321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_receive_cv); 832321b17ecSEdward Tomasz Napierala return (SU_OK); 833321b17ecSEdward Tomasz Napierala } 834321b17ecSEdward Tomasz Napierala 835321b17ecSEdward Tomasz Napierala static int 836321b17ecSEdward Tomasz Napierala icl_pdu_finalize(struct icl_pdu *request) 837321b17ecSEdward Tomasz Napierala { 838321b17ecSEdward Tomasz Napierala size_t padding, pdu_len; 839321b17ecSEdward Tomasz Napierala uint32_t digest, zero = 0; 840321b17ecSEdward Tomasz Napierala int ok; 841321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 842321b17ecSEdward Tomasz Napierala 843321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 844321b17ecSEdward Tomasz Napierala 845321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(request, request->ip_data_len); 846321b17ecSEdward Tomasz Napierala 847321b17ecSEdward Tomasz Napierala pdu_len = icl_pdu_size(request); 848321b17ecSEdward Tomasz Napierala 849321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c) { 850321b17ecSEdward Tomasz Napierala digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 851321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_bhs_mbuf, sizeof(digest), 852321b17ecSEdward Tomasz Napierala (void *)&digest); 853321b17ecSEdward Tomasz Napierala if (ok != 1) { 854321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append header digest"); 855321b17ecSEdward Tomasz Napierala return (1); 856321b17ecSEdward Tomasz Napierala } 857321b17ecSEdward Tomasz Napierala } 858321b17ecSEdward Tomasz Napierala 859321b17ecSEdward Tomasz Napierala if (request->ip_data_len != 0) { 860321b17ecSEdward Tomasz Napierala padding = icl_pdu_padding(request); 861321b17ecSEdward Tomasz Napierala if (padding > 0) { 862321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, padding, 863321b17ecSEdward Tomasz Napierala (void *)&zero); 864321b17ecSEdward Tomasz Napierala if (ok != 1) { 865321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append padding"); 866321b17ecSEdward Tomasz Napierala return (1); 867321b17ecSEdward Tomasz Napierala } 868321b17ecSEdward Tomasz Napierala } 869321b17ecSEdward Tomasz Napierala 870321b17ecSEdward Tomasz Napierala if (ic->ic_data_crc32c) { 871321b17ecSEdward Tomasz Napierala digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 872321b17ecSEdward Tomasz Napierala 873321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, sizeof(digest), 874321b17ecSEdward Tomasz Napierala (void *)&digest); 875321b17ecSEdward Tomasz Napierala if (ok != 1) { 876321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append data digest"); 877321b17ecSEdward Tomasz Napierala return (1); 878321b17ecSEdward Tomasz Napierala } 879321b17ecSEdward Tomasz Napierala } 880321b17ecSEdward Tomasz Napierala 881321b17ecSEdward Tomasz Napierala m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); 882321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = NULL; 883321b17ecSEdward Tomasz Napierala } 884321b17ecSEdward Tomasz Napierala 885321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; 886321b17ecSEdward Tomasz Napierala 887321b17ecSEdward Tomasz Napierala return (0); 888321b17ecSEdward Tomasz Napierala } 889321b17ecSEdward Tomasz Napierala 890321b17ecSEdward Tomasz Napierala static void 891321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) 892321b17ecSEdward Tomasz Napierala { 893321b17ecSEdward Tomasz Napierala struct icl_pdu *request, *request2; 894321b17ecSEdward Tomasz Napierala struct socket *so; 895321b17ecSEdward Tomasz Napierala size_t available, size, size2; 896321b17ecSEdward Tomasz Napierala int coalesced, error; 897321b17ecSEdward Tomasz Napierala 898321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 899321b17ecSEdward Tomasz Napierala 900321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 901321b17ecSEdward Tomasz Napierala 902321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 903321b17ecSEdward Tomasz Napierala /* 904321b17ecSEdward Tomasz Napierala * Check how much space do we have for transmit. We can't just 905321b17ecSEdward Tomasz Napierala * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, 906321b17ecSEdward Tomasz Napierala * as it always frees the mbuf chain passed to it, even in case 907321b17ecSEdward Tomasz Napierala * of error. 908321b17ecSEdward Tomasz Napierala */ 909321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 910321b17ecSEdward Tomasz Napierala 911321b17ecSEdward Tomasz Napierala /* 912321b17ecSEdward Tomasz Napierala * Notify the socket upcall that we don't need wakeups 913321b17ecSEdward Tomasz Napierala * for the time being. 914321b17ecSEdward Tomasz Napierala */ 915321b17ecSEdward Tomasz Napierala so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; 916321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 917321b17ecSEdward Tomasz Napierala 918321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(queue)) { 919321b17ecSEdward Tomasz Napierala request = STAILQ_FIRST(queue); 920321b17ecSEdward Tomasz Napierala size = icl_pdu_size(request); 921321b17ecSEdward Tomasz Napierala if (available < size) { 922321b17ecSEdward Tomasz Napierala 923321b17ecSEdward Tomasz Napierala /* 924321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 925321b17ecSEdward Tomasz Napierala * sowriteable() in icl_soupcall_send() 926266078c6SPedro F. Giffuni * to avoid unnecessary wakeups until there 927321b17ecSEdward Tomasz Napierala * is enough space for the PDU to fit. 928321b17ecSEdward Tomasz Napierala */ 929321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 930321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 931321b17ecSEdward Tomasz Napierala if (available < size) { 932321b17ecSEdward Tomasz Napierala #if 1 933321b17ecSEdward Tomasz Napierala ICL_DEBUG("no space to send; " 934321b17ecSEdward Tomasz Napierala "have %zd, need %zd", 935321b17ecSEdward Tomasz Napierala available, size); 936321b17ecSEdward Tomasz Napierala #endif 937321b17ecSEdward Tomasz Napierala so->so_snd.sb_lowat = size; 938321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 939321b17ecSEdward Tomasz Napierala return; 940321b17ecSEdward Tomasz Napierala } 941321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 942321b17ecSEdward Tomasz Napierala } 943321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 944321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request); 945321b17ecSEdward Tomasz Napierala if (error != 0) { 946321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 947321b17ecSEdward Tomasz Napierala "dropping connection"); 948321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 949321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 950321b17ecSEdward Tomasz Napierala return; 951321b17ecSEdward Tomasz Napierala } 952321b17ecSEdward Tomasz Napierala if (coalesce) { 953321b17ecSEdward Tomasz Napierala coalesced = 1; 954321b17ecSEdward Tomasz Napierala for (;;) { 955321b17ecSEdward Tomasz Napierala request2 = STAILQ_FIRST(queue); 956321b17ecSEdward Tomasz Napierala if (request2 == NULL) 957321b17ecSEdward Tomasz Napierala break; 958321b17ecSEdward Tomasz Napierala size2 = icl_pdu_size(request2); 959321b17ecSEdward Tomasz Napierala if (available < size + size2) 960321b17ecSEdward Tomasz Napierala break; 961321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 962321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request2); 963321b17ecSEdward Tomasz Napierala if (error != 0) { 964321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 965321b17ecSEdward Tomasz Napierala "dropping connection"); 966321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 967321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 968321b17ecSEdward Tomasz Napierala icl_pdu_free(request2); 969321b17ecSEdward Tomasz Napierala return; 970321b17ecSEdward Tomasz Napierala } 971321b17ecSEdward Tomasz Napierala m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf); 972321b17ecSEdward Tomasz Napierala request2->ip_bhs_mbuf = NULL; 973321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len += size2; 974321b17ecSEdward Tomasz Napierala size += size2; 975321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_AFTER(queue, request, ip_next); 976321b17ecSEdward Tomasz Napierala icl_pdu_free(request2); 977321b17ecSEdward Tomasz Napierala coalesced++; 978321b17ecSEdward Tomasz Napierala } 979321b17ecSEdward Tomasz Napierala #if 0 980321b17ecSEdward Tomasz Napierala if (coalesced > 1) { 981321b17ecSEdward Tomasz Napierala ICL_DEBUG("coalesced %d PDUs into %zd bytes", 982321b17ecSEdward Tomasz Napierala coalesced, size); 983321b17ecSEdward Tomasz Napierala } 984321b17ecSEdward Tomasz Napierala #endif 985321b17ecSEdward Tomasz Napierala } 986321b17ecSEdward Tomasz Napierala available -= size; 987321b17ecSEdward Tomasz Napierala error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, 988321b17ecSEdward Tomasz Napierala NULL, MSG_DONTWAIT, curthread); 989321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ 990321b17ecSEdward Tomasz Napierala if (error != 0) { 991321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to send PDU, error %d; " 992321b17ecSEdward Tomasz Napierala "dropping connection", error); 993321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 994321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 995321b17ecSEdward Tomasz Napierala return; 996321b17ecSEdward Tomasz Napierala } 997321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 998321b17ecSEdward Tomasz Napierala } 999321b17ecSEdward Tomasz Napierala } 1000321b17ecSEdward Tomasz Napierala 1001321b17ecSEdward Tomasz Napierala static void 1002321b17ecSEdward Tomasz Napierala icl_send_thread(void *arg) 1003321b17ecSEdward Tomasz Napierala { 1004321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1005321b17ecSEdward Tomasz Napierala struct icl_pdu_stailq queue; 1006321b17ecSEdward Tomasz Napierala 1007321b17ecSEdward Tomasz Napierala ic = arg; 1008321b17ecSEdward Tomasz Napierala 1009321b17ecSEdward Tomasz Napierala STAILQ_INIT(&queue); 1010321b17ecSEdward Tomasz Napierala 1011321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1012321b17ecSEdward Tomasz Napierala for (;;) { 1013321b17ecSEdward Tomasz Napierala for (;;) { 1014321b17ecSEdward Tomasz Napierala /* 1015321b17ecSEdward Tomasz Napierala * If the local queue is empty, populate it from 1016321b17ecSEdward Tomasz Napierala * the main one. This way the icl_conn_send_pdus() 1017321b17ecSEdward Tomasz Napierala * can go through all the queued PDUs without holding 1018321b17ecSEdward Tomasz Napierala * any locks. 1019321b17ecSEdward Tomasz Napierala */ 1020321b17ecSEdward Tomasz Napierala if (STAILQ_EMPTY(&queue)) 1021321b17ecSEdward Tomasz Napierala STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu); 1022321b17ecSEdward Tomasz Napierala 1023321b17ecSEdward Tomasz Napierala ic->ic_check_send_space = false; 1024321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1025321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(ic, &queue); 1026321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1027321b17ecSEdward Tomasz Napierala 1028321b17ecSEdward Tomasz Napierala /* 1029321b17ecSEdward Tomasz Napierala * The icl_soupcall_send() was called since the last 1030321b17ecSEdward Tomasz Napierala * call to sbspace(); go around; 1031321b17ecSEdward Tomasz Napierala */ 1032321b17ecSEdward Tomasz Napierala if (ic->ic_check_send_space) 1033321b17ecSEdward Tomasz Napierala continue; 1034321b17ecSEdward Tomasz Napierala 1035321b17ecSEdward Tomasz Napierala /* 1036321b17ecSEdward Tomasz Napierala * Local queue is empty, but we still have PDUs 1037321b17ecSEdward Tomasz Napierala * in the main one; go around. 1038321b17ecSEdward Tomasz Napierala */ 1039321b17ecSEdward Tomasz Napierala if (STAILQ_EMPTY(&queue) && 1040321b17ecSEdward Tomasz Napierala !STAILQ_EMPTY(&ic->ic_to_send)) 1041321b17ecSEdward Tomasz Napierala continue; 1042321b17ecSEdward Tomasz Napierala 1043321b17ecSEdward Tomasz Napierala /* 1044321b17ecSEdward Tomasz Napierala * There might be some stuff in the local queue, 1045321b17ecSEdward Tomasz Napierala * which didn't get sent due to not having enough send 1046321b17ecSEdward Tomasz Napierala * space. Wait for socket upcall. 1047321b17ecSEdward Tomasz Napierala */ 1048321b17ecSEdward Tomasz Napierala break; 1049321b17ecSEdward Tomasz Napierala } 1050321b17ecSEdward Tomasz Napierala 1051321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 1052321b17ecSEdward Tomasz Napierala //ICL_DEBUG("terminating"); 1053321b17ecSEdward Tomasz Napierala break; 1054321b17ecSEdward Tomasz Napierala } 1055321b17ecSEdward Tomasz Napierala 1056321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_send_cv, ic->ic_lock); 1057321b17ecSEdward Tomasz Napierala } 1058321b17ecSEdward Tomasz Napierala 1059321b17ecSEdward Tomasz Napierala /* 1060321b17ecSEdward Tomasz Napierala * We're exiting; move PDUs back to the main queue, so they can 1061321b17ecSEdward Tomasz Napierala * get freed properly. At this point ordering doesn't matter. 1062321b17ecSEdward Tomasz Napierala */ 1063321b17ecSEdward Tomasz Napierala STAILQ_CONCAT(&ic->ic_to_send, &queue); 1064321b17ecSEdward Tomasz Napierala 1065321b17ecSEdward Tomasz Napierala ic->ic_send_running = false; 1066321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1067321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1068321b17ecSEdward Tomasz Napierala kthread_exit(); 1069321b17ecSEdward Tomasz Napierala } 1070321b17ecSEdward Tomasz Napierala 1071321b17ecSEdward Tomasz Napierala static int 1072321b17ecSEdward Tomasz Napierala icl_soupcall_send(struct socket *so, void *arg, int waitflag) 1073321b17ecSEdward Tomasz Napierala { 1074321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1075321b17ecSEdward Tomasz Napierala 1076321b17ecSEdward Tomasz Napierala if (!sowriteable(so)) 1077321b17ecSEdward Tomasz Napierala return (SU_OK); 1078321b17ecSEdward Tomasz Napierala 1079321b17ecSEdward Tomasz Napierala ic = arg; 1080321b17ecSEdward Tomasz Napierala 1081321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1082321b17ecSEdward Tomasz Napierala ic->ic_check_send_space = true; 1083321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1084321b17ecSEdward Tomasz Napierala 1085321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1086321b17ecSEdward Tomasz Napierala 1087321b17ecSEdward Tomasz Napierala return (SU_OK); 1088321b17ecSEdward Tomasz Napierala } 1089321b17ecSEdward Tomasz Napierala 1090321b17ecSEdward Tomasz Napierala static int 1091321b17ecSEdward Tomasz Napierala icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len, 1092321b17ecSEdward Tomasz Napierala int flags) 1093321b17ecSEdward Tomasz Napierala { 1094321b17ecSEdward Tomasz Napierala struct mbuf *mb, *newmb; 1095321b17ecSEdward Tomasz Napierala size_t copylen, off = 0; 1096321b17ecSEdward Tomasz Napierala 1097321b17ecSEdward Tomasz Napierala KASSERT(len > 0, ("len == 0")); 1098321b17ecSEdward Tomasz Napierala 1099898fd11fSAlexander Motin newmb = m_getm2(NULL, len, flags, MT_DATA, 0); 1100321b17ecSEdward Tomasz Napierala if (newmb == NULL) { 1101321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate mbuf for %zd bytes", len); 1102321b17ecSEdward Tomasz Napierala return (ENOMEM); 1103321b17ecSEdward Tomasz Napierala } 1104321b17ecSEdward Tomasz Napierala 1105321b17ecSEdward Tomasz Napierala for (mb = newmb; mb != NULL; mb = mb->m_next) { 1106321b17ecSEdward Tomasz Napierala copylen = min(M_TRAILINGSPACE(mb), len - off); 1107321b17ecSEdward Tomasz Napierala memcpy(mtod(mb, char *), (const char *)addr + off, copylen); 1108321b17ecSEdward Tomasz Napierala mb->m_len = copylen; 1109321b17ecSEdward Tomasz Napierala off += copylen; 1110321b17ecSEdward Tomasz Napierala } 1111321b17ecSEdward Tomasz Napierala KASSERT(off == len, ("%s: off != len", __func__)); 1112321b17ecSEdward Tomasz Napierala 1113321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) { 1114321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = newmb; 1115321b17ecSEdward Tomasz Napierala request->ip_data_len = len; 1116321b17ecSEdward Tomasz Napierala } else { 1117321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, newmb); 1118321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 1119321b17ecSEdward Tomasz Napierala } 1120321b17ecSEdward Tomasz Napierala 1121321b17ecSEdward Tomasz Napierala return (0); 1122321b17ecSEdward Tomasz Napierala } 1123321b17ecSEdward Tomasz Napierala 1124321b17ecSEdward Tomasz Napierala int 1125321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 1126321b17ecSEdward Tomasz Napierala const void *addr, size_t len, int flags) 1127321b17ecSEdward Tomasz Napierala { 1128321b17ecSEdward Tomasz Napierala 1129321b17ecSEdward Tomasz Napierala return (icl_pdu_append_data(request, addr, len, flags)); 1130321b17ecSEdward Tomasz Napierala } 1131321b17ecSEdward Tomasz Napierala 1132321b17ecSEdward Tomasz Napierala static void 1133321b17ecSEdward Tomasz Napierala icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len) 1134321b17ecSEdward Tomasz Napierala { 1135321b17ecSEdward Tomasz Napierala 1136321b17ecSEdward Tomasz Napierala m_copydata(ip->ip_data_mbuf, off, len, addr); 1137321b17ecSEdward Tomasz Napierala } 1138321b17ecSEdward Tomasz Napierala 1139321b17ecSEdward Tomasz Napierala void 1140321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1141321b17ecSEdward Tomasz Napierala size_t off, void *addr, size_t len) 1142321b17ecSEdward Tomasz Napierala { 1143321b17ecSEdward Tomasz Napierala 1144321b17ecSEdward Tomasz Napierala return (icl_pdu_get_data(ip, off, addr, len)); 1145321b17ecSEdward Tomasz Napierala } 1146321b17ecSEdward Tomasz Napierala 1147321b17ecSEdward Tomasz Napierala static void 1148321b17ecSEdward Tomasz Napierala icl_pdu_queue(struct icl_pdu *ip) 1149321b17ecSEdward Tomasz Napierala { 1150321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1151321b17ecSEdward Tomasz Napierala 1152321b17ecSEdward Tomasz Napierala ic = ip->ip_conn; 1153321b17ecSEdward Tomasz Napierala 1154321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT(ic); 1155321b17ecSEdward Tomasz Napierala 1156321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1157321b17ecSEdward Tomasz Napierala ICL_DEBUG("icl_pdu_queue on closed connection"); 1158321b17ecSEdward Tomasz Napierala icl_pdu_free(ip); 1159321b17ecSEdward Tomasz Napierala return; 1160321b17ecSEdward Tomasz Napierala } 1161321b17ecSEdward Tomasz Napierala 1162321b17ecSEdward Tomasz Napierala if (!STAILQ_EMPTY(&ic->ic_to_send)) { 1163321b17ecSEdward Tomasz Napierala STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1164321b17ecSEdward Tomasz Napierala /* 1165321b17ecSEdward Tomasz Napierala * If the queue is not empty, someone else had already 1166321b17ecSEdward Tomasz Napierala * signaled the send thread; no need to do that again, 1167321b17ecSEdward Tomasz Napierala * just return. 1168321b17ecSEdward Tomasz Napierala */ 1169321b17ecSEdward Tomasz Napierala return; 1170321b17ecSEdward Tomasz Napierala } 1171321b17ecSEdward Tomasz Napierala 1172321b17ecSEdward Tomasz Napierala STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1173321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1174321b17ecSEdward Tomasz Napierala } 1175321b17ecSEdward Tomasz Napierala 1176321b17ecSEdward Tomasz Napierala void 1177321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 1178321b17ecSEdward Tomasz Napierala { 1179321b17ecSEdward Tomasz Napierala 1180321b17ecSEdward Tomasz Napierala icl_pdu_queue(ip); 1181321b17ecSEdward Tomasz Napierala } 1182321b17ecSEdward Tomasz Napierala 1183321b17ecSEdward Tomasz Napierala static struct icl_conn * 1184321b17ecSEdward Tomasz Napierala icl_soft_new_conn(const char *name, struct mtx *lock) 1185321b17ecSEdward Tomasz Napierala { 1186321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1187321b17ecSEdward Tomasz Napierala 1188321b17ecSEdward Tomasz Napierala refcount_acquire(&icl_ncons); 1189321b17ecSEdward Tomasz Napierala 1190321b17ecSEdward Tomasz Napierala ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO); 1191321b17ecSEdward Tomasz Napierala 1192321b17ecSEdward Tomasz Napierala STAILQ_INIT(&ic->ic_to_send); 1193321b17ecSEdward Tomasz Napierala ic->ic_lock = lock; 1194321b17ecSEdward Tomasz Napierala cv_init(&ic->ic_send_cv, "icl_tx"); 1195321b17ecSEdward Tomasz Napierala cv_init(&ic->ic_receive_cv, "icl_rx"); 1196321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 1197321b17ecSEdward Tomasz Napierala refcount_init(&ic->ic_outstanding_pdus, 0); 1198321b17ecSEdward Tomasz Napierala #endif 1199321b17ecSEdward Tomasz Napierala ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH; 1200321b17ecSEdward Tomasz Napierala ic->ic_name = name; 1201d4b195d3SEdward Tomasz Napierala ic->ic_offload = "None"; 12027deb68abSEdward Tomasz Napierala ic->ic_unmapped = false; 1203321b17ecSEdward Tomasz Napierala 1204321b17ecSEdward Tomasz Napierala return (ic); 1205321b17ecSEdward Tomasz Napierala } 1206321b17ecSEdward Tomasz Napierala 1207321b17ecSEdward Tomasz Napierala void 1208321b17ecSEdward Tomasz Napierala icl_soft_conn_free(struct icl_conn *ic) 1209321b17ecSEdward Tomasz Napierala { 1210321b17ecSEdward Tomasz Napierala 1211321b17ecSEdward Tomasz Napierala cv_destroy(&ic->ic_send_cv); 1212321b17ecSEdward Tomasz Napierala cv_destroy(&ic->ic_receive_cv); 1213321b17ecSEdward Tomasz Napierala kobj_delete((struct kobj *)ic, M_ICL_SOFT); 1214321b17ecSEdward Tomasz Napierala refcount_release(&icl_ncons); 1215321b17ecSEdward Tomasz Napierala } 1216321b17ecSEdward Tomasz Napierala 1217321b17ecSEdward Tomasz Napierala static int 1218321b17ecSEdward Tomasz Napierala icl_conn_start(struct icl_conn *ic) 1219321b17ecSEdward Tomasz Napierala { 1220321b17ecSEdward Tomasz Napierala size_t minspace; 1221321b17ecSEdward Tomasz Napierala struct sockopt opt; 1222321b17ecSEdward Tomasz Napierala int error, one = 1; 1223321b17ecSEdward Tomasz Napierala 1224321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1225321b17ecSEdward Tomasz Napierala 1226321b17ecSEdward Tomasz Napierala /* 1227321b17ecSEdward Tomasz Napierala * XXX: Ugly hack. 1228321b17ecSEdward Tomasz Napierala */ 1229321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1230321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1231321b17ecSEdward Tomasz Napierala return (EINVAL); 1232321b17ecSEdward Tomasz Napierala } 1233321b17ecSEdward Tomasz Napierala 1234321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_BHS; 1235321b17ecSEdward Tomasz Napierala ic->ic_receive_len = sizeof(struct iscsi_bhs); 1236321b17ecSEdward Tomasz Napierala ic->ic_disconnecting = false; 1237321b17ecSEdward Tomasz Napierala 1238321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1239321b17ecSEdward Tomasz Napierala 1240321b17ecSEdward Tomasz Napierala /* 1241321b17ecSEdward Tomasz Napierala * For sendspace, this is required because the current code cannot 1242321b17ecSEdward Tomasz Napierala * send a PDU in pieces; thus, the minimum buffer size is equal 1243321b17ecSEdward Tomasz Napierala * to the maximum PDU size. "+4" is to account for possible padding. 1244321b17ecSEdward Tomasz Napierala * 1245321b17ecSEdward Tomasz Napierala * What we should actually do here is to use autoscaling, but set 1246321b17ecSEdward Tomasz Napierala * some minimal buffer size to "minspace". I don't know a way to do 1247321b17ecSEdward Tomasz Napierala * that, though. 1248321b17ecSEdward Tomasz Napierala */ 1249321b17ecSEdward Tomasz Napierala minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + 1250321b17ecSEdward Tomasz Napierala ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1251321b17ecSEdward Tomasz Napierala if (sendspace < minspace) { 1252321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.sendspace too low; must be at least %zd", 1253321b17ecSEdward Tomasz Napierala minspace); 1254321b17ecSEdward Tomasz Napierala sendspace = minspace; 1255321b17ecSEdward Tomasz Napierala } 1256321b17ecSEdward Tomasz Napierala if (recvspace < minspace) { 1257321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.recvspace too low; must be at least %zd", 1258321b17ecSEdward Tomasz Napierala minspace); 1259321b17ecSEdward Tomasz Napierala recvspace = minspace; 1260321b17ecSEdward Tomasz Napierala } 1261321b17ecSEdward Tomasz Napierala 1262321b17ecSEdward Tomasz Napierala error = soreserve(ic->ic_socket, sendspace, recvspace); 1263321b17ecSEdward Tomasz Napierala if (error != 0) { 1264321b17ecSEdward Tomasz Napierala ICL_WARN("soreserve failed with error %d", error); 12655aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(ic); 1266321b17ecSEdward Tomasz Napierala return (error); 1267321b17ecSEdward Tomasz Napierala } 1268321b17ecSEdward Tomasz Napierala ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; 1269321b17ecSEdward Tomasz Napierala ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; 1270321b17ecSEdward Tomasz Napierala 1271321b17ecSEdward Tomasz Napierala /* 1272321b17ecSEdward Tomasz Napierala * Disable Nagle. 1273321b17ecSEdward Tomasz Napierala */ 1274321b17ecSEdward Tomasz Napierala bzero(&opt, sizeof(opt)); 1275321b17ecSEdward Tomasz Napierala opt.sopt_dir = SOPT_SET; 1276321b17ecSEdward Tomasz Napierala opt.sopt_level = IPPROTO_TCP; 1277321b17ecSEdward Tomasz Napierala opt.sopt_name = TCP_NODELAY; 1278321b17ecSEdward Tomasz Napierala opt.sopt_val = &one; 1279321b17ecSEdward Tomasz Napierala opt.sopt_valsize = sizeof(one); 1280321b17ecSEdward Tomasz Napierala error = sosetopt(ic->ic_socket, &opt); 1281321b17ecSEdward Tomasz Napierala if (error != 0) { 1282321b17ecSEdward Tomasz Napierala ICL_WARN("disabling TCP_NODELAY failed with error %d", error); 12835aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(ic); 1284321b17ecSEdward Tomasz Napierala return (error); 1285321b17ecSEdward Tomasz Napierala } 1286321b17ecSEdward Tomasz Napierala 1287321b17ecSEdward Tomasz Napierala /* 1288321b17ecSEdward Tomasz Napierala * Register socket upcall, to get notified about incoming PDUs 1289321b17ecSEdward Tomasz Napierala * and free space to send outgoing ones. 1290321b17ecSEdward Tomasz Napierala */ 1291321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1292321b17ecSEdward Tomasz Napierala soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); 1293321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1294321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1295321b17ecSEdward Tomasz Napierala soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); 1296321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1297321b17ecSEdward Tomasz Napierala 12985b157f21SAlexander Motin /* 12995b157f21SAlexander Motin * Start threads. 13005b157f21SAlexander Motin */ 13015b157f21SAlexander Motin ICL_CONN_LOCK(ic); 13025b157f21SAlexander Motin ic->ic_send_running = ic->ic_receive_running = true; 13035b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 13045b157f21SAlexander Motin error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", 13055b157f21SAlexander Motin ic->ic_name); 13065b157f21SAlexander Motin if (error != 0) { 13075b157f21SAlexander Motin ICL_WARN("kthread_add(9) failed with error %d", error); 13085b157f21SAlexander Motin ICL_CONN_LOCK(ic); 13095b157f21SAlexander Motin ic->ic_send_running = ic->ic_receive_running = false; 13105b157f21SAlexander Motin cv_signal(&ic->ic_send_cv); 13115b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 13125b157f21SAlexander Motin icl_soft_conn_close(ic); 13135b157f21SAlexander Motin return (error); 13145b157f21SAlexander Motin } 13155b157f21SAlexander Motin error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", 13165b157f21SAlexander Motin ic->ic_name); 13175b157f21SAlexander Motin if (error != 0) { 13185b157f21SAlexander Motin ICL_WARN("kthread_add(9) failed with error %d", error); 13195b157f21SAlexander Motin ICL_CONN_LOCK(ic); 13205b157f21SAlexander Motin ic->ic_receive_running = false; 13215b157f21SAlexander Motin cv_signal(&ic->ic_send_cv); 13225b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 13235b157f21SAlexander Motin icl_soft_conn_close(ic); 13245b157f21SAlexander Motin return (error); 13255b157f21SAlexander Motin } 13265b157f21SAlexander Motin 1327321b17ecSEdward Tomasz Napierala return (0); 1328321b17ecSEdward Tomasz Napierala } 1329321b17ecSEdward Tomasz Napierala 1330321b17ecSEdward Tomasz Napierala int 1331321b17ecSEdward Tomasz Napierala icl_soft_conn_handoff(struct icl_conn *ic, int fd) 1332321b17ecSEdward Tomasz Napierala { 1333321b17ecSEdward Tomasz Napierala struct file *fp; 1334321b17ecSEdward Tomasz Napierala struct socket *so; 1335321b17ecSEdward Tomasz Napierala cap_rights_t rights; 1336321b17ecSEdward Tomasz Napierala int error; 1337321b17ecSEdward Tomasz Napierala 1338321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1339321b17ecSEdward Tomasz Napierala 1340906a424bSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1341906a424bSEdward Tomasz Napierala /* 1342906a424bSEdward Tomasz Napierala * We're transitioning to Full Feature phase, and we don't 1343906a424bSEdward Tomasz Napierala * really care. 1344906a424bSEdward Tomasz Napierala */ 1345906a424bSEdward Tomasz Napierala if (fd == 0) { 1346906a424bSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1347906a424bSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1348906a424bSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1349906a424bSEdward Tomasz Napierala ICL_WARN("proxy handoff without connect"); 1350906a424bSEdward Tomasz Napierala return (EINVAL); 1351906a424bSEdward Tomasz Napierala } 1352906a424bSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1353906a424bSEdward Tomasz Napierala return (0); 1354906a424bSEdward Tomasz Napierala } 1355906a424bSEdward Tomasz Napierala #endif 1356906a424bSEdward Tomasz Napierala 1357321b17ecSEdward Tomasz Napierala /* 1358321b17ecSEdward Tomasz Napierala * Steal the socket from userland. 1359321b17ecSEdward Tomasz Napierala */ 1360321b17ecSEdward Tomasz Napierala error = fget(curthread, fd, 1361321b17ecSEdward Tomasz Napierala cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); 1362321b17ecSEdward Tomasz Napierala if (error != 0) 1363321b17ecSEdward Tomasz Napierala return (error); 1364321b17ecSEdward Tomasz Napierala if (fp->f_type != DTYPE_SOCKET) { 1365321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1366321b17ecSEdward Tomasz Napierala return (EINVAL); 1367321b17ecSEdward Tomasz Napierala } 1368321b17ecSEdward Tomasz Napierala so = fp->f_data; 1369321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) { 1370321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1371321b17ecSEdward Tomasz Napierala return (EINVAL); 1372321b17ecSEdward Tomasz Napierala } 1373321b17ecSEdward Tomasz Napierala 1374321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1375321b17ecSEdward Tomasz Napierala 1376321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1377321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1378321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1379321b17ecSEdward Tomasz Napierala return (EBUSY); 1380321b17ecSEdward Tomasz Napierala } 1381321b17ecSEdward Tomasz Napierala 1382321b17ecSEdward Tomasz Napierala ic->ic_socket = fp->f_data; 1383321b17ecSEdward Tomasz Napierala fp->f_ops = &badfileops; 1384321b17ecSEdward Tomasz Napierala fp->f_data = NULL; 1385321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1386321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1387321b17ecSEdward Tomasz Napierala 1388321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1389321b17ecSEdward Tomasz Napierala 1390321b17ecSEdward Tomasz Napierala return (error); 1391321b17ecSEdward Tomasz Napierala } 1392321b17ecSEdward Tomasz Napierala 1393321b17ecSEdward Tomasz Napierala void 13945aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(struct icl_conn *ic) 1395321b17ecSEdward Tomasz Napierala { 1396321b17ecSEdward Tomasz Napierala struct icl_pdu *pdu; 13975b157f21SAlexander Motin struct socket *so; 1398321b17ecSEdward Tomasz Napierala 1399321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 14005b157f21SAlexander Motin 14015b157f21SAlexander Motin /* 14025b157f21SAlexander Motin * Wake up the threads, so they can properly terminate. 14035b157f21SAlexander Motin */ 14045b157f21SAlexander Motin ic->ic_disconnecting = true; 14055b157f21SAlexander Motin while (ic->ic_receive_running || ic->ic_send_running) { 14065b157f21SAlexander Motin cv_signal(&ic->ic_receive_cv); 14075b157f21SAlexander Motin cv_signal(&ic->ic_send_cv); 14085b157f21SAlexander Motin cv_wait(&ic->ic_send_cv, ic->ic_lock); 14095b157f21SAlexander Motin } 14105b157f21SAlexander Motin 14115b157f21SAlexander Motin /* Some other thread could close the connection same time. */ 14125b157f21SAlexander Motin so = ic->ic_socket; 14135b157f21SAlexander Motin if (so == NULL) { 1414321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1415321b17ecSEdward Tomasz Napierala return; 1416321b17ecSEdward Tomasz Napierala } 14175b157f21SAlexander Motin ic->ic_socket = NULL; 1418321b17ecSEdward Tomasz Napierala 1419321b17ecSEdward Tomasz Napierala /* 1420321b17ecSEdward Tomasz Napierala * Deregister socket upcalls. 1421321b17ecSEdward Tomasz Napierala */ 1422321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 14235b157f21SAlexander Motin SOCKBUF_LOCK(&so->so_snd); 14245b157f21SAlexander Motin if (so->so_snd.sb_upcall != NULL) 14255b157f21SAlexander Motin soupcall_clear(so, SO_SND); 14265b157f21SAlexander Motin SOCKBUF_UNLOCK(&so->so_snd); 14275b157f21SAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 14285b157f21SAlexander Motin if (so->so_rcv.sb_upcall != NULL) 14295b157f21SAlexander Motin soupcall_clear(so, SO_RCV); 14305b157f21SAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 14315b157f21SAlexander Motin soclose(so); 1432321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1433321b17ecSEdward Tomasz Napierala 1434321b17ecSEdward Tomasz Napierala if (ic->ic_receive_pdu != NULL) { 1435321b17ecSEdward Tomasz Napierala //ICL_DEBUG("freeing partially received PDU"); 1436321b17ecSEdward Tomasz Napierala icl_pdu_free(ic->ic_receive_pdu); 1437321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = NULL; 1438321b17ecSEdward Tomasz Napierala } 1439321b17ecSEdward Tomasz Napierala 1440321b17ecSEdward Tomasz Napierala /* 1441321b17ecSEdward Tomasz Napierala * Remove any outstanding PDUs from the send queue. 1442321b17ecSEdward Tomasz Napierala */ 1443321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(&ic->ic_to_send)) { 1444321b17ecSEdward Tomasz Napierala pdu = STAILQ_FIRST(&ic->ic_to_send); 1445321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); 1446321b17ecSEdward Tomasz Napierala icl_pdu_free(pdu); 1447321b17ecSEdward Tomasz Napierala } 1448321b17ecSEdward Tomasz Napierala 1449321b17ecSEdward Tomasz Napierala KASSERT(STAILQ_EMPTY(&ic->ic_to_send), 1450321b17ecSEdward Tomasz Napierala ("destroying session with non-empty send queue")); 1451321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 1452321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_outstanding_pdus == 0, 1453321b17ecSEdward Tomasz Napierala ("destroying session with %d outstanding PDUs", 1454321b17ecSEdward Tomasz Napierala ic->ic_outstanding_pdus)); 1455321b17ecSEdward Tomasz Napierala #endif 1456321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1457321b17ecSEdward Tomasz Napierala } 1458321b17ecSEdward Tomasz Napierala 14597a03d007SEdward Tomasz Napierala int 1460604c023fSEdward Tomasz Napierala icl_soft_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 1461604c023fSEdward Tomasz Napierala struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp) 14627a03d007SEdward Tomasz Napierala { 14637a03d007SEdward Tomasz Napierala 14647a03d007SEdward Tomasz Napierala return (0); 14657a03d007SEdward Tomasz Napierala } 14667a03d007SEdward Tomasz Napierala 14677a03d007SEdward Tomasz Napierala void 14687a03d007SEdward Tomasz Napierala icl_soft_conn_task_done(struct icl_conn *ic, void *prv) 14697a03d007SEdward Tomasz Napierala { 14707a03d007SEdward Tomasz Napierala } 14717a03d007SEdward Tomasz Napierala 14727a03d007SEdward Tomasz Napierala int 14737a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, 14747a03d007SEdward Tomasz Napierala uint32_t *transfer_tag, void **prvp) 14757a03d007SEdward Tomasz Napierala { 14767a03d007SEdward Tomasz Napierala 14777a03d007SEdward Tomasz Napierala return (0); 14787a03d007SEdward Tomasz Napierala } 14797a03d007SEdward Tomasz Napierala 14807a03d007SEdward Tomasz Napierala void 14817a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv) 14827a03d007SEdward Tomasz Napierala { 14837a03d007SEdward Tomasz Napierala } 14847a03d007SEdward Tomasz Napierala 1485321b17ecSEdward Tomasz Napierala static int 148697b84d34SNavdeep Parhar icl_soft_limits(struct icl_drv_limits *idl) 1487321b17ecSEdward Tomasz Napierala { 1488321b17ecSEdward Tomasz Napierala 148997b84d34SNavdeep Parhar idl->idl_max_recv_data_segment_length = 128 * 1024; 1490321b17ecSEdward Tomasz Napierala 1491321b17ecSEdward Tomasz Napierala return (0); 1492321b17ecSEdward Tomasz Napierala } 1493321b17ecSEdward Tomasz Napierala 1494321b17ecSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1495321b17ecSEdward Tomasz Napierala int 1496f41492b0SEdward Tomasz Napierala icl_soft_conn_connect(struct icl_conn *ic, int domain, int socktype, 1497f41492b0SEdward Tomasz Napierala int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa) 1498f41492b0SEdward Tomasz Napierala { 1499f41492b0SEdward Tomasz Napierala 1500f41492b0SEdward Tomasz Napierala return (icl_soft_proxy_connect(ic, domain, socktype, protocol, 1501f41492b0SEdward Tomasz Napierala from_sa, to_sa)); 1502f41492b0SEdward Tomasz Napierala } 1503f41492b0SEdward Tomasz Napierala 1504f41492b0SEdward Tomasz Napierala int 1505f41492b0SEdward Tomasz Napierala icl_soft_handoff_sock(struct icl_conn *ic, struct socket *so) 1506321b17ecSEdward Tomasz Napierala { 1507321b17ecSEdward Tomasz Napierala int error; 1508321b17ecSEdward Tomasz Napierala 1509321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1510321b17ecSEdward Tomasz Napierala 1511321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) 1512321b17ecSEdward Tomasz Napierala return (EINVAL); 1513321b17ecSEdward Tomasz Napierala 1514321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1515321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1516321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1517321b17ecSEdward Tomasz Napierala return (EBUSY); 1518321b17ecSEdward Tomasz Napierala } 1519321b17ecSEdward Tomasz Napierala ic->ic_socket = so; 1520321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1521321b17ecSEdward Tomasz Napierala 1522321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1523321b17ecSEdward Tomasz Napierala 1524321b17ecSEdward Tomasz Napierala return (error); 1525321b17ecSEdward Tomasz Napierala } 1526321b17ecSEdward Tomasz Napierala #endif /* ICL_KERNEL_PROXY */ 1527321b17ecSEdward Tomasz Napierala 1528321b17ecSEdward Tomasz Napierala static int 1529321b17ecSEdward Tomasz Napierala icl_soft_load(void) 1530321b17ecSEdward Tomasz Napierala { 1531321b17ecSEdward Tomasz Napierala int error; 1532321b17ecSEdward Tomasz Napierala 1533321b17ecSEdward Tomasz Napierala icl_pdu_zone = uma_zcreate("icl_pdu", 1534321b17ecSEdward Tomasz Napierala sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, 1535321b17ecSEdward Tomasz Napierala UMA_ALIGN_PTR, 0); 1536321b17ecSEdward Tomasz Napierala refcount_init(&icl_ncons, 0); 1537321b17ecSEdward Tomasz Napierala 1538321b17ecSEdward Tomasz Napierala /* 1539321b17ecSEdward Tomasz Napierala * The reason we call this "none" is that to the user, 1540321b17ecSEdward Tomasz Napierala * it's known as "offload driver"; "offload driver: soft" 1541321b17ecSEdward Tomasz Napierala * doesn't make much sense. 1542321b17ecSEdward Tomasz Napierala */ 1543b8911594SEdward Tomasz Napierala error = icl_register("none", false, 0, 1544b8911594SEdward Tomasz Napierala icl_soft_limits, icl_soft_new_conn); 1545321b17ecSEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1546321b17ecSEdward Tomasz Napierala 1547b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0 1548b8911594SEdward Tomasz Napierala /* 1549b8911594SEdward Tomasz Napierala * Debugging aid for kernel proxy functionality. 1550b8911594SEdward Tomasz Napierala */ 1551b8911594SEdward Tomasz Napierala error = icl_register("proxytest", true, 0, 1552b8911594SEdward Tomasz Napierala icl_soft_limits, icl_soft_new_conn); 1553b8911594SEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1554b8911594SEdward Tomasz Napierala #endif 1555b8911594SEdward Tomasz Napierala 1556321b17ecSEdward Tomasz Napierala return (error); 1557321b17ecSEdward Tomasz Napierala } 1558321b17ecSEdward Tomasz Napierala 1559321b17ecSEdward Tomasz Napierala static int 1560321b17ecSEdward Tomasz Napierala icl_soft_unload(void) 1561321b17ecSEdward Tomasz Napierala { 1562321b17ecSEdward Tomasz Napierala 1563321b17ecSEdward Tomasz Napierala if (icl_ncons != 0) 1564321b17ecSEdward Tomasz Napierala return (EBUSY); 1565321b17ecSEdward Tomasz Napierala 1566b8911594SEdward Tomasz Napierala icl_unregister("none", false); 1567b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0 1568b8911594SEdward Tomasz Napierala icl_unregister("proxytest", true); 1569b8911594SEdward Tomasz Napierala #endif 1570321b17ecSEdward Tomasz Napierala 1571321b17ecSEdward Tomasz Napierala uma_zdestroy(icl_pdu_zone); 1572321b17ecSEdward Tomasz Napierala 1573321b17ecSEdward Tomasz Napierala return (0); 1574321b17ecSEdward Tomasz Napierala } 1575321b17ecSEdward Tomasz Napierala 1576321b17ecSEdward Tomasz Napierala static int 1577321b17ecSEdward Tomasz Napierala icl_soft_modevent(module_t mod, int what, void *arg) 1578321b17ecSEdward Tomasz Napierala { 1579321b17ecSEdward Tomasz Napierala 1580321b17ecSEdward Tomasz Napierala switch (what) { 1581321b17ecSEdward Tomasz Napierala case MOD_LOAD: 1582321b17ecSEdward Tomasz Napierala return (icl_soft_load()); 1583321b17ecSEdward Tomasz Napierala case MOD_UNLOAD: 1584321b17ecSEdward Tomasz Napierala return (icl_soft_unload()); 1585321b17ecSEdward Tomasz Napierala default: 1586321b17ecSEdward Tomasz Napierala return (EINVAL); 1587321b17ecSEdward Tomasz Napierala } 1588321b17ecSEdward Tomasz Napierala } 1589321b17ecSEdward Tomasz Napierala 1590321b17ecSEdward Tomasz Napierala moduledata_t icl_soft_data = { 1591321b17ecSEdward Tomasz Napierala "icl_soft", 1592321b17ecSEdward Tomasz Napierala icl_soft_modevent, 1593321b17ecSEdward Tomasz Napierala 0 1594321b17ecSEdward Tomasz Napierala }; 1595321b17ecSEdward Tomasz Napierala 1596321b17ecSEdward Tomasz Napierala DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 1597321b17ecSEdward Tomasz Napierala MODULE_DEPEND(icl_soft, icl, 1, 1, 1); 1598872d2d92SEdward Tomasz Napierala MODULE_VERSION(icl_soft, 1); 1599