1*321b17ecSEdward Tomasz Napierala /*- 2*321b17ecSEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 3*321b17ecSEdward Tomasz Napierala * All rights reserved. 4*321b17ecSEdward Tomasz Napierala * 5*321b17ecSEdward Tomasz Napierala * This software was developed by Edward Tomasz Napierala under sponsorship 6*321b17ecSEdward Tomasz Napierala * from the FreeBSD Foundation. 7*321b17ecSEdward Tomasz Napierala * 8*321b17ecSEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 9*321b17ecSEdward Tomasz Napierala * modification, are permitted provided that the following conditions 10*321b17ecSEdward Tomasz Napierala * are met: 11*321b17ecSEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 12*321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 13*321b17ecSEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 14*321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 15*321b17ecSEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 16*321b17ecSEdward Tomasz Napierala * 17*321b17ecSEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18*321b17ecSEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19*321b17ecSEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20*321b17ecSEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21*321b17ecSEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22*321b17ecSEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23*321b17ecSEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24*321b17ecSEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25*321b17ecSEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26*321b17ecSEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27*321b17ecSEdward Tomasz Napierala * SUCH DAMAGE. 28*321b17ecSEdward Tomasz Napierala * 29*321b17ecSEdward Tomasz Napierala */ 30*321b17ecSEdward Tomasz Napierala 31*321b17ecSEdward Tomasz Napierala /* 32*321b17ecSEdward Tomasz Napierala * iSCSI Common Layer. It's used by both the initiator and target to send 33*321b17ecSEdward Tomasz Napierala * and receive iSCSI PDUs. 34*321b17ecSEdward Tomasz Napierala */ 35*321b17ecSEdward Tomasz Napierala 36*321b17ecSEdward Tomasz Napierala #include <sys/cdefs.h> 37*321b17ecSEdward Tomasz Napierala __FBSDID("$FreeBSD$"); 38*321b17ecSEdward Tomasz Napierala 39*321b17ecSEdward Tomasz Napierala #include <sys/param.h> 40*321b17ecSEdward Tomasz Napierala #include <sys/capsicum.h> 41*321b17ecSEdward Tomasz Napierala #include <sys/condvar.h> 42*321b17ecSEdward Tomasz Napierala #include <sys/conf.h> 43*321b17ecSEdward Tomasz Napierala #include <sys/file.h> 44*321b17ecSEdward Tomasz Napierala #include <sys/kernel.h> 45*321b17ecSEdward Tomasz Napierala #include <sys/kthread.h> 46*321b17ecSEdward Tomasz Napierala #include <sys/lock.h> 47*321b17ecSEdward Tomasz Napierala #include <sys/mbuf.h> 48*321b17ecSEdward Tomasz Napierala #include <sys/mutex.h> 49*321b17ecSEdward Tomasz Napierala #include <sys/module.h> 50*321b17ecSEdward Tomasz Napierala #include <sys/protosw.h> 51*321b17ecSEdward Tomasz Napierala #include <sys/socket.h> 52*321b17ecSEdward Tomasz Napierala #include <sys/socketvar.h> 53*321b17ecSEdward Tomasz Napierala #include <sys/sysctl.h> 54*321b17ecSEdward Tomasz Napierala #include <sys/systm.h> 55*321b17ecSEdward Tomasz Napierala #include <sys/sx.h> 56*321b17ecSEdward Tomasz Napierala #include <sys/uio.h> 57*321b17ecSEdward Tomasz Napierala #include <vm/uma.h> 58*321b17ecSEdward Tomasz Napierala #include <netinet/in.h> 59*321b17ecSEdward Tomasz Napierala #include <netinet/tcp.h> 60*321b17ecSEdward Tomasz Napierala 61*321b17ecSEdward Tomasz Napierala #include <dev/iscsi/icl.h> 62*321b17ecSEdward Tomasz Napierala #include <dev/iscsi/iscsi_proto.h> 63*321b17ecSEdward Tomasz Napierala #include <icl_conn_if.h> 64*321b17ecSEdward Tomasz Napierala 65*321b17ecSEdward Tomasz Napierala static int coalesce = 1; 66*321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN, 67*321b17ecSEdward Tomasz Napierala &coalesce, 0, "Try to coalesce PDUs before sending"); 68*321b17ecSEdward Tomasz Napierala static int partial_receive_len = 128 * 1024; 69*321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 70*321b17ecSEdward Tomasz Napierala &partial_receive_len, 0, "Minimum read size for partially received " 71*321b17ecSEdward Tomasz Napierala "data segment"); 72*321b17ecSEdward Tomasz Napierala static int sendspace = 1048576; 73*321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN, 74*321b17ecSEdward Tomasz Napierala &sendspace, 0, "Default send socket buffer size"); 75*321b17ecSEdward Tomasz Napierala static int recvspace = 1048576; 76*321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN, 77*321b17ecSEdward Tomasz Napierala &recvspace, 0, "Default receive socket buffer size"); 78*321b17ecSEdward Tomasz Napierala 79*321b17ecSEdward Tomasz Napierala static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); 80*321b17ecSEdward Tomasz Napierala static uma_zone_t icl_pdu_zone; 81*321b17ecSEdward Tomasz Napierala 82*321b17ecSEdward Tomasz Napierala static volatile u_int icl_ncons; 83*321b17ecSEdward Tomasz Napierala 84*321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) 85*321b17ecSEdward Tomasz Napierala #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) 86*321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) 87*321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) 88*321b17ecSEdward Tomasz Napierala 89*321b17ecSEdward Tomasz Napierala STAILQ_HEAD(icl_pdu_stailq, icl_pdu); 90*321b17ecSEdward Tomasz Napierala 91*321b17ecSEdward Tomasz Napierala static icl_conn_new_pdu_t icl_soft_conn_new_pdu; 92*321b17ecSEdward Tomasz Napierala static icl_conn_pdu_free_t icl_soft_conn_pdu_free; 93*321b17ecSEdward Tomasz Napierala static icl_conn_pdu_data_segment_length_t 94*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length; 95*321b17ecSEdward Tomasz Napierala static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; 96*321b17ecSEdward Tomasz Napierala static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; 97*321b17ecSEdward Tomasz Napierala static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; 98*321b17ecSEdward Tomasz Napierala static icl_conn_handoff_t icl_soft_conn_handoff; 99*321b17ecSEdward Tomasz Napierala static icl_conn_free_t icl_soft_conn_free; 100*321b17ecSEdward Tomasz Napierala static icl_conn_close_t icl_soft_conn_close; 101*321b17ecSEdward Tomasz Napierala static icl_conn_connected_t icl_soft_conn_connected; 102*321b17ecSEdward Tomasz Napierala 103*321b17ecSEdward Tomasz Napierala static kobj_method_t icl_soft_methods[] = { 104*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), 105*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), 106*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_data_segment_length, 107*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length), 108*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), 109*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), 110*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), 111*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), 112*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_free, icl_soft_conn_free), 113*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_close, icl_soft_conn_close), 114*321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_connected, icl_soft_conn_connected), 115*321b17ecSEdward Tomasz Napierala { 0, 0 } 116*321b17ecSEdward Tomasz Napierala }; 117*321b17ecSEdward Tomasz Napierala 118*321b17ecSEdward Tomasz Napierala DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn)); 119*321b17ecSEdward Tomasz Napierala 120*321b17ecSEdward Tomasz Napierala static void icl_conn_close(struct icl_conn *ic); 121*321b17ecSEdward Tomasz Napierala 122*321b17ecSEdward Tomasz Napierala static void 123*321b17ecSEdward Tomasz Napierala icl_conn_fail(struct icl_conn *ic) 124*321b17ecSEdward Tomasz Napierala { 125*321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) 126*321b17ecSEdward Tomasz Napierala return; 127*321b17ecSEdward Tomasz Napierala 128*321b17ecSEdward Tomasz Napierala /* 129*321b17ecSEdward Tomasz Napierala * XXX 130*321b17ecSEdward Tomasz Napierala */ 131*321b17ecSEdward Tomasz Napierala ic->ic_socket->so_error = EDOOFUS; 132*321b17ecSEdward Tomasz Napierala (ic->ic_error)(ic); 133*321b17ecSEdward Tomasz Napierala } 134*321b17ecSEdward Tomasz Napierala 135*321b17ecSEdward Tomasz Napierala static struct mbuf * 136*321b17ecSEdward Tomasz Napierala icl_conn_receive(struct icl_conn *ic, size_t len) 137*321b17ecSEdward Tomasz Napierala { 138*321b17ecSEdward Tomasz Napierala struct uio uio; 139*321b17ecSEdward Tomasz Napierala struct socket *so; 140*321b17ecSEdward Tomasz Napierala struct mbuf *m; 141*321b17ecSEdward Tomasz Napierala int error, flags; 142*321b17ecSEdward Tomasz Napierala 143*321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 144*321b17ecSEdward Tomasz Napierala 145*321b17ecSEdward Tomasz Napierala memset(&uio, 0, sizeof(uio)); 146*321b17ecSEdward Tomasz Napierala uio.uio_resid = len; 147*321b17ecSEdward Tomasz Napierala 148*321b17ecSEdward Tomasz Napierala flags = MSG_DONTWAIT; 149*321b17ecSEdward Tomasz Napierala error = soreceive(so, NULL, &uio, &m, NULL, &flags); 150*321b17ecSEdward Tomasz Napierala if (error != 0) { 151*321b17ecSEdward Tomasz Napierala ICL_DEBUG("soreceive error %d", error); 152*321b17ecSEdward Tomasz Napierala return (NULL); 153*321b17ecSEdward Tomasz Napierala } 154*321b17ecSEdward Tomasz Napierala if (uio.uio_resid != 0) { 155*321b17ecSEdward Tomasz Napierala m_freem(m); 156*321b17ecSEdward Tomasz Napierala ICL_DEBUG("short read"); 157*321b17ecSEdward Tomasz Napierala return (NULL); 158*321b17ecSEdward Tomasz Napierala } 159*321b17ecSEdward Tomasz Napierala 160*321b17ecSEdward Tomasz Napierala return (m); 161*321b17ecSEdward Tomasz Napierala } 162*321b17ecSEdward Tomasz Napierala 163*321b17ecSEdward Tomasz Napierala static struct icl_pdu * 164*321b17ecSEdward Tomasz Napierala icl_pdu_new_empty(struct icl_conn *ic, int flags) 165*321b17ecSEdward Tomasz Napierala { 166*321b17ecSEdward Tomasz Napierala struct icl_pdu *ip; 167*321b17ecSEdward Tomasz Napierala 168*321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 169*321b17ecSEdward Tomasz Napierala refcount_acquire(&ic->ic_outstanding_pdus); 170*321b17ecSEdward Tomasz Napierala #endif 171*321b17ecSEdward Tomasz Napierala ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO); 172*321b17ecSEdward Tomasz Napierala if (ip == NULL) { 173*321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); 174*321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 175*321b17ecSEdward Tomasz Napierala refcount_release(&ic->ic_outstanding_pdus); 176*321b17ecSEdward Tomasz Napierala #endif 177*321b17ecSEdward Tomasz Napierala return (NULL); 178*321b17ecSEdward Tomasz Napierala } 179*321b17ecSEdward Tomasz Napierala 180*321b17ecSEdward Tomasz Napierala ip->ip_conn = ic; 181*321b17ecSEdward Tomasz Napierala 182*321b17ecSEdward Tomasz Napierala return (ip); 183*321b17ecSEdward Tomasz Napierala } 184*321b17ecSEdward Tomasz Napierala 185*321b17ecSEdward Tomasz Napierala static void 186*321b17ecSEdward Tomasz Napierala icl_pdu_free(struct icl_pdu *ip) 187*321b17ecSEdward Tomasz Napierala { 188*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 189*321b17ecSEdward Tomasz Napierala 190*321b17ecSEdward Tomasz Napierala ic = ip->ip_conn; 191*321b17ecSEdward Tomasz Napierala 192*321b17ecSEdward Tomasz Napierala m_freem(ip->ip_bhs_mbuf); 193*321b17ecSEdward Tomasz Napierala m_freem(ip->ip_ahs_mbuf); 194*321b17ecSEdward Tomasz Napierala m_freem(ip->ip_data_mbuf); 195*321b17ecSEdward Tomasz Napierala uma_zfree(icl_pdu_zone, ip); 196*321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 197*321b17ecSEdward Tomasz Napierala refcount_release(&ic->ic_outstanding_pdus); 198*321b17ecSEdward Tomasz Napierala #endif 199*321b17ecSEdward Tomasz Napierala } 200*321b17ecSEdward Tomasz Napierala 201*321b17ecSEdward Tomasz Napierala void 202*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 203*321b17ecSEdward Tomasz Napierala { 204*321b17ecSEdward Tomasz Napierala icl_pdu_free(ip); 205*321b17ecSEdward Tomasz Napierala } 206*321b17ecSEdward Tomasz Napierala 207*321b17ecSEdward Tomasz Napierala /* 208*321b17ecSEdward Tomasz Napierala * Allocate icl_pdu with empty BHS to fill up by the caller. 209*321b17ecSEdward Tomasz Napierala */ 210*321b17ecSEdward Tomasz Napierala struct icl_pdu * 211*321b17ecSEdward Tomasz Napierala icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) 212*321b17ecSEdward Tomasz Napierala { 213*321b17ecSEdward Tomasz Napierala struct icl_pdu *ip; 214*321b17ecSEdward Tomasz Napierala 215*321b17ecSEdward Tomasz Napierala ip = icl_pdu_new_empty(ic, flags); 216*321b17ecSEdward Tomasz Napierala if (ip == NULL) 217*321b17ecSEdward Tomasz Napierala return (NULL); 218*321b17ecSEdward Tomasz Napierala 219*321b17ecSEdward Tomasz Napierala ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs), 220*321b17ecSEdward Tomasz Napierala flags, MT_DATA, M_PKTHDR); 221*321b17ecSEdward Tomasz Napierala if (ip->ip_bhs_mbuf == NULL) { 222*321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate %zd bytes", sizeof(*ip)); 223*321b17ecSEdward Tomasz Napierala icl_pdu_free(ip); 224*321b17ecSEdward Tomasz Napierala return (NULL); 225*321b17ecSEdward Tomasz Napierala } 226*321b17ecSEdward Tomasz Napierala ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); 227*321b17ecSEdward Tomasz Napierala memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); 228*321b17ecSEdward Tomasz Napierala ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); 229*321b17ecSEdward Tomasz Napierala 230*321b17ecSEdward Tomasz Napierala return (ip); 231*321b17ecSEdward Tomasz Napierala } 232*321b17ecSEdward Tomasz Napierala 233*321b17ecSEdward Tomasz Napierala static int 234*321b17ecSEdward Tomasz Napierala icl_pdu_ahs_length(const struct icl_pdu *request) 235*321b17ecSEdward Tomasz Napierala { 236*321b17ecSEdward Tomasz Napierala 237*321b17ecSEdward Tomasz Napierala return (request->ip_bhs->bhs_total_ahs_len * 4); 238*321b17ecSEdward Tomasz Napierala } 239*321b17ecSEdward Tomasz Napierala 240*321b17ecSEdward Tomasz Napierala static size_t 241*321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_length(const struct icl_pdu *request) 242*321b17ecSEdward Tomasz Napierala { 243*321b17ecSEdward Tomasz Napierala uint32_t len = 0; 244*321b17ecSEdward Tomasz Napierala 245*321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[0]; 246*321b17ecSEdward Tomasz Napierala len <<= 8; 247*321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[1]; 248*321b17ecSEdward Tomasz Napierala len <<= 8; 249*321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[2]; 250*321b17ecSEdward Tomasz Napierala 251*321b17ecSEdward Tomasz Napierala return (len); 252*321b17ecSEdward Tomasz Napierala } 253*321b17ecSEdward Tomasz Napierala 254*321b17ecSEdward Tomasz Napierala size_t 255*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, 256*321b17ecSEdward Tomasz Napierala const struct icl_pdu *request) 257*321b17ecSEdward Tomasz Napierala { 258*321b17ecSEdward Tomasz Napierala 259*321b17ecSEdward Tomasz Napierala return (icl_pdu_data_segment_length(request)); 260*321b17ecSEdward Tomasz Napierala } 261*321b17ecSEdward Tomasz Napierala 262*321b17ecSEdward Tomasz Napierala static void 263*321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) 264*321b17ecSEdward Tomasz Napierala { 265*321b17ecSEdward Tomasz Napierala 266*321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[2] = len; 267*321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[1] = len >> 8; 268*321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[0] = len >> 16; 269*321b17ecSEdward Tomasz Napierala } 270*321b17ecSEdward Tomasz Napierala 271*321b17ecSEdward Tomasz Napierala static size_t 272*321b17ecSEdward Tomasz Napierala icl_pdu_padding(const struct icl_pdu *ip) 273*321b17ecSEdward Tomasz Napierala { 274*321b17ecSEdward Tomasz Napierala 275*321b17ecSEdward Tomasz Napierala if ((ip->ip_data_len % 4) != 0) 276*321b17ecSEdward Tomasz Napierala return (4 - (ip->ip_data_len % 4)); 277*321b17ecSEdward Tomasz Napierala 278*321b17ecSEdward Tomasz Napierala return (0); 279*321b17ecSEdward Tomasz Napierala } 280*321b17ecSEdward Tomasz Napierala 281*321b17ecSEdward Tomasz Napierala static size_t 282*321b17ecSEdward Tomasz Napierala icl_pdu_size(const struct icl_pdu *response) 283*321b17ecSEdward Tomasz Napierala { 284*321b17ecSEdward Tomasz Napierala size_t len; 285*321b17ecSEdward Tomasz Napierala 286*321b17ecSEdward Tomasz Napierala KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); 287*321b17ecSEdward Tomasz Napierala 288*321b17ecSEdward Tomasz Napierala len = sizeof(struct iscsi_bhs) + response->ip_data_len + 289*321b17ecSEdward Tomasz Napierala icl_pdu_padding(response); 290*321b17ecSEdward Tomasz Napierala if (response->ip_conn->ic_header_crc32c) 291*321b17ecSEdward Tomasz Napierala len += ISCSI_HEADER_DIGEST_SIZE; 292*321b17ecSEdward Tomasz Napierala if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) 293*321b17ecSEdward Tomasz Napierala len += ISCSI_DATA_DIGEST_SIZE; 294*321b17ecSEdward Tomasz Napierala 295*321b17ecSEdward Tomasz Napierala return (len); 296*321b17ecSEdward Tomasz Napierala } 297*321b17ecSEdward Tomasz Napierala 298*321b17ecSEdward Tomasz Napierala static int 299*321b17ecSEdward Tomasz Napierala icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep) 300*321b17ecSEdward Tomasz Napierala { 301*321b17ecSEdward Tomasz Napierala struct mbuf *m; 302*321b17ecSEdward Tomasz Napierala 303*321b17ecSEdward Tomasz Napierala m = icl_conn_receive(request->ip_conn, sizeof(struct iscsi_bhs)); 304*321b17ecSEdward Tomasz Napierala if (m == NULL) { 305*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive BHS"); 306*321b17ecSEdward Tomasz Napierala return (-1); 307*321b17ecSEdward Tomasz Napierala } 308*321b17ecSEdward Tomasz Napierala 309*321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf = m_pullup(m, sizeof(struct iscsi_bhs)); 310*321b17ecSEdward Tomasz Napierala if (request->ip_bhs_mbuf == NULL) { 311*321b17ecSEdward Tomasz Napierala ICL_WARN("m_pullup failed"); 312*321b17ecSEdward Tomasz Napierala return (-1); 313*321b17ecSEdward Tomasz Napierala } 314*321b17ecSEdward Tomasz Napierala request->ip_bhs = mtod(request->ip_bhs_mbuf, struct iscsi_bhs *); 315*321b17ecSEdward Tomasz Napierala 316*321b17ecSEdward Tomasz Napierala /* 317*321b17ecSEdward Tomasz Napierala * XXX: For architectures with strict alignment requirements 318*321b17ecSEdward Tomasz Napierala * we may need to allocate ip_bhs and copy the data into it. 319*321b17ecSEdward Tomasz Napierala * For some reason, though, not doing this doesn't seem 320*321b17ecSEdward Tomasz Napierala * to cause problems; tested on sparc64. 321*321b17ecSEdward Tomasz Napierala */ 322*321b17ecSEdward Tomasz Napierala 323*321b17ecSEdward Tomasz Napierala *availablep -= sizeof(struct iscsi_bhs); 324*321b17ecSEdward Tomasz Napierala return (0); 325*321b17ecSEdward Tomasz Napierala } 326*321b17ecSEdward Tomasz Napierala 327*321b17ecSEdward Tomasz Napierala static int 328*321b17ecSEdward Tomasz Napierala icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep) 329*321b17ecSEdward Tomasz Napierala { 330*321b17ecSEdward Tomasz Napierala 331*321b17ecSEdward Tomasz Napierala request->ip_ahs_len = icl_pdu_ahs_length(request); 332*321b17ecSEdward Tomasz Napierala if (request->ip_ahs_len == 0) 333*321b17ecSEdward Tomasz Napierala return (0); 334*321b17ecSEdward Tomasz Napierala 335*321b17ecSEdward Tomasz Napierala request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn, 336*321b17ecSEdward Tomasz Napierala request->ip_ahs_len); 337*321b17ecSEdward Tomasz Napierala if (request->ip_ahs_mbuf == NULL) { 338*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive AHS"); 339*321b17ecSEdward Tomasz Napierala return (-1); 340*321b17ecSEdward Tomasz Napierala } 341*321b17ecSEdward Tomasz Napierala 342*321b17ecSEdward Tomasz Napierala *availablep -= request->ip_ahs_len; 343*321b17ecSEdward Tomasz Napierala return (0); 344*321b17ecSEdward Tomasz Napierala } 345*321b17ecSEdward Tomasz Napierala 346*321b17ecSEdward Tomasz Napierala static uint32_t 347*321b17ecSEdward Tomasz Napierala icl_mbuf_to_crc32c(const struct mbuf *m0) 348*321b17ecSEdward Tomasz Napierala { 349*321b17ecSEdward Tomasz Napierala uint32_t digest = 0xffffffff; 350*321b17ecSEdward Tomasz Napierala const struct mbuf *m; 351*321b17ecSEdward Tomasz Napierala 352*321b17ecSEdward Tomasz Napierala for (m = m0; m != NULL; m = m->m_next) 353*321b17ecSEdward Tomasz Napierala digest = calculate_crc32c(digest, 354*321b17ecSEdward Tomasz Napierala mtod(m, const void *), m->m_len); 355*321b17ecSEdward Tomasz Napierala 356*321b17ecSEdward Tomasz Napierala digest = digest ^ 0xffffffff; 357*321b17ecSEdward Tomasz Napierala 358*321b17ecSEdward Tomasz Napierala return (digest); 359*321b17ecSEdward Tomasz Napierala } 360*321b17ecSEdward Tomasz Napierala 361*321b17ecSEdward Tomasz Napierala static int 362*321b17ecSEdward Tomasz Napierala icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep) 363*321b17ecSEdward Tomasz Napierala { 364*321b17ecSEdward Tomasz Napierala struct mbuf *m; 365*321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 366*321b17ecSEdward Tomasz Napierala 367*321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_header_crc32c == false) 368*321b17ecSEdward Tomasz Napierala return (0); 369*321b17ecSEdward Tomasz Napierala 370*321b17ecSEdward Tomasz Napierala m = icl_conn_receive(request->ip_conn, ISCSI_HEADER_DIGEST_SIZE); 371*321b17ecSEdward Tomasz Napierala if (m == NULL) { 372*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive header digest"); 373*321b17ecSEdward Tomasz Napierala return (-1); 374*321b17ecSEdward Tomasz Napierala } 375*321b17ecSEdward Tomasz Napierala 376*321b17ecSEdward Tomasz Napierala CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); 377*321b17ecSEdward Tomasz Napierala m_copydata(m, 0, ISCSI_HEADER_DIGEST_SIZE, (void *)&received_digest); 378*321b17ecSEdward Tomasz Napierala m_freem(m); 379*321b17ecSEdward Tomasz Napierala 380*321b17ecSEdward Tomasz Napierala *availablep -= ISCSI_HEADER_DIGEST_SIZE; 381*321b17ecSEdward Tomasz Napierala 382*321b17ecSEdward Tomasz Napierala /* 383*321b17ecSEdward Tomasz Napierala * XXX: Handle AHS. 384*321b17ecSEdward Tomasz Napierala */ 385*321b17ecSEdward Tomasz Napierala valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 386*321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 387*321b17ecSEdward Tomasz Napierala ICL_WARN("header digest check failed; got 0x%x, " 388*321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 389*321b17ecSEdward Tomasz Napierala return (-1); 390*321b17ecSEdward Tomasz Napierala } 391*321b17ecSEdward Tomasz Napierala 392*321b17ecSEdward Tomasz Napierala return (0); 393*321b17ecSEdward Tomasz Napierala } 394*321b17ecSEdward Tomasz Napierala 395*321b17ecSEdward Tomasz Napierala /* 396*321b17ecSEdward Tomasz Napierala * Return the number of bytes that should be waiting in the receive socket 397*321b17ecSEdward Tomasz Napierala * before icl_pdu_receive_data_segment() gets called. 398*321b17ecSEdward Tomasz Napierala */ 399*321b17ecSEdward Tomasz Napierala static size_t 400*321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(const struct icl_pdu *request) 401*321b17ecSEdward Tomasz Napierala { 402*321b17ecSEdward Tomasz Napierala size_t len; 403*321b17ecSEdward Tomasz Napierala 404*321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 405*321b17ecSEdward Tomasz Napierala if (len == 0) 406*321b17ecSEdward Tomasz Napierala return (0); 407*321b17ecSEdward Tomasz Napierala 408*321b17ecSEdward Tomasz Napierala /* 409*321b17ecSEdward Tomasz Napierala * Account for the parts of data segment already read from 410*321b17ecSEdward Tomasz Napierala * the socket buffer. 411*321b17ecSEdward Tomasz Napierala */ 412*321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 413*321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 414*321b17ecSEdward Tomasz Napierala 415*321b17ecSEdward Tomasz Napierala /* 416*321b17ecSEdward Tomasz Napierala * Don't always wait for the full data segment to be delivered 417*321b17ecSEdward Tomasz Napierala * to the socket; this might badly affect performance due to 418*321b17ecSEdward Tomasz Napierala * TCP window scaling. 419*321b17ecSEdward Tomasz Napierala */ 420*321b17ecSEdward Tomasz Napierala if (len > partial_receive_len) { 421*321b17ecSEdward Tomasz Napierala #if 0 422*321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data, limiting to %zd", 423*321b17ecSEdward Tomasz Napierala len, partial_receive_len)); 424*321b17ecSEdward Tomasz Napierala #endif 425*321b17ecSEdward Tomasz Napierala len = partial_receive_len; 426*321b17ecSEdward Tomasz Napierala 427*321b17ecSEdward Tomasz Napierala return (len); 428*321b17ecSEdward Tomasz Napierala } 429*321b17ecSEdward Tomasz Napierala 430*321b17ecSEdward Tomasz Napierala /* 431*321b17ecSEdward Tomasz Napierala * Account for padding. Note that due to the way code is written, 432*321b17ecSEdward Tomasz Napierala * the icl_pdu_receive_data_segment() must always receive padding 433*321b17ecSEdward Tomasz Napierala * along with the last part of data segment, because it would be 434*321b17ecSEdward Tomasz Napierala * impossible to tell whether we've already received the full data 435*321b17ecSEdward Tomasz Napierala * segment including padding, or without it. 436*321b17ecSEdward Tomasz Napierala */ 437*321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 438*321b17ecSEdward Tomasz Napierala len += 4 - (len % 4); 439*321b17ecSEdward Tomasz Napierala 440*321b17ecSEdward Tomasz Napierala #if 0 441*321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data", len)); 442*321b17ecSEdward Tomasz Napierala #endif 443*321b17ecSEdward Tomasz Napierala 444*321b17ecSEdward Tomasz Napierala return (len); 445*321b17ecSEdward Tomasz Napierala } 446*321b17ecSEdward Tomasz Napierala 447*321b17ecSEdward Tomasz Napierala static int 448*321b17ecSEdward Tomasz Napierala icl_pdu_receive_data_segment(struct icl_pdu *request, 449*321b17ecSEdward Tomasz Napierala size_t *availablep, bool *more_neededp) 450*321b17ecSEdward Tomasz Napierala { 451*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 452*321b17ecSEdward Tomasz Napierala size_t len, padding = 0; 453*321b17ecSEdward Tomasz Napierala struct mbuf *m; 454*321b17ecSEdward Tomasz Napierala 455*321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 456*321b17ecSEdward Tomasz Napierala 457*321b17ecSEdward Tomasz Napierala *more_neededp = false; 458*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 459*321b17ecSEdward Tomasz Napierala 460*321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 461*321b17ecSEdward Tomasz Napierala if (len == 0) 462*321b17ecSEdward Tomasz Napierala return (0); 463*321b17ecSEdward Tomasz Napierala 464*321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 465*321b17ecSEdward Tomasz Napierala padding = 4 - (len % 4); 466*321b17ecSEdward Tomasz Napierala 467*321b17ecSEdward Tomasz Napierala /* 468*321b17ecSEdward Tomasz Napierala * Account for already received parts of data segment. 469*321b17ecSEdward Tomasz Napierala */ 470*321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 471*321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 472*321b17ecSEdward Tomasz Napierala 473*321b17ecSEdward Tomasz Napierala if (len + padding > *availablep) { 474*321b17ecSEdward Tomasz Napierala /* 475*321b17ecSEdward Tomasz Napierala * Not enough data in the socket buffer. Receive as much 476*321b17ecSEdward Tomasz Napierala * as we can. Don't receive padding, since, obviously, it's 477*321b17ecSEdward Tomasz Napierala * not the end of data segment yet. 478*321b17ecSEdward Tomasz Napierala */ 479*321b17ecSEdward Tomasz Napierala #if 0 480*321b17ecSEdward Tomasz Napierala ICL_DEBUG("limited from %zd to %zd", 481*321b17ecSEdward Tomasz Napierala len + padding, *availablep - padding)); 482*321b17ecSEdward Tomasz Napierala #endif 483*321b17ecSEdward Tomasz Napierala len = *availablep - padding; 484*321b17ecSEdward Tomasz Napierala *more_neededp = true; 485*321b17ecSEdward Tomasz Napierala padding = 0; 486*321b17ecSEdward Tomasz Napierala } 487*321b17ecSEdward Tomasz Napierala 488*321b17ecSEdward Tomasz Napierala /* 489*321b17ecSEdward Tomasz Napierala * Must not try to receive padding without at least one byte 490*321b17ecSEdward Tomasz Napierala * of actual data segment. 491*321b17ecSEdward Tomasz Napierala */ 492*321b17ecSEdward Tomasz Napierala if (len > 0) { 493*321b17ecSEdward Tomasz Napierala m = icl_conn_receive(request->ip_conn, len + padding); 494*321b17ecSEdward Tomasz Napierala if (m == NULL) { 495*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data segment"); 496*321b17ecSEdward Tomasz Napierala return (-1); 497*321b17ecSEdward Tomasz Napierala } 498*321b17ecSEdward Tomasz Napierala 499*321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) 500*321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = m; 501*321b17ecSEdward Tomasz Napierala else 502*321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, m); 503*321b17ecSEdward Tomasz Napierala 504*321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 505*321b17ecSEdward Tomasz Napierala *availablep -= len + padding; 506*321b17ecSEdward Tomasz Napierala } else 507*321b17ecSEdward Tomasz Napierala ICL_DEBUG("len 0"); 508*321b17ecSEdward Tomasz Napierala 509*321b17ecSEdward Tomasz Napierala if (*more_neededp) 510*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 511*321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(request); 512*321b17ecSEdward Tomasz Napierala 513*321b17ecSEdward Tomasz Napierala return (0); 514*321b17ecSEdward Tomasz Napierala } 515*321b17ecSEdward Tomasz Napierala 516*321b17ecSEdward Tomasz Napierala static int 517*321b17ecSEdward Tomasz Napierala icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep) 518*321b17ecSEdward Tomasz Napierala { 519*321b17ecSEdward Tomasz Napierala struct mbuf *m; 520*321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 521*321b17ecSEdward Tomasz Napierala 522*321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_data_crc32c == false) 523*321b17ecSEdward Tomasz Napierala return (0); 524*321b17ecSEdward Tomasz Napierala 525*321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0) 526*321b17ecSEdward Tomasz Napierala return (0); 527*321b17ecSEdward Tomasz Napierala 528*321b17ecSEdward Tomasz Napierala m = icl_conn_receive(request->ip_conn, ISCSI_DATA_DIGEST_SIZE); 529*321b17ecSEdward Tomasz Napierala if (m == NULL) { 530*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data digest"); 531*321b17ecSEdward Tomasz Napierala return (-1); 532*321b17ecSEdward Tomasz Napierala } 533*321b17ecSEdward Tomasz Napierala 534*321b17ecSEdward Tomasz Napierala CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); 535*321b17ecSEdward Tomasz Napierala m_copydata(m, 0, ISCSI_DATA_DIGEST_SIZE, (void *)&received_digest); 536*321b17ecSEdward Tomasz Napierala m_freem(m); 537*321b17ecSEdward Tomasz Napierala 538*321b17ecSEdward Tomasz Napierala *availablep -= ISCSI_DATA_DIGEST_SIZE; 539*321b17ecSEdward Tomasz Napierala 540*321b17ecSEdward Tomasz Napierala /* 541*321b17ecSEdward Tomasz Napierala * Note that ip_data_mbuf also contains padding; since digest 542*321b17ecSEdward Tomasz Napierala * calculation is supposed to include that, we iterate over 543*321b17ecSEdward Tomasz Napierala * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. 544*321b17ecSEdward Tomasz Napierala */ 545*321b17ecSEdward Tomasz Napierala valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 546*321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 547*321b17ecSEdward Tomasz Napierala ICL_WARN("data digest check failed; got 0x%x, " 548*321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 549*321b17ecSEdward Tomasz Napierala return (-1); 550*321b17ecSEdward Tomasz Napierala } 551*321b17ecSEdward Tomasz Napierala 552*321b17ecSEdward Tomasz Napierala return (0); 553*321b17ecSEdward Tomasz Napierala } 554*321b17ecSEdward Tomasz Napierala 555*321b17ecSEdward Tomasz Napierala /* 556*321b17ecSEdward Tomasz Napierala * Somewhat contrary to the name, this attempts to receive only one 557*321b17ecSEdward Tomasz Napierala * "part" of PDU at a time; call it repeatedly until it returns non-NULL. 558*321b17ecSEdward Tomasz Napierala */ 559*321b17ecSEdward Tomasz Napierala static struct icl_pdu * 560*321b17ecSEdward Tomasz Napierala icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep) 561*321b17ecSEdward Tomasz Napierala { 562*321b17ecSEdward Tomasz Napierala struct icl_pdu *request; 563*321b17ecSEdward Tomasz Napierala struct socket *so; 564*321b17ecSEdward Tomasz Napierala size_t len; 565*321b17ecSEdward Tomasz Napierala int error; 566*321b17ecSEdward Tomasz Napierala bool more_needed; 567*321b17ecSEdward Tomasz Napierala 568*321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 569*321b17ecSEdward Tomasz Napierala 570*321b17ecSEdward Tomasz Napierala if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { 571*321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_receive_pdu == NULL, 572*321b17ecSEdward Tomasz Napierala ("ic->ic_receive_pdu != NULL")); 573*321b17ecSEdward Tomasz Napierala request = icl_pdu_new_empty(ic, M_NOWAIT); 574*321b17ecSEdward Tomasz Napierala if (request == NULL) { 575*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to allocate PDU; " 576*321b17ecSEdward Tomasz Napierala "dropping connection"); 577*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 578*321b17ecSEdward Tomasz Napierala return (NULL); 579*321b17ecSEdward Tomasz Napierala } 580*321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = request; 581*321b17ecSEdward Tomasz Napierala } else { 582*321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_receive_pdu != NULL, 583*321b17ecSEdward Tomasz Napierala ("ic->ic_receive_pdu == NULL")); 584*321b17ecSEdward Tomasz Napierala request = ic->ic_receive_pdu; 585*321b17ecSEdward Tomasz Napierala } 586*321b17ecSEdward Tomasz Napierala 587*321b17ecSEdward Tomasz Napierala if (*availablep < ic->ic_receive_len) { 588*321b17ecSEdward Tomasz Napierala #if 0 589*321b17ecSEdward Tomasz Napierala ICL_DEBUG("not enough data; need %zd, " 590*321b17ecSEdward Tomasz Napierala "have %zd", ic->ic_receive_len, *availablep); 591*321b17ecSEdward Tomasz Napierala #endif 592*321b17ecSEdward Tomasz Napierala return (NULL); 593*321b17ecSEdward Tomasz Napierala } 594*321b17ecSEdward Tomasz Napierala 595*321b17ecSEdward Tomasz Napierala switch (ic->ic_receive_state) { 596*321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_BHS: 597*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving BHS"); 598*321b17ecSEdward Tomasz Napierala error = icl_pdu_receive_bhs(request, availablep); 599*321b17ecSEdward Tomasz Napierala if (error != 0) { 600*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive BHS; " 601*321b17ecSEdward Tomasz Napierala "dropping connection"); 602*321b17ecSEdward Tomasz Napierala break; 603*321b17ecSEdward Tomasz Napierala } 604*321b17ecSEdward Tomasz Napierala 605*321b17ecSEdward Tomasz Napierala /* 606*321b17ecSEdward Tomasz Napierala * We don't enforce any limit for AHS length; 607*321b17ecSEdward Tomasz Napierala * its length is stored in 8 bit field. 608*321b17ecSEdward Tomasz Napierala */ 609*321b17ecSEdward Tomasz Napierala 610*321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 611*321b17ecSEdward Tomasz Napierala if (len > ic->ic_max_data_segment_length) { 612*321b17ecSEdward Tomasz Napierala ICL_WARN("received data segment " 613*321b17ecSEdward Tomasz Napierala "length %zd is larger than negotiated " 614*321b17ecSEdward Tomasz Napierala "MaxDataSegmentLength %zd; " 615*321b17ecSEdward Tomasz Napierala "dropping connection", 616*321b17ecSEdward Tomasz Napierala len, ic->ic_max_data_segment_length); 617*321b17ecSEdward Tomasz Napierala error = EINVAL; 618*321b17ecSEdward Tomasz Napierala break; 619*321b17ecSEdward Tomasz Napierala } 620*321b17ecSEdward Tomasz Napierala 621*321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_AHS; 622*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = icl_pdu_ahs_length(request); 623*321b17ecSEdward Tomasz Napierala break; 624*321b17ecSEdward Tomasz Napierala 625*321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_AHS: 626*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving AHS"); 627*321b17ecSEdward Tomasz Napierala error = icl_pdu_receive_ahs(request, availablep); 628*321b17ecSEdward Tomasz Napierala if (error != 0) { 629*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive AHS; " 630*321b17ecSEdward Tomasz Napierala "dropping connection"); 631*321b17ecSEdward Tomasz Napierala break; 632*321b17ecSEdward Tomasz Napierala } 633*321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; 634*321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c == false) 635*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 636*321b17ecSEdward Tomasz Napierala else 637*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; 638*321b17ecSEdward Tomasz Napierala break; 639*321b17ecSEdward Tomasz Napierala 640*321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_HEADER_DIGEST: 641*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving header digest"); 642*321b17ecSEdward Tomasz Napierala error = icl_pdu_check_header_digest(request, availablep); 643*321b17ecSEdward Tomasz Napierala if (error != 0) { 644*321b17ecSEdward Tomasz Napierala ICL_DEBUG("header digest failed; " 645*321b17ecSEdward Tomasz Napierala "dropping connection"); 646*321b17ecSEdward Tomasz Napierala break; 647*321b17ecSEdward Tomasz Napierala } 648*321b17ecSEdward Tomasz Napierala 649*321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_DATA; 650*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 651*321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(request); 652*321b17ecSEdward Tomasz Napierala break; 653*321b17ecSEdward Tomasz Napierala 654*321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA: 655*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data segment"); 656*321b17ecSEdward Tomasz Napierala error = icl_pdu_receive_data_segment(request, availablep, 657*321b17ecSEdward Tomasz Napierala &more_needed); 658*321b17ecSEdward Tomasz Napierala if (error != 0) { 659*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data segment;" 660*321b17ecSEdward Tomasz Napierala "dropping connection"); 661*321b17ecSEdward Tomasz Napierala break; 662*321b17ecSEdward Tomasz Napierala } 663*321b17ecSEdward Tomasz Napierala 664*321b17ecSEdward Tomasz Napierala if (more_needed) 665*321b17ecSEdward Tomasz Napierala break; 666*321b17ecSEdward Tomasz Napierala 667*321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; 668*321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) 669*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 670*321b17ecSEdward Tomasz Napierala else 671*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; 672*321b17ecSEdward Tomasz Napierala break; 673*321b17ecSEdward Tomasz Napierala 674*321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA_DIGEST: 675*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data digest"); 676*321b17ecSEdward Tomasz Napierala error = icl_pdu_check_data_digest(request, availablep); 677*321b17ecSEdward Tomasz Napierala if (error != 0) { 678*321b17ecSEdward Tomasz Napierala ICL_DEBUG("data digest failed; " 679*321b17ecSEdward Tomasz Napierala "dropping connection"); 680*321b17ecSEdward Tomasz Napierala break; 681*321b17ecSEdward Tomasz Napierala } 682*321b17ecSEdward Tomasz Napierala 683*321b17ecSEdward Tomasz Napierala /* 684*321b17ecSEdward Tomasz Napierala * We've received complete PDU; reset the receive state machine 685*321b17ecSEdward Tomasz Napierala * and return the PDU. 686*321b17ecSEdward Tomasz Napierala */ 687*321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_BHS; 688*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = sizeof(struct iscsi_bhs); 689*321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = NULL; 690*321b17ecSEdward Tomasz Napierala return (request); 691*321b17ecSEdward Tomasz Napierala 692*321b17ecSEdward Tomasz Napierala default: 693*321b17ecSEdward Tomasz Napierala panic("invalid ic_receive_state %d\n", ic->ic_receive_state); 694*321b17ecSEdward Tomasz Napierala } 695*321b17ecSEdward Tomasz Napierala 696*321b17ecSEdward Tomasz Napierala if (error != 0) { 697*321b17ecSEdward Tomasz Napierala /* 698*321b17ecSEdward Tomasz Napierala * Don't free the PDU; it's pointed to by ic->ic_receive_pdu 699*321b17ecSEdward Tomasz Napierala * and will get freed in icl_conn_close(). 700*321b17ecSEdward Tomasz Napierala */ 701*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 702*321b17ecSEdward Tomasz Napierala } 703*321b17ecSEdward Tomasz Napierala 704*321b17ecSEdward Tomasz Napierala return (NULL); 705*321b17ecSEdward Tomasz Napierala } 706*321b17ecSEdward Tomasz Napierala 707*321b17ecSEdward Tomasz Napierala static void 708*321b17ecSEdward Tomasz Napierala icl_conn_receive_pdus(struct icl_conn *ic, size_t available) 709*321b17ecSEdward Tomasz Napierala { 710*321b17ecSEdward Tomasz Napierala struct icl_pdu *response; 711*321b17ecSEdward Tomasz Napierala struct socket *so; 712*321b17ecSEdward Tomasz Napierala 713*321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 714*321b17ecSEdward Tomasz Napierala 715*321b17ecSEdward Tomasz Napierala /* 716*321b17ecSEdward Tomasz Napierala * This can never happen; we're careful to only mess with ic->ic_socket 717*321b17ecSEdward Tomasz Napierala * pointer when the send/receive threads are not running. 718*321b17ecSEdward Tomasz Napierala */ 719*321b17ecSEdward Tomasz Napierala KASSERT(so != NULL, ("NULL socket")); 720*321b17ecSEdward Tomasz Napierala 721*321b17ecSEdward Tomasz Napierala for (;;) { 722*321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) 723*321b17ecSEdward Tomasz Napierala return; 724*321b17ecSEdward Tomasz Napierala 725*321b17ecSEdward Tomasz Napierala if (so->so_error != 0) { 726*321b17ecSEdward Tomasz Napierala ICL_DEBUG("connection error %d; " 727*321b17ecSEdward Tomasz Napierala "dropping connection", so->so_error); 728*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 729*321b17ecSEdward Tomasz Napierala return; 730*321b17ecSEdward Tomasz Napierala } 731*321b17ecSEdward Tomasz Napierala 732*321b17ecSEdward Tomasz Napierala /* 733*321b17ecSEdward Tomasz Napierala * Loop until we have a complete PDU or there is not enough 734*321b17ecSEdward Tomasz Napierala * data in the socket buffer. 735*321b17ecSEdward Tomasz Napierala */ 736*321b17ecSEdward Tomasz Napierala if (available < ic->ic_receive_len) { 737*321b17ecSEdward Tomasz Napierala #if 0 738*321b17ecSEdward Tomasz Napierala ICL_DEBUG("not enough data; have %zd, " 739*321b17ecSEdward Tomasz Napierala "need %zd", available, 740*321b17ecSEdward Tomasz Napierala ic->ic_receive_len); 741*321b17ecSEdward Tomasz Napierala #endif 742*321b17ecSEdward Tomasz Napierala return; 743*321b17ecSEdward Tomasz Napierala } 744*321b17ecSEdward Tomasz Napierala 745*321b17ecSEdward Tomasz Napierala response = icl_conn_receive_pdu(ic, &available); 746*321b17ecSEdward Tomasz Napierala if (response == NULL) 747*321b17ecSEdward Tomasz Napierala continue; 748*321b17ecSEdward Tomasz Napierala 749*321b17ecSEdward Tomasz Napierala if (response->ip_ahs_len > 0) { 750*321b17ecSEdward Tomasz Napierala ICL_WARN("received PDU with unsupported " 751*321b17ecSEdward Tomasz Napierala "AHS; opcode 0x%x; dropping connection", 752*321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_opcode); 753*321b17ecSEdward Tomasz Napierala icl_pdu_free(response); 754*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 755*321b17ecSEdward Tomasz Napierala return; 756*321b17ecSEdward Tomasz Napierala } 757*321b17ecSEdward Tomasz Napierala 758*321b17ecSEdward Tomasz Napierala (ic->ic_receive)(response); 759*321b17ecSEdward Tomasz Napierala } 760*321b17ecSEdward Tomasz Napierala } 761*321b17ecSEdward Tomasz Napierala 762*321b17ecSEdward Tomasz Napierala static void 763*321b17ecSEdward Tomasz Napierala icl_receive_thread(void *arg) 764*321b17ecSEdward Tomasz Napierala { 765*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 766*321b17ecSEdward Tomasz Napierala size_t available; 767*321b17ecSEdward Tomasz Napierala struct socket *so; 768*321b17ecSEdward Tomasz Napierala 769*321b17ecSEdward Tomasz Napierala ic = arg; 770*321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 771*321b17ecSEdward Tomasz Napierala 772*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 773*321b17ecSEdward Tomasz Napierala ic->ic_receive_running = true; 774*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 775*321b17ecSEdward Tomasz Napierala 776*321b17ecSEdward Tomasz Napierala for (;;) { 777*321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 778*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("terminating"); 779*321b17ecSEdward Tomasz Napierala break; 780*321b17ecSEdward Tomasz Napierala } 781*321b17ecSEdward Tomasz Napierala 782*321b17ecSEdward Tomasz Napierala /* 783*321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 784*321b17ecSEdward Tomasz Napierala * soreadable() in icl_soupcall_receive() 785*321b17ecSEdward Tomasz Napierala * to avoid unneccessary wakeups until there 786*321b17ecSEdward Tomasz Napierala * is enough data received to read the PDU. 787*321b17ecSEdward Tomasz Napierala */ 788*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_rcv); 789*321b17ecSEdward Tomasz Napierala available = sbavail(&so->so_rcv); 790*321b17ecSEdward Tomasz Napierala if (available < ic->ic_receive_len) { 791*321b17ecSEdward Tomasz Napierala so->so_rcv.sb_lowat = ic->ic_receive_len; 792*321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); 793*321b17ecSEdward Tomasz Napierala } else 794*321b17ecSEdward Tomasz Napierala so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; 795*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_rcv); 796*321b17ecSEdward Tomasz Napierala 797*321b17ecSEdward Tomasz Napierala icl_conn_receive_pdus(ic, available); 798*321b17ecSEdward Tomasz Napierala } 799*321b17ecSEdward Tomasz Napierala 800*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 801*321b17ecSEdward Tomasz Napierala ic->ic_receive_running = false; 802*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 803*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 804*321b17ecSEdward Tomasz Napierala kthread_exit(); 805*321b17ecSEdward Tomasz Napierala } 806*321b17ecSEdward Tomasz Napierala 807*321b17ecSEdward Tomasz Napierala static int 808*321b17ecSEdward Tomasz Napierala icl_soupcall_receive(struct socket *so, void *arg, int waitflag) 809*321b17ecSEdward Tomasz Napierala { 810*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 811*321b17ecSEdward Tomasz Napierala 812*321b17ecSEdward Tomasz Napierala if (!soreadable(so)) 813*321b17ecSEdward Tomasz Napierala return (SU_OK); 814*321b17ecSEdward Tomasz Napierala 815*321b17ecSEdward Tomasz Napierala ic = arg; 816*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_receive_cv); 817*321b17ecSEdward Tomasz Napierala return (SU_OK); 818*321b17ecSEdward Tomasz Napierala } 819*321b17ecSEdward Tomasz Napierala 820*321b17ecSEdward Tomasz Napierala static int 821*321b17ecSEdward Tomasz Napierala icl_pdu_finalize(struct icl_pdu *request) 822*321b17ecSEdward Tomasz Napierala { 823*321b17ecSEdward Tomasz Napierala size_t padding, pdu_len; 824*321b17ecSEdward Tomasz Napierala uint32_t digest, zero = 0; 825*321b17ecSEdward Tomasz Napierala int ok; 826*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 827*321b17ecSEdward Tomasz Napierala 828*321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 829*321b17ecSEdward Tomasz Napierala 830*321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(request, request->ip_data_len); 831*321b17ecSEdward Tomasz Napierala 832*321b17ecSEdward Tomasz Napierala pdu_len = icl_pdu_size(request); 833*321b17ecSEdward Tomasz Napierala 834*321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c) { 835*321b17ecSEdward Tomasz Napierala digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 836*321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_bhs_mbuf, sizeof(digest), 837*321b17ecSEdward Tomasz Napierala (void *)&digest); 838*321b17ecSEdward Tomasz Napierala if (ok != 1) { 839*321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append header digest"); 840*321b17ecSEdward Tomasz Napierala return (1); 841*321b17ecSEdward Tomasz Napierala } 842*321b17ecSEdward Tomasz Napierala } 843*321b17ecSEdward Tomasz Napierala 844*321b17ecSEdward Tomasz Napierala if (request->ip_data_len != 0) { 845*321b17ecSEdward Tomasz Napierala padding = icl_pdu_padding(request); 846*321b17ecSEdward Tomasz Napierala if (padding > 0) { 847*321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, padding, 848*321b17ecSEdward Tomasz Napierala (void *)&zero); 849*321b17ecSEdward Tomasz Napierala if (ok != 1) { 850*321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append padding"); 851*321b17ecSEdward Tomasz Napierala return (1); 852*321b17ecSEdward Tomasz Napierala } 853*321b17ecSEdward Tomasz Napierala } 854*321b17ecSEdward Tomasz Napierala 855*321b17ecSEdward Tomasz Napierala if (ic->ic_data_crc32c) { 856*321b17ecSEdward Tomasz Napierala digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 857*321b17ecSEdward Tomasz Napierala 858*321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, sizeof(digest), 859*321b17ecSEdward Tomasz Napierala (void *)&digest); 860*321b17ecSEdward Tomasz Napierala if (ok != 1) { 861*321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append data digest"); 862*321b17ecSEdward Tomasz Napierala return (1); 863*321b17ecSEdward Tomasz Napierala } 864*321b17ecSEdward Tomasz Napierala } 865*321b17ecSEdward Tomasz Napierala 866*321b17ecSEdward Tomasz Napierala m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); 867*321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = NULL; 868*321b17ecSEdward Tomasz Napierala } 869*321b17ecSEdward Tomasz Napierala 870*321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; 871*321b17ecSEdward Tomasz Napierala 872*321b17ecSEdward Tomasz Napierala return (0); 873*321b17ecSEdward Tomasz Napierala } 874*321b17ecSEdward Tomasz Napierala 875*321b17ecSEdward Tomasz Napierala static void 876*321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) 877*321b17ecSEdward Tomasz Napierala { 878*321b17ecSEdward Tomasz Napierala struct icl_pdu *request, *request2; 879*321b17ecSEdward Tomasz Napierala struct socket *so; 880*321b17ecSEdward Tomasz Napierala size_t available, size, size2; 881*321b17ecSEdward Tomasz Napierala int coalesced, error; 882*321b17ecSEdward Tomasz Napierala 883*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 884*321b17ecSEdward Tomasz Napierala 885*321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 886*321b17ecSEdward Tomasz Napierala 887*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 888*321b17ecSEdward Tomasz Napierala /* 889*321b17ecSEdward Tomasz Napierala * Check how much space do we have for transmit. We can't just 890*321b17ecSEdward Tomasz Napierala * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, 891*321b17ecSEdward Tomasz Napierala * as it always frees the mbuf chain passed to it, even in case 892*321b17ecSEdward Tomasz Napierala * of error. 893*321b17ecSEdward Tomasz Napierala */ 894*321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 895*321b17ecSEdward Tomasz Napierala 896*321b17ecSEdward Tomasz Napierala /* 897*321b17ecSEdward Tomasz Napierala * Notify the socket upcall that we don't need wakeups 898*321b17ecSEdward Tomasz Napierala * for the time being. 899*321b17ecSEdward Tomasz Napierala */ 900*321b17ecSEdward Tomasz Napierala so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; 901*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 902*321b17ecSEdward Tomasz Napierala 903*321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(queue)) { 904*321b17ecSEdward Tomasz Napierala request = STAILQ_FIRST(queue); 905*321b17ecSEdward Tomasz Napierala size = icl_pdu_size(request); 906*321b17ecSEdward Tomasz Napierala if (available < size) { 907*321b17ecSEdward Tomasz Napierala 908*321b17ecSEdward Tomasz Napierala /* 909*321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 910*321b17ecSEdward Tomasz Napierala * sowriteable() in icl_soupcall_send() 911*321b17ecSEdward Tomasz Napierala * to avoid unneccessary wakeups until there 912*321b17ecSEdward Tomasz Napierala * is enough space for the PDU to fit. 913*321b17ecSEdward Tomasz Napierala */ 914*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 915*321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 916*321b17ecSEdward Tomasz Napierala if (available < size) { 917*321b17ecSEdward Tomasz Napierala #if 1 918*321b17ecSEdward Tomasz Napierala ICL_DEBUG("no space to send; " 919*321b17ecSEdward Tomasz Napierala "have %zd, need %zd", 920*321b17ecSEdward Tomasz Napierala available, size); 921*321b17ecSEdward Tomasz Napierala #endif 922*321b17ecSEdward Tomasz Napierala so->so_snd.sb_lowat = size; 923*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 924*321b17ecSEdward Tomasz Napierala return; 925*321b17ecSEdward Tomasz Napierala } 926*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 927*321b17ecSEdward Tomasz Napierala } 928*321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 929*321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request); 930*321b17ecSEdward Tomasz Napierala if (error != 0) { 931*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 932*321b17ecSEdward Tomasz Napierala "dropping connection"); 933*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 934*321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 935*321b17ecSEdward Tomasz Napierala return; 936*321b17ecSEdward Tomasz Napierala } 937*321b17ecSEdward Tomasz Napierala if (coalesce) { 938*321b17ecSEdward Tomasz Napierala coalesced = 1; 939*321b17ecSEdward Tomasz Napierala for (;;) { 940*321b17ecSEdward Tomasz Napierala request2 = STAILQ_FIRST(queue); 941*321b17ecSEdward Tomasz Napierala if (request2 == NULL) 942*321b17ecSEdward Tomasz Napierala break; 943*321b17ecSEdward Tomasz Napierala size2 = icl_pdu_size(request2); 944*321b17ecSEdward Tomasz Napierala if (available < size + size2) 945*321b17ecSEdward Tomasz Napierala break; 946*321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 947*321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request2); 948*321b17ecSEdward Tomasz Napierala if (error != 0) { 949*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 950*321b17ecSEdward Tomasz Napierala "dropping connection"); 951*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 952*321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 953*321b17ecSEdward Tomasz Napierala icl_pdu_free(request2); 954*321b17ecSEdward Tomasz Napierala return; 955*321b17ecSEdward Tomasz Napierala } 956*321b17ecSEdward Tomasz Napierala m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf); 957*321b17ecSEdward Tomasz Napierala request2->ip_bhs_mbuf = NULL; 958*321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len += size2; 959*321b17ecSEdward Tomasz Napierala size += size2; 960*321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_AFTER(queue, request, ip_next); 961*321b17ecSEdward Tomasz Napierala icl_pdu_free(request2); 962*321b17ecSEdward Tomasz Napierala coalesced++; 963*321b17ecSEdward Tomasz Napierala } 964*321b17ecSEdward Tomasz Napierala #if 0 965*321b17ecSEdward Tomasz Napierala if (coalesced > 1) { 966*321b17ecSEdward Tomasz Napierala ICL_DEBUG("coalesced %d PDUs into %zd bytes", 967*321b17ecSEdward Tomasz Napierala coalesced, size); 968*321b17ecSEdward Tomasz Napierala } 969*321b17ecSEdward Tomasz Napierala #endif 970*321b17ecSEdward Tomasz Napierala } 971*321b17ecSEdward Tomasz Napierala available -= size; 972*321b17ecSEdward Tomasz Napierala error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, 973*321b17ecSEdward Tomasz Napierala NULL, MSG_DONTWAIT, curthread); 974*321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ 975*321b17ecSEdward Tomasz Napierala if (error != 0) { 976*321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to send PDU, error %d; " 977*321b17ecSEdward Tomasz Napierala "dropping connection", error); 978*321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 979*321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 980*321b17ecSEdward Tomasz Napierala return; 981*321b17ecSEdward Tomasz Napierala } 982*321b17ecSEdward Tomasz Napierala icl_pdu_free(request); 983*321b17ecSEdward Tomasz Napierala } 984*321b17ecSEdward Tomasz Napierala } 985*321b17ecSEdward Tomasz Napierala 986*321b17ecSEdward Tomasz Napierala static void 987*321b17ecSEdward Tomasz Napierala icl_send_thread(void *arg) 988*321b17ecSEdward Tomasz Napierala { 989*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 990*321b17ecSEdward Tomasz Napierala struct icl_pdu_stailq queue; 991*321b17ecSEdward Tomasz Napierala 992*321b17ecSEdward Tomasz Napierala ic = arg; 993*321b17ecSEdward Tomasz Napierala 994*321b17ecSEdward Tomasz Napierala STAILQ_INIT(&queue); 995*321b17ecSEdward Tomasz Napierala 996*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 997*321b17ecSEdward Tomasz Napierala ic->ic_send_running = true; 998*321b17ecSEdward Tomasz Napierala 999*321b17ecSEdward Tomasz Napierala for (;;) { 1000*321b17ecSEdward Tomasz Napierala for (;;) { 1001*321b17ecSEdward Tomasz Napierala /* 1002*321b17ecSEdward Tomasz Napierala * If the local queue is empty, populate it from 1003*321b17ecSEdward Tomasz Napierala * the main one. This way the icl_conn_send_pdus() 1004*321b17ecSEdward Tomasz Napierala * can go through all the queued PDUs without holding 1005*321b17ecSEdward Tomasz Napierala * any locks. 1006*321b17ecSEdward Tomasz Napierala */ 1007*321b17ecSEdward Tomasz Napierala if (STAILQ_EMPTY(&queue)) 1008*321b17ecSEdward Tomasz Napierala STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu); 1009*321b17ecSEdward Tomasz Napierala 1010*321b17ecSEdward Tomasz Napierala ic->ic_check_send_space = false; 1011*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1012*321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(ic, &queue); 1013*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1014*321b17ecSEdward Tomasz Napierala 1015*321b17ecSEdward Tomasz Napierala /* 1016*321b17ecSEdward Tomasz Napierala * The icl_soupcall_send() was called since the last 1017*321b17ecSEdward Tomasz Napierala * call to sbspace(); go around; 1018*321b17ecSEdward Tomasz Napierala */ 1019*321b17ecSEdward Tomasz Napierala if (ic->ic_check_send_space) 1020*321b17ecSEdward Tomasz Napierala continue; 1021*321b17ecSEdward Tomasz Napierala 1022*321b17ecSEdward Tomasz Napierala /* 1023*321b17ecSEdward Tomasz Napierala * Local queue is empty, but we still have PDUs 1024*321b17ecSEdward Tomasz Napierala * in the main one; go around. 1025*321b17ecSEdward Tomasz Napierala */ 1026*321b17ecSEdward Tomasz Napierala if (STAILQ_EMPTY(&queue) && 1027*321b17ecSEdward Tomasz Napierala !STAILQ_EMPTY(&ic->ic_to_send)) 1028*321b17ecSEdward Tomasz Napierala continue; 1029*321b17ecSEdward Tomasz Napierala 1030*321b17ecSEdward Tomasz Napierala /* 1031*321b17ecSEdward Tomasz Napierala * There might be some stuff in the local queue, 1032*321b17ecSEdward Tomasz Napierala * which didn't get sent due to not having enough send 1033*321b17ecSEdward Tomasz Napierala * space. Wait for socket upcall. 1034*321b17ecSEdward Tomasz Napierala */ 1035*321b17ecSEdward Tomasz Napierala break; 1036*321b17ecSEdward Tomasz Napierala } 1037*321b17ecSEdward Tomasz Napierala 1038*321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 1039*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("terminating"); 1040*321b17ecSEdward Tomasz Napierala break; 1041*321b17ecSEdward Tomasz Napierala } 1042*321b17ecSEdward Tomasz Napierala 1043*321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_send_cv, ic->ic_lock); 1044*321b17ecSEdward Tomasz Napierala } 1045*321b17ecSEdward Tomasz Napierala 1046*321b17ecSEdward Tomasz Napierala /* 1047*321b17ecSEdward Tomasz Napierala * We're exiting; move PDUs back to the main queue, so they can 1048*321b17ecSEdward Tomasz Napierala * get freed properly. At this point ordering doesn't matter. 1049*321b17ecSEdward Tomasz Napierala */ 1050*321b17ecSEdward Tomasz Napierala STAILQ_CONCAT(&ic->ic_to_send, &queue); 1051*321b17ecSEdward Tomasz Napierala 1052*321b17ecSEdward Tomasz Napierala ic->ic_send_running = false; 1053*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1054*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1055*321b17ecSEdward Tomasz Napierala kthread_exit(); 1056*321b17ecSEdward Tomasz Napierala } 1057*321b17ecSEdward Tomasz Napierala 1058*321b17ecSEdward Tomasz Napierala static int 1059*321b17ecSEdward Tomasz Napierala icl_soupcall_send(struct socket *so, void *arg, int waitflag) 1060*321b17ecSEdward Tomasz Napierala { 1061*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1062*321b17ecSEdward Tomasz Napierala 1063*321b17ecSEdward Tomasz Napierala if (!sowriteable(so)) 1064*321b17ecSEdward Tomasz Napierala return (SU_OK); 1065*321b17ecSEdward Tomasz Napierala 1066*321b17ecSEdward Tomasz Napierala ic = arg; 1067*321b17ecSEdward Tomasz Napierala 1068*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1069*321b17ecSEdward Tomasz Napierala ic->ic_check_send_space = true; 1070*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1071*321b17ecSEdward Tomasz Napierala 1072*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1073*321b17ecSEdward Tomasz Napierala 1074*321b17ecSEdward Tomasz Napierala return (SU_OK); 1075*321b17ecSEdward Tomasz Napierala } 1076*321b17ecSEdward Tomasz Napierala 1077*321b17ecSEdward Tomasz Napierala static int 1078*321b17ecSEdward Tomasz Napierala icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len, 1079*321b17ecSEdward Tomasz Napierala int flags) 1080*321b17ecSEdward Tomasz Napierala { 1081*321b17ecSEdward Tomasz Napierala struct mbuf *mb, *newmb; 1082*321b17ecSEdward Tomasz Napierala size_t copylen, off = 0; 1083*321b17ecSEdward Tomasz Napierala 1084*321b17ecSEdward Tomasz Napierala KASSERT(len > 0, ("len == 0")); 1085*321b17ecSEdward Tomasz Napierala 1086*321b17ecSEdward Tomasz Napierala newmb = m_getm2(NULL, len, flags, MT_DATA, M_PKTHDR); 1087*321b17ecSEdward Tomasz Napierala if (newmb == NULL) { 1088*321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate mbuf for %zd bytes", len); 1089*321b17ecSEdward Tomasz Napierala return (ENOMEM); 1090*321b17ecSEdward Tomasz Napierala } 1091*321b17ecSEdward Tomasz Napierala 1092*321b17ecSEdward Tomasz Napierala for (mb = newmb; mb != NULL; mb = mb->m_next) { 1093*321b17ecSEdward Tomasz Napierala copylen = min(M_TRAILINGSPACE(mb), len - off); 1094*321b17ecSEdward Tomasz Napierala memcpy(mtod(mb, char *), (const char *)addr + off, copylen); 1095*321b17ecSEdward Tomasz Napierala mb->m_len = copylen; 1096*321b17ecSEdward Tomasz Napierala off += copylen; 1097*321b17ecSEdward Tomasz Napierala } 1098*321b17ecSEdward Tomasz Napierala KASSERT(off == len, ("%s: off != len", __func__)); 1099*321b17ecSEdward Tomasz Napierala 1100*321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) { 1101*321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = newmb; 1102*321b17ecSEdward Tomasz Napierala request->ip_data_len = len; 1103*321b17ecSEdward Tomasz Napierala } else { 1104*321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, newmb); 1105*321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 1106*321b17ecSEdward Tomasz Napierala } 1107*321b17ecSEdward Tomasz Napierala 1108*321b17ecSEdward Tomasz Napierala return (0); 1109*321b17ecSEdward Tomasz Napierala } 1110*321b17ecSEdward Tomasz Napierala 1111*321b17ecSEdward Tomasz Napierala int 1112*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 1113*321b17ecSEdward Tomasz Napierala const void *addr, size_t len, int flags) 1114*321b17ecSEdward Tomasz Napierala { 1115*321b17ecSEdward Tomasz Napierala 1116*321b17ecSEdward Tomasz Napierala return (icl_pdu_append_data(request, addr, len, flags)); 1117*321b17ecSEdward Tomasz Napierala } 1118*321b17ecSEdward Tomasz Napierala 1119*321b17ecSEdward Tomasz Napierala static void 1120*321b17ecSEdward Tomasz Napierala icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len) 1121*321b17ecSEdward Tomasz Napierala { 1122*321b17ecSEdward Tomasz Napierala 1123*321b17ecSEdward Tomasz Napierala m_copydata(ip->ip_data_mbuf, off, len, addr); 1124*321b17ecSEdward Tomasz Napierala } 1125*321b17ecSEdward Tomasz Napierala 1126*321b17ecSEdward Tomasz Napierala void 1127*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1128*321b17ecSEdward Tomasz Napierala size_t off, void *addr, size_t len) 1129*321b17ecSEdward Tomasz Napierala { 1130*321b17ecSEdward Tomasz Napierala 1131*321b17ecSEdward Tomasz Napierala return (icl_pdu_get_data(ip, off, addr, len)); 1132*321b17ecSEdward Tomasz Napierala } 1133*321b17ecSEdward Tomasz Napierala 1134*321b17ecSEdward Tomasz Napierala static void 1135*321b17ecSEdward Tomasz Napierala icl_pdu_queue(struct icl_pdu *ip) 1136*321b17ecSEdward Tomasz Napierala { 1137*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1138*321b17ecSEdward Tomasz Napierala 1139*321b17ecSEdward Tomasz Napierala ic = ip->ip_conn; 1140*321b17ecSEdward Tomasz Napierala 1141*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT(ic); 1142*321b17ecSEdward Tomasz Napierala 1143*321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1144*321b17ecSEdward Tomasz Napierala ICL_DEBUG("icl_pdu_queue on closed connection"); 1145*321b17ecSEdward Tomasz Napierala icl_pdu_free(ip); 1146*321b17ecSEdward Tomasz Napierala return; 1147*321b17ecSEdward Tomasz Napierala } 1148*321b17ecSEdward Tomasz Napierala 1149*321b17ecSEdward Tomasz Napierala if (!STAILQ_EMPTY(&ic->ic_to_send)) { 1150*321b17ecSEdward Tomasz Napierala STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1151*321b17ecSEdward Tomasz Napierala /* 1152*321b17ecSEdward Tomasz Napierala * If the queue is not empty, someone else had already 1153*321b17ecSEdward Tomasz Napierala * signaled the send thread; no need to do that again, 1154*321b17ecSEdward Tomasz Napierala * just return. 1155*321b17ecSEdward Tomasz Napierala */ 1156*321b17ecSEdward Tomasz Napierala return; 1157*321b17ecSEdward Tomasz Napierala } 1158*321b17ecSEdward Tomasz Napierala 1159*321b17ecSEdward Tomasz Napierala STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1160*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1161*321b17ecSEdward Tomasz Napierala } 1162*321b17ecSEdward Tomasz Napierala 1163*321b17ecSEdward Tomasz Napierala void 1164*321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 1165*321b17ecSEdward Tomasz Napierala { 1166*321b17ecSEdward Tomasz Napierala 1167*321b17ecSEdward Tomasz Napierala icl_pdu_queue(ip); 1168*321b17ecSEdward Tomasz Napierala } 1169*321b17ecSEdward Tomasz Napierala 1170*321b17ecSEdward Tomasz Napierala static struct icl_conn * 1171*321b17ecSEdward Tomasz Napierala icl_soft_new_conn(const char *name, struct mtx *lock) 1172*321b17ecSEdward Tomasz Napierala { 1173*321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1174*321b17ecSEdward Tomasz Napierala 1175*321b17ecSEdward Tomasz Napierala refcount_acquire(&icl_ncons); 1176*321b17ecSEdward Tomasz Napierala 1177*321b17ecSEdward Tomasz Napierala ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO); 1178*321b17ecSEdward Tomasz Napierala 1179*321b17ecSEdward Tomasz Napierala STAILQ_INIT(&ic->ic_to_send); 1180*321b17ecSEdward Tomasz Napierala ic->ic_lock = lock; 1181*321b17ecSEdward Tomasz Napierala cv_init(&ic->ic_send_cv, "icl_tx"); 1182*321b17ecSEdward Tomasz Napierala cv_init(&ic->ic_receive_cv, "icl_rx"); 1183*321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 1184*321b17ecSEdward Tomasz Napierala refcount_init(&ic->ic_outstanding_pdus, 0); 1185*321b17ecSEdward Tomasz Napierala #endif 1186*321b17ecSEdward Tomasz Napierala ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH; 1187*321b17ecSEdward Tomasz Napierala ic->ic_name = name; 1188*321b17ecSEdward Tomasz Napierala 1189*321b17ecSEdward Tomasz Napierala return (ic); 1190*321b17ecSEdward Tomasz Napierala } 1191*321b17ecSEdward Tomasz Napierala 1192*321b17ecSEdward Tomasz Napierala void 1193*321b17ecSEdward Tomasz Napierala icl_soft_conn_free(struct icl_conn *ic) 1194*321b17ecSEdward Tomasz Napierala { 1195*321b17ecSEdward Tomasz Napierala 1196*321b17ecSEdward Tomasz Napierala cv_destroy(&ic->ic_send_cv); 1197*321b17ecSEdward Tomasz Napierala cv_destroy(&ic->ic_receive_cv); 1198*321b17ecSEdward Tomasz Napierala kobj_delete((struct kobj *)ic, M_ICL_SOFT); 1199*321b17ecSEdward Tomasz Napierala refcount_release(&icl_ncons); 1200*321b17ecSEdward Tomasz Napierala } 1201*321b17ecSEdward Tomasz Napierala 1202*321b17ecSEdward Tomasz Napierala static int 1203*321b17ecSEdward Tomasz Napierala icl_conn_start(struct icl_conn *ic) 1204*321b17ecSEdward Tomasz Napierala { 1205*321b17ecSEdward Tomasz Napierala size_t minspace; 1206*321b17ecSEdward Tomasz Napierala struct sockopt opt; 1207*321b17ecSEdward Tomasz Napierala int error, one = 1; 1208*321b17ecSEdward Tomasz Napierala 1209*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1210*321b17ecSEdward Tomasz Napierala 1211*321b17ecSEdward Tomasz Napierala /* 1212*321b17ecSEdward Tomasz Napierala * XXX: Ugly hack. 1213*321b17ecSEdward Tomasz Napierala */ 1214*321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1215*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1216*321b17ecSEdward Tomasz Napierala return (EINVAL); 1217*321b17ecSEdward Tomasz Napierala } 1218*321b17ecSEdward Tomasz Napierala 1219*321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_BHS; 1220*321b17ecSEdward Tomasz Napierala ic->ic_receive_len = sizeof(struct iscsi_bhs); 1221*321b17ecSEdward Tomasz Napierala ic->ic_disconnecting = false; 1222*321b17ecSEdward Tomasz Napierala 1223*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1224*321b17ecSEdward Tomasz Napierala 1225*321b17ecSEdward Tomasz Napierala /* 1226*321b17ecSEdward Tomasz Napierala * For sendspace, this is required because the current code cannot 1227*321b17ecSEdward Tomasz Napierala * send a PDU in pieces; thus, the minimum buffer size is equal 1228*321b17ecSEdward Tomasz Napierala * to the maximum PDU size. "+4" is to account for possible padding. 1229*321b17ecSEdward Tomasz Napierala * 1230*321b17ecSEdward Tomasz Napierala * What we should actually do here is to use autoscaling, but set 1231*321b17ecSEdward Tomasz Napierala * some minimal buffer size to "minspace". I don't know a way to do 1232*321b17ecSEdward Tomasz Napierala * that, though. 1233*321b17ecSEdward Tomasz Napierala */ 1234*321b17ecSEdward Tomasz Napierala minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + 1235*321b17ecSEdward Tomasz Napierala ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1236*321b17ecSEdward Tomasz Napierala if (sendspace < minspace) { 1237*321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.sendspace too low; must be at least %zd", 1238*321b17ecSEdward Tomasz Napierala minspace); 1239*321b17ecSEdward Tomasz Napierala sendspace = minspace; 1240*321b17ecSEdward Tomasz Napierala } 1241*321b17ecSEdward Tomasz Napierala if (recvspace < minspace) { 1242*321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.recvspace too low; must be at least %zd", 1243*321b17ecSEdward Tomasz Napierala minspace); 1244*321b17ecSEdward Tomasz Napierala recvspace = minspace; 1245*321b17ecSEdward Tomasz Napierala } 1246*321b17ecSEdward Tomasz Napierala 1247*321b17ecSEdward Tomasz Napierala error = soreserve(ic->ic_socket, sendspace, recvspace); 1248*321b17ecSEdward Tomasz Napierala if (error != 0) { 1249*321b17ecSEdward Tomasz Napierala ICL_WARN("soreserve failed with error %d", error); 1250*321b17ecSEdward Tomasz Napierala icl_conn_close(ic); 1251*321b17ecSEdward Tomasz Napierala return (error); 1252*321b17ecSEdward Tomasz Napierala } 1253*321b17ecSEdward Tomasz Napierala ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; 1254*321b17ecSEdward Tomasz Napierala ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; 1255*321b17ecSEdward Tomasz Napierala 1256*321b17ecSEdward Tomasz Napierala /* 1257*321b17ecSEdward Tomasz Napierala * Disable Nagle. 1258*321b17ecSEdward Tomasz Napierala */ 1259*321b17ecSEdward Tomasz Napierala bzero(&opt, sizeof(opt)); 1260*321b17ecSEdward Tomasz Napierala opt.sopt_dir = SOPT_SET; 1261*321b17ecSEdward Tomasz Napierala opt.sopt_level = IPPROTO_TCP; 1262*321b17ecSEdward Tomasz Napierala opt.sopt_name = TCP_NODELAY; 1263*321b17ecSEdward Tomasz Napierala opt.sopt_val = &one; 1264*321b17ecSEdward Tomasz Napierala opt.sopt_valsize = sizeof(one); 1265*321b17ecSEdward Tomasz Napierala error = sosetopt(ic->ic_socket, &opt); 1266*321b17ecSEdward Tomasz Napierala if (error != 0) { 1267*321b17ecSEdward Tomasz Napierala ICL_WARN("disabling TCP_NODELAY failed with error %d", error); 1268*321b17ecSEdward Tomasz Napierala icl_conn_close(ic); 1269*321b17ecSEdward Tomasz Napierala return (error); 1270*321b17ecSEdward Tomasz Napierala } 1271*321b17ecSEdward Tomasz Napierala 1272*321b17ecSEdward Tomasz Napierala /* 1273*321b17ecSEdward Tomasz Napierala * Start threads. 1274*321b17ecSEdward Tomasz Napierala */ 1275*321b17ecSEdward Tomasz Napierala error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", 1276*321b17ecSEdward Tomasz Napierala ic->ic_name); 1277*321b17ecSEdward Tomasz Napierala if (error != 0) { 1278*321b17ecSEdward Tomasz Napierala ICL_WARN("kthread_add(9) failed with error %d", error); 1279*321b17ecSEdward Tomasz Napierala icl_conn_close(ic); 1280*321b17ecSEdward Tomasz Napierala return (error); 1281*321b17ecSEdward Tomasz Napierala } 1282*321b17ecSEdward Tomasz Napierala 1283*321b17ecSEdward Tomasz Napierala error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", 1284*321b17ecSEdward Tomasz Napierala ic->ic_name); 1285*321b17ecSEdward Tomasz Napierala if (error != 0) { 1286*321b17ecSEdward Tomasz Napierala ICL_WARN("kthread_add(9) failed with error %d", error); 1287*321b17ecSEdward Tomasz Napierala icl_conn_close(ic); 1288*321b17ecSEdward Tomasz Napierala return (error); 1289*321b17ecSEdward Tomasz Napierala } 1290*321b17ecSEdward Tomasz Napierala 1291*321b17ecSEdward Tomasz Napierala /* 1292*321b17ecSEdward Tomasz Napierala * Register socket upcall, to get notified about incoming PDUs 1293*321b17ecSEdward Tomasz Napierala * and free space to send outgoing ones. 1294*321b17ecSEdward Tomasz Napierala */ 1295*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1296*321b17ecSEdward Tomasz Napierala soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); 1297*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1298*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1299*321b17ecSEdward Tomasz Napierala soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); 1300*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1301*321b17ecSEdward Tomasz Napierala 1302*321b17ecSEdward Tomasz Napierala return (0); 1303*321b17ecSEdward Tomasz Napierala } 1304*321b17ecSEdward Tomasz Napierala 1305*321b17ecSEdward Tomasz Napierala int 1306*321b17ecSEdward Tomasz Napierala icl_soft_conn_handoff(struct icl_conn *ic, int fd) 1307*321b17ecSEdward Tomasz Napierala { 1308*321b17ecSEdward Tomasz Napierala struct file *fp; 1309*321b17ecSEdward Tomasz Napierala struct socket *so; 1310*321b17ecSEdward Tomasz Napierala cap_rights_t rights; 1311*321b17ecSEdward Tomasz Napierala int error; 1312*321b17ecSEdward Tomasz Napierala 1313*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1314*321b17ecSEdward Tomasz Napierala 1315*321b17ecSEdward Tomasz Napierala /* 1316*321b17ecSEdward Tomasz Napierala * Steal the socket from userland. 1317*321b17ecSEdward Tomasz Napierala */ 1318*321b17ecSEdward Tomasz Napierala error = fget(curthread, fd, 1319*321b17ecSEdward Tomasz Napierala cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp); 1320*321b17ecSEdward Tomasz Napierala if (error != 0) 1321*321b17ecSEdward Tomasz Napierala return (error); 1322*321b17ecSEdward Tomasz Napierala if (fp->f_type != DTYPE_SOCKET) { 1323*321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1324*321b17ecSEdward Tomasz Napierala return (EINVAL); 1325*321b17ecSEdward Tomasz Napierala } 1326*321b17ecSEdward Tomasz Napierala so = fp->f_data; 1327*321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) { 1328*321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1329*321b17ecSEdward Tomasz Napierala return (EINVAL); 1330*321b17ecSEdward Tomasz Napierala } 1331*321b17ecSEdward Tomasz Napierala 1332*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1333*321b17ecSEdward Tomasz Napierala 1334*321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1335*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1336*321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1337*321b17ecSEdward Tomasz Napierala return (EBUSY); 1338*321b17ecSEdward Tomasz Napierala } 1339*321b17ecSEdward Tomasz Napierala 1340*321b17ecSEdward Tomasz Napierala ic->ic_socket = fp->f_data; 1341*321b17ecSEdward Tomasz Napierala fp->f_ops = &badfileops; 1342*321b17ecSEdward Tomasz Napierala fp->f_data = NULL; 1343*321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1344*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1345*321b17ecSEdward Tomasz Napierala 1346*321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1347*321b17ecSEdward Tomasz Napierala 1348*321b17ecSEdward Tomasz Napierala return (error); 1349*321b17ecSEdward Tomasz Napierala } 1350*321b17ecSEdward Tomasz Napierala 1351*321b17ecSEdward Tomasz Napierala void 1352*321b17ecSEdward Tomasz Napierala icl_conn_close(struct icl_conn *ic) 1353*321b17ecSEdward Tomasz Napierala { 1354*321b17ecSEdward Tomasz Napierala struct icl_pdu *pdu; 1355*321b17ecSEdward Tomasz Napierala 1356*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1357*321b17ecSEdward Tomasz Napierala 1358*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1359*321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1360*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1361*321b17ecSEdward Tomasz Napierala return; 1362*321b17ecSEdward Tomasz Napierala } 1363*321b17ecSEdward Tomasz Napierala 1364*321b17ecSEdward Tomasz Napierala /* 1365*321b17ecSEdward Tomasz Napierala * Deregister socket upcalls. 1366*321b17ecSEdward Tomasz Napierala */ 1367*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1368*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1369*321b17ecSEdward Tomasz Napierala if (ic->ic_socket->so_snd.sb_upcall != NULL) 1370*321b17ecSEdward Tomasz Napierala soupcall_clear(ic->ic_socket, SO_SND); 1371*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1372*321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1373*321b17ecSEdward Tomasz Napierala if (ic->ic_socket->so_rcv.sb_upcall != NULL) 1374*321b17ecSEdward Tomasz Napierala soupcall_clear(ic->ic_socket, SO_RCV); 1375*321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1376*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1377*321b17ecSEdward Tomasz Napierala 1378*321b17ecSEdward Tomasz Napierala ic->ic_disconnecting = true; 1379*321b17ecSEdward Tomasz Napierala 1380*321b17ecSEdward Tomasz Napierala /* 1381*321b17ecSEdward Tomasz Napierala * Wake up the threads, so they can properly terminate. 1382*321b17ecSEdward Tomasz Napierala */ 1383*321b17ecSEdward Tomasz Napierala while (ic->ic_receive_running || ic->ic_send_running) { 1384*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("waiting for send/receive threads to terminate"); 1385*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_receive_cv); 1386*321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1387*321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_send_cv, ic->ic_lock); 1388*321b17ecSEdward Tomasz Napierala } 1389*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("send/receive threads terminated"); 1390*321b17ecSEdward Tomasz Napierala 1391*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1392*321b17ecSEdward Tomasz Napierala soclose(ic->ic_socket); 1393*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1394*321b17ecSEdward Tomasz Napierala ic->ic_socket = NULL; 1395*321b17ecSEdward Tomasz Napierala 1396*321b17ecSEdward Tomasz Napierala if (ic->ic_receive_pdu != NULL) { 1397*321b17ecSEdward Tomasz Napierala //ICL_DEBUG("freeing partially received PDU"); 1398*321b17ecSEdward Tomasz Napierala icl_pdu_free(ic->ic_receive_pdu); 1399*321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = NULL; 1400*321b17ecSEdward Tomasz Napierala } 1401*321b17ecSEdward Tomasz Napierala 1402*321b17ecSEdward Tomasz Napierala /* 1403*321b17ecSEdward Tomasz Napierala * Remove any outstanding PDUs from the send queue. 1404*321b17ecSEdward Tomasz Napierala */ 1405*321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(&ic->ic_to_send)) { 1406*321b17ecSEdward Tomasz Napierala pdu = STAILQ_FIRST(&ic->ic_to_send); 1407*321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); 1408*321b17ecSEdward Tomasz Napierala icl_pdu_free(pdu); 1409*321b17ecSEdward Tomasz Napierala } 1410*321b17ecSEdward Tomasz Napierala 1411*321b17ecSEdward Tomasz Napierala KASSERT(STAILQ_EMPTY(&ic->ic_to_send), 1412*321b17ecSEdward Tomasz Napierala ("destroying session with non-empty send queue")); 1413*321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 1414*321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_outstanding_pdus == 0, 1415*321b17ecSEdward Tomasz Napierala ("destroying session with %d outstanding PDUs", 1416*321b17ecSEdward Tomasz Napierala ic->ic_outstanding_pdus)); 1417*321b17ecSEdward Tomasz Napierala #endif 1418*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1419*321b17ecSEdward Tomasz Napierala } 1420*321b17ecSEdward Tomasz Napierala 1421*321b17ecSEdward Tomasz Napierala void 1422*321b17ecSEdward Tomasz Napierala icl_soft_conn_close(struct icl_conn *ic) 1423*321b17ecSEdward Tomasz Napierala { 1424*321b17ecSEdward Tomasz Napierala 1425*321b17ecSEdward Tomasz Napierala icl_conn_close(ic); 1426*321b17ecSEdward Tomasz Napierala } 1427*321b17ecSEdward Tomasz Napierala 1428*321b17ecSEdward Tomasz Napierala bool 1429*321b17ecSEdward Tomasz Napierala icl_soft_conn_connected(struct icl_conn *ic) 1430*321b17ecSEdward Tomasz Napierala { 1431*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1432*321b17ecSEdward Tomasz Napierala 1433*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1434*321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1435*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1436*321b17ecSEdward Tomasz Napierala return (false); 1437*321b17ecSEdward Tomasz Napierala } 1438*321b17ecSEdward Tomasz Napierala if (ic->ic_socket->so_error != 0) { 1439*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1440*321b17ecSEdward Tomasz Napierala return (false); 1441*321b17ecSEdward Tomasz Napierala } 1442*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1443*321b17ecSEdward Tomasz Napierala return (true); 1444*321b17ecSEdward Tomasz Napierala } 1445*321b17ecSEdward Tomasz Napierala 1446*321b17ecSEdward Tomasz Napierala static int 1447*321b17ecSEdward Tomasz Napierala icl_soft_limits(size_t *limitp) 1448*321b17ecSEdward Tomasz Napierala { 1449*321b17ecSEdward Tomasz Napierala 1450*321b17ecSEdward Tomasz Napierala *limitp = 128 * 1024; 1451*321b17ecSEdward Tomasz Napierala 1452*321b17ecSEdward Tomasz Napierala return (0); 1453*321b17ecSEdward Tomasz Napierala } 1454*321b17ecSEdward Tomasz Napierala 1455*321b17ecSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1456*321b17ecSEdward Tomasz Napierala int 1457*321b17ecSEdward Tomasz Napierala icl_conn_handoff_sock(struct icl_conn *ic, struct socket *so) 1458*321b17ecSEdward Tomasz Napierala { 1459*321b17ecSEdward Tomasz Napierala int error; 1460*321b17ecSEdward Tomasz Napierala 1461*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1462*321b17ecSEdward Tomasz Napierala 1463*321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) 1464*321b17ecSEdward Tomasz Napierala return (EINVAL); 1465*321b17ecSEdward Tomasz Napierala 1466*321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1467*321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1468*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1469*321b17ecSEdward Tomasz Napierala return (EBUSY); 1470*321b17ecSEdward Tomasz Napierala } 1471*321b17ecSEdward Tomasz Napierala ic->ic_socket = so; 1472*321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1473*321b17ecSEdward Tomasz Napierala 1474*321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1475*321b17ecSEdward Tomasz Napierala 1476*321b17ecSEdward Tomasz Napierala return (error); 1477*321b17ecSEdward Tomasz Napierala } 1478*321b17ecSEdward Tomasz Napierala #endif /* ICL_KERNEL_PROXY */ 1479*321b17ecSEdward Tomasz Napierala 1480*321b17ecSEdward Tomasz Napierala static int 1481*321b17ecSEdward Tomasz Napierala icl_soft_load(void) 1482*321b17ecSEdward Tomasz Napierala { 1483*321b17ecSEdward Tomasz Napierala int error; 1484*321b17ecSEdward Tomasz Napierala 1485*321b17ecSEdward Tomasz Napierala icl_pdu_zone = uma_zcreate("icl_pdu", 1486*321b17ecSEdward Tomasz Napierala sizeof(struct icl_pdu), NULL, NULL, NULL, NULL, 1487*321b17ecSEdward Tomasz Napierala UMA_ALIGN_PTR, 0); 1488*321b17ecSEdward Tomasz Napierala refcount_init(&icl_ncons, 0); 1489*321b17ecSEdward Tomasz Napierala 1490*321b17ecSEdward Tomasz Napierala /* 1491*321b17ecSEdward Tomasz Napierala * The reason we call this "none" is that to the user, 1492*321b17ecSEdward Tomasz Napierala * it's known as "offload driver"; "offload driver: soft" 1493*321b17ecSEdward Tomasz Napierala * doesn't make much sense. 1494*321b17ecSEdward Tomasz Napierala */ 1495*321b17ecSEdward Tomasz Napierala error = icl_register("none", 0, icl_soft_limits, icl_soft_new_conn); 1496*321b17ecSEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1497*321b17ecSEdward Tomasz Napierala 1498*321b17ecSEdward Tomasz Napierala return (error); 1499*321b17ecSEdward Tomasz Napierala } 1500*321b17ecSEdward Tomasz Napierala 1501*321b17ecSEdward Tomasz Napierala static int 1502*321b17ecSEdward Tomasz Napierala icl_soft_unload(void) 1503*321b17ecSEdward Tomasz Napierala { 1504*321b17ecSEdward Tomasz Napierala 1505*321b17ecSEdward Tomasz Napierala if (icl_ncons != 0) 1506*321b17ecSEdward Tomasz Napierala return (EBUSY); 1507*321b17ecSEdward Tomasz Napierala 1508*321b17ecSEdward Tomasz Napierala icl_unregister("none"); 1509*321b17ecSEdward Tomasz Napierala 1510*321b17ecSEdward Tomasz Napierala uma_zdestroy(icl_pdu_zone); 1511*321b17ecSEdward Tomasz Napierala 1512*321b17ecSEdward Tomasz Napierala return (0); 1513*321b17ecSEdward Tomasz Napierala } 1514*321b17ecSEdward Tomasz Napierala 1515*321b17ecSEdward Tomasz Napierala static int 1516*321b17ecSEdward Tomasz Napierala icl_soft_modevent(module_t mod, int what, void *arg) 1517*321b17ecSEdward Tomasz Napierala { 1518*321b17ecSEdward Tomasz Napierala 1519*321b17ecSEdward Tomasz Napierala switch (what) { 1520*321b17ecSEdward Tomasz Napierala case MOD_LOAD: 1521*321b17ecSEdward Tomasz Napierala return (icl_soft_load()); 1522*321b17ecSEdward Tomasz Napierala case MOD_UNLOAD: 1523*321b17ecSEdward Tomasz Napierala return (icl_soft_unload()); 1524*321b17ecSEdward Tomasz Napierala default: 1525*321b17ecSEdward Tomasz Napierala return (EINVAL); 1526*321b17ecSEdward Tomasz Napierala } 1527*321b17ecSEdward Tomasz Napierala } 1528*321b17ecSEdward Tomasz Napierala 1529*321b17ecSEdward Tomasz Napierala moduledata_t icl_soft_data = { 1530*321b17ecSEdward Tomasz Napierala "icl_soft", 1531*321b17ecSEdward Tomasz Napierala icl_soft_modevent, 1532*321b17ecSEdward Tomasz Napierala 0 1533*321b17ecSEdward Tomasz Napierala }; 1534*321b17ecSEdward Tomasz Napierala 1535*321b17ecSEdward Tomasz Napierala DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 1536*321b17ecSEdward Tomasz Napierala MODULE_DEPEND(icl_soft, icl, 1, 1, 1); 1537*321b17ecSEdward Tomasz Napierala MODULE_VERSION(icl, 1); 1538