1321b17ecSEdward Tomasz Napierala /*- 243ee6e9dSEdward Tomasz Napierala * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 343ee6e9dSEdward Tomasz Napierala * 4321b17ecSEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 5321b17ecSEdward Tomasz Napierala * 6321b17ecSEdward Tomasz Napierala * This software was developed by Edward Tomasz Napierala under sponsorship 7321b17ecSEdward Tomasz Napierala * from the FreeBSD Foundation. 8321b17ecSEdward Tomasz Napierala * 9321b17ecSEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 10321b17ecSEdward Tomasz Napierala * modification, are permitted provided that the following conditions 11321b17ecSEdward Tomasz Napierala * are met: 12321b17ecSEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 13321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 14321b17ecSEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 15321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 16321b17ecSEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 17321b17ecSEdward Tomasz Napierala * 18321b17ecSEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19321b17ecSEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20321b17ecSEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21321b17ecSEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22321b17ecSEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23321b17ecSEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24321b17ecSEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25321b17ecSEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26321b17ecSEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27321b17ecSEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28321b17ecSEdward Tomasz Napierala * SUCH DAMAGE. 29321b17ecSEdward Tomasz Napierala * 30321b17ecSEdward Tomasz Napierala */ 31321b17ecSEdward Tomasz Napierala 32321b17ecSEdward Tomasz Napierala /* 335aabcd7cSEdward Tomasz Napierala * Software implementation of iSCSI Common Layer kobj(9) interface. 34321b17ecSEdward Tomasz Napierala */ 35321b17ecSEdward Tomasz Napierala 36321b17ecSEdward Tomasz Napierala #include <sys/cdefs.h> 37321b17ecSEdward Tomasz Napierala __FBSDID("$FreeBSD$"); 38321b17ecSEdward Tomasz Napierala 39321b17ecSEdward Tomasz Napierala #include <sys/param.h> 40321b17ecSEdward Tomasz Napierala #include <sys/capsicum.h> 41321b17ecSEdward Tomasz Napierala #include <sys/condvar.h> 42321b17ecSEdward Tomasz Napierala #include <sys/conf.h> 43f89d2072SXin LI #include <sys/gsb_crc32.h> 44321b17ecSEdward Tomasz Napierala #include <sys/file.h> 45321b17ecSEdward Tomasz Napierala #include <sys/kernel.h> 46321b17ecSEdward Tomasz Napierala #include <sys/kthread.h> 47321b17ecSEdward Tomasz Napierala #include <sys/lock.h> 48321b17ecSEdward Tomasz Napierala #include <sys/mbuf.h> 49321b17ecSEdward Tomasz Napierala #include <sys/mutex.h> 50321b17ecSEdward Tomasz Napierala #include <sys/module.h> 51321b17ecSEdward Tomasz Napierala #include <sys/protosw.h> 52321b17ecSEdward Tomasz Napierala #include <sys/socket.h> 53321b17ecSEdward Tomasz Napierala #include <sys/socketvar.h> 54321b17ecSEdward Tomasz Napierala #include <sys/sysctl.h> 55321b17ecSEdward Tomasz Napierala #include <sys/systm.h> 56321b17ecSEdward Tomasz Napierala #include <sys/sx.h> 57321b17ecSEdward Tomasz Napierala #include <sys/uio.h> 58321b17ecSEdward Tomasz Napierala #include <vm/uma.h> 59321b17ecSEdward Tomasz Napierala #include <netinet/in.h> 60321b17ecSEdward Tomasz Napierala #include <netinet/tcp.h> 61321b17ecSEdward Tomasz Napierala 62321b17ecSEdward Tomasz Napierala #include <dev/iscsi/icl.h> 63321b17ecSEdward Tomasz Napierala #include <dev/iscsi/iscsi_proto.h> 64321b17ecSEdward Tomasz Napierala #include <icl_conn_if.h> 65321b17ecSEdward Tomasz Napierala 669a4510acSAlexander Motin struct icl_soft_pdu { 679a4510acSAlexander Motin struct icl_pdu ip; 689a4510acSAlexander Motin 699a4510acSAlexander Motin /* soft specific stuff goes here. */ 709a4510acSAlexander Motin u_int ref_cnt; 719a4510acSAlexander Motin icl_pdu_cb cb; 729a4510acSAlexander Motin int error; 739a4510acSAlexander Motin }; 749a4510acSAlexander Motin 75b75168edSAlexander Motin SYSCTL_NODE(_kern_icl, OID_AUTO, soft, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 76b75168edSAlexander Motin "Software iSCSI"); 77321b17ecSEdward Tomasz Napierala static int coalesce = 1; 78b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, coalesce, CTLFLAG_RWTUN, 79321b17ecSEdward Tomasz Napierala &coalesce, 0, "Try to coalesce PDUs before sending"); 80b75168edSAlexander Motin static int partial_receive_len = 256 * 1024; 81b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 82321b17ecSEdward Tomasz Napierala &partial_receive_len, 0, "Minimum read size for partially received " 83321b17ecSEdward Tomasz Napierala "data segment"); 84b75168edSAlexander Motin static int max_data_segment_length = 256 * 1024; 85b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, max_data_segment_length, CTLFLAG_RWTUN, 86b75168edSAlexander Motin &max_data_segment_length, 0, "Maximum data segment length"); 87b75168edSAlexander Motin static int first_burst_length = 1024 * 1024; 88b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, first_burst_length, CTLFLAG_RWTUN, 89b75168edSAlexander Motin &first_burst_length, 0, "First burst length"); 90b75168edSAlexander Motin static int max_burst_length = 1024 * 1024; 91b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, max_burst_length, CTLFLAG_RWTUN, 92b75168edSAlexander Motin &max_burst_length, 0, "Maximum burst length"); 93b75168edSAlexander Motin static int sendspace = 1536 * 1024; 94b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, sendspace, CTLFLAG_RWTUN, 95321b17ecSEdward Tomasz Napierala &sendspace, 0, "Default send socket buffer size"); 96b75168edSAlexander Motin static int recvspace = 1536 * 1024; 97b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, recvspace, CTLFLAG_RWTUN, 98321b17ecSEdward Tomasz Napierala &recvspace, 0, "Default receive socket buffer size"); 99321b17ecSEdward Tomasz Napierala 100321b17ecSEdward Tomasz Napierala static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); 1019a4510acSAlexander Motin static uma_zone_t icl_soft_pdu_zone; 102321b17ecSEdward Tomasz Napierala 103321b17ecSEdward Tomasz Napierala static volatile u_int icl_ncons; 104321b17ecSEdward Tomasz Napierala 105321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) 106321b17ecSEdward Tomasz Napierala #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock) 107321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED) 108321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED) 109321b17ecSEdward Tomasz Napierala 110321b17ecSEdward Tomasz Napierala STAILQ_HEAD(icl_pdu_stailq, icl_pdu); 111321b17ecSEdward Tomasz Napierala 112321b17ecSEdward Tomasz Napierala static icl_conn_new_pdu_t icl_soft_conn_new_pdu; 113321b17ecSEdward Tomasz Napierala static icl_conn_pdu_free_t icl_soft_conn_pdu_free; 114321b17ecSEdward Tomasz Napierala static icl_conn_pdu_data_segment_length_t 115321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length; 116321b17ecSEdward Tomasz Napierala static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; 117321b17ecSEdward Tomasz Napierala static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; 118321b17ecSEdward Tomasz Napierala static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; 1199a4510acSAlexander Motin static icl_conn_pdu_queue_cb_t icl_soft_conn_pdu_queue_cb; 120321b17ecSEdward Tomasz Napierala static icl_conn_handoff_t icl_soft_conn_handoff; 121321b17ecSEdward Tomasz Napierala static icl_conn_free_t icl_soft_conn_free; 122321b17ecSEdward Tomasz Napierala static icl_conn_close_t icl_soft_conn_close; 1237a03d007SEdward Tomasz Napierala static icl_conn_task_setup_t icl_soft_conn_task_setup; 1247a03d007SEdward Tomasz Napierala static icl_conn_task_done_t icl_soft_conn_task_done; 1257a03d007SEdward Tomasz Napierala static icl_conn_transfer_setup_t icl_soft_conn_transfer_setup; 1267a03d007SEdward Tomasz Napierala static icl_conn_transfer_done_t icl_soft_conn_transfer_done; 127f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 128f41492b0SEdward Tomasz Napierala static icl_conn_connect_t icl_soft_conn_connect; 129f41492b0SEdward Tomasz Napierala #endif 130321b17ecSEdward Tomasz Napierala 131321b17ecSEdward Tomasz Napierala static kobj_method_t icl_soft_methods[] = { 132321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), 133321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), 134321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_data_segment_length, 135321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length), 136321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), 137321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), 138321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), 1399a4510acSAlexander Motin KOBJMETHOD(icl_conn_pdu_queue_cb, icl_soft_conn_pdu_queue_cb), 140321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), 141321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_free, icl_soft_conn_free), 142321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_close, icl_soft_conn_close), 1437a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup), 1447a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done), 1457a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup), 1467a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done), 147f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 148f41492b0SEdward Tomasz Napierala KOBJMETHOD(icl_conn_connect, icl_soft_conn_connect), 149f41492b0SEdward Tomasz Napierala #endif 150321b17ecSEdward Tomasz Napierala { 0, 0 } 151321b17ecSEdward Tomasz Napierala }; 152321b17ecSEdward Tomasz Napierala 153321b17ecSEdward Tomasz Napierala DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn)); 154321b17ecSEdward Tomasz Napierala 155321b17ecSEdward Tomasz Napierala static void 156321b17ecSEdward Tomasz Napierala icl_conn_fail(struct icl_conn *ic) 157321b17ecSEdward Tomasz Napierala { 158321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) 159321b17ecSEdward Tomasz Napierala return; 160321b17ecSEdward Tomasz Napierala 161321b17ecSEdward Tomasz Napierala /* 162321b17ecSEdward Tomasz Napierala * XXX 163321b17ecSEdward Tomasz Napierala */ 164321b17ecSEdward Tomasz Napierala ic->ic_socket->so_error = EDOOFUS; 165321b17ecSEdward Tomasz Napierala (ic->ic_error)(ic); 166321b17ecSEdward Tomasz Napierala } 167321b17ecSEdward Tomasz Napierala 16882f7fa7aSAlexander Motin static void 16982f7fa7aSAlexander Motin icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 17082f7fa7aSAlexander Motin { 1719a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 17282f7fa7aSAlexander Motin 1739a4510acSAlexander Motin KASSERT(isp->ref_cnt == 0, ("freeing active PDU")); 17482f7fa7aSAlexander Motin m_freem(ip->ip_bhs_mbuf); 17582f7fa7aSAlexander Motin m_freem(ip->ip_ahs_mbuf); 17682f7fa7aSAlexander Motin m_freem(ip->ip_data_mbuf); 1779a4510acSAlexander Motin uma_zfree(icl_soft_pdu_zone, isp); 17882f7fa7aSAlexander Motin #ifdef DIAGNOSTIC 17982f7fa7aSAlexander Motin refcount_release(&ic->ic_outstanding_pdus); 18082f7fa7aSAlexander Motin #endif 18182f7fa7aSAlexander Motin } 18282f7fa7aSAlexander Motin 1839a4510acSAlexander Motin static void 1849a4510acSAlexander Motin icl_soft_pdu_call_cb(struct icl_pdu *ip) 1859a4510acSAlexander Motin { 1869a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 1879a4510acSAlexander Motin 1889a4510acSAlexander Motin if (isp->cb != NULL) 1899a4510acSAlexander Motin isp->cb(ip, isp->error); 1909a4510acSAlexander Motin #ifdef DIAGNOSTIC 1919a4510acSAlexander Motin refcount_release(&ip->ip_conn->ic_outstanding_pdus); 1929a4510acSAlexander Motin #endif 1939a4510acSAlexander Motin uma_zfree(icl_soft_pdu_zone, isp); 1949a4510acSAlexander Motin } 1959a4510acSAlexander Motin 1969a4510acSAlexander Motin static void 1979a4510acSAlexander Motin icl_soft_pdu_done(struct icl_pdu *ip, int error) 1989a4510acSAlexander Motin { 1999a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 2009a4510acSAlexander Motin 2019a4510acSAlexander Motin if (error != 0) 2029a4510acSAlexander Motin isp->error = error; 2039a4510acSAlexander Motin 2049a4510acSAlexander Motin m_freem(ip->ip_bhs_mbuf); 2059a4510acSAlexander Motin ip->ip_bhs_mbuf = NULL; 2069a4510acSAlexander Motin m_freem(ip->ip_ahs_mbuf); 2079a4510acSAlexander Motin ip->ip_ahs_mbuf = NULL; 2089a4510acSAlexander Motin m_freem(ip->ip_data_mbuf); 2099a4510acSAlexander Motin ip->ip_data_mbuf = NULL; 2109a4510acSAlexander Motin 2119a4510acSAlexander Motin if (atomic_fetchadd_int(&isp->ref_cnt, -1) == 1) 2129a4510acSAlexander Motin icl_soft_pdu_call_cb(ip); 2139a4510acSAlexander Motin } 2149a4510acSAlexander Motin 2159a4510acSAlexander Motin static void 2169a4510acSAlexander Motin icl_soft_mbuf_done(struct mbuf *mb) 2179a4510acSAlexander Motin { 2189a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)mb->m_ext.ext_arg1; 2199a4510acSAlexander Motin 2209a4510acSAlexander Motin icl_soft_pdu_call_cb(&isp->ip); 2219a4510acSAlexander Motin } 2229a4510acSAlexander Motin 22382f7fa7aSAlexander Motin /* 22482f7fa7aSAlexander Motin * Allocate icl_pdu with empty BHS to fill up by the caller. 22582f7fa7aSAlexander Motin */ 22682f7fa7aSAlexander Motin struct icl_pdu * 22782f7fa7aSAlexander Motin icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) 228321b17ecSEdward Tomasz Napierala { 2299a4510acSAlexander Motin struct icl_soft_pdu *isp; 230321b17ecSEdward Tomasz Napierala struct icl_pdu *ip; 231321b17ecSEdward Tomasz Napierala 232321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 233321b17ecSEdward Tomasz Napierala refcount_acquire(&ic->ic_outstanding_pdus); 234321b17ecSEdward Tomasz Napierala #endif 2359a4510acSAlexander Motin isp = uma_zalloc(icl_soft_pdu_zone, flags | M_ZERO); 2369a4510acSAlexander Motin if (isp == NULL) { 2379a4510acSAlexander Motin ICL_WARN("failed to allocate soft PDU"); 238321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 239321b17ecSEdward Tomasz Napierala refcount_release(&ic->ic_outstanding_pdus); 240321b17ecSEdward Tomasz Napierala #endif 241321b17ecSEdward Tomasz Napierala return (NULL); 242321b17ecSEdward Tomasz Napierala } 2439a4510acSAlexander Motin ip = &isp->ip; 244321b17ecSEdward Tomasz Napierala ip->ip_conn = ic; 245321b17ecSEdward Tomasz Napierala 24633d9db92SAlexander Motin CTASSERT(sizeof(struct iscsi_bhs) <= MHLEN); 24733d9db92SAlexander Motin ip->ip_bhs_mbuf = m_gethdr(flags, MT_DATA); 248321b17ecSEdward Tomasz Napierala if (ip->ip_bhs_mbuf == NULL) { 249d0d587c7SAlexander Motin ICL_WARN("failed to allocate BHS mbuf"); 25082f7fa7aSAlexander Motin icl_soft_conn_pdu_free(ic, ip); 251321b17ecSEdward Tomasz Napierala return (NULL); 252321b17ecSEdward Tomasz Napierala } 253321b17ecSEdward Tomasz Napierala ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); 254321b17ecSEdward Tomasz Napierala memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); 255321b17ecSEdward Tomasz Napierala ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); 256321b17ecSEdward Tomasz Napierala 257321b17ecSEdward Tomasz Napierala return (ip); 258321b17ecSEdward Tomasz Napierala } 259321b17ecSEdward Tomasz Napierala 260321b17ecSEdward Tomasz Napierala static int 261321b17ecSEdward Tomasz Napierala icl_pdu_ahs_length(const struct icl_pdu *request) 262321b17ecSEdward Tomasz Napierala { 263321b17ecSEdward Tomasz Napierala 264321b17ecSEdward Tomasz Napierala return (request->ip_bhs->bhs_total_ahs_len * 4); 265321b17ecSEdward Tomasz Napierala } 266321b17ecSEdward Tomasz Napierala 267321b17ecSEdward Tomasz Napierala static size_t 268321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_length(const struct icl_pdu *request) 269321b17ecSEdward Tomasz Napierala { 270321b17ecSEdward Tomasz Napierala uint32_t len = 0; 271321b17ecSEdward Tomasz Napierala 272321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[0]; 273321b17ecSEdward Tomasz Napierala len <<= 8; 274321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[1]; 275321b17ecSEdward Tomasz Napierala len <<= 8; 276321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[2]; 277321b17ecSEdward Tomasz Napierala 278321b17ecSEdward Tomasz Napierala return (len); 279321b17ecSEdward Tomasz Napierala } 280321b17ecSEdward Tomasz Napierala 281321b17ecSEdward Tomasz Napierala size_t 282321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, 283321b17ecSEdward Tomasz Napierala const struct icl_pdu *request) 284321b17ecSEdward Tomasz Napierala { 285321b17ecSEdward Tomasz Napierala 286321b17ecSEdward Tomasz Napierala return (icl_pdu_data_segment_length(request)); 287321b17ecSEdward Tomasz Napierala } 288321b17ecSEdward Tomasz Napierala 289321b17ecSEdward Tomasz Napierala static void 290321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) 291321b17ecSEdward Tomasz Napierala { 292321b17ecSEdward Tomasz Napierala 293321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[2] = len; 294321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[1] = len >> 8; 295321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[0] = len >> 16; 296321b17ecSEdward Tomasz Napierala } 297321b17ecSEdward Tomasz Napierala 298321b17ecSEdward Tomasz Napierala static size_t 299321b17ecSEdward Tomasz Napierala icl_pdu_padding(const struct icl_pdu *ip) 300321b17ecSEdward Tomasz Napierala { 301321b17ecSEdward Tomasz Napierala 302321b17ecSEdward Tomasz Napierala if ((ip->ip_data_len % 4) != 0) 303321b17ecSEdward Tomasz Napierala return (4 - (ip->ip_data_len % 4)); 304321b17ecSEdward Tomasz Napierala 305321b17ecSEdward Tomasz Napierala return (0); 306321b17ecSEdward Tomasz Napierala } 307321b17ecSEdward Tomasz Napierala 308321b17ecSEdward Tomasz Napierala static size_t 309321b17ecSEdward Tomasz Napierala icl_pdu_size(const struct icl_pdu *response) 310321b17ecSEdward Tomasz Napierala { 311321b17ecSEdward Tomasz Napierala size_t len; 312321b17ecSEdward Tomasz Napierala 313321b17ecSEdward Tomasz Napierala KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); 314321b17ecSEdward Tomasz Napierala 315321b17ecSEdward Tomasz Napierala len = sizeof(struct iscsi_bhs) + response->ip_data_len + 316321b17ecSEdward Tomasz Napierala icl_pdu_padding(response); 317321b17ecSEdward Tomasz Napierala if (response->ip_conn->ic_header_crc32c) 318321b17ecSEdward Tomasz Napierala len += ISCSI_HEADER_DIGEST_SIZE; 319321b17ecSEdward Tomasz Napierala if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) 320321b17ecSEdward Tomasz Napierala len += ISCSI_DATA_DIGEST_SIZE; 321321b17ecSEdward Tomasz Napierala 322321b17ecSEdward Tomasz Napierala return (len); 323321b17ecSEdward Tomasz Napierala } 324321b17ecSEdward Tomasz Napierala 3256895f89fSAlexander Motin static void 3266895f89fSAlexander Motin icl_soft_receive_buf(struct mbuf **r, size_t *rs, void *buf, size_t s) 327321b17ecSEdward Tomasz Napierala { 328321b17ecSEdward Tomasz Napierala 3296895f89fSAlexander Motin m_copydata(*r, 0, s, buf); 3306895f89fSAlexander Motin m_adj(*r, s); 3316895f89fSAlexander Motin while ((*r) != NULL && (*r)->m_len == 0) 3326895f89fSAlexander Motin *r = m_free(*r); 3336895f89fSAlexander Motin *rs -= s; 334321b17ecSEdward Tomasz Napierala } 335321b17ecSEdward Tomasz Napierala 3366895f89fSAlexander Motin static void 3376895f89fSAlexander Motin icl_pdu_receive_ahs(struct icl_pdu *request, struct mbuf **r, size_t *rs) 338321b17ecSEdward Tomasz Napierala { 339321b17ecSEdward Tomasz Napierala 340321b17ecSEdward Tomasz Napierala request->ip_ahs_len = icl_pdu_ahs_length(request); 341321b17ecSEdward Tomasz Napierala if (request->ip_ahs_len == 0) 3426895f89fSAlexander Motin return; 343321b17ecSEdward Tomasz Napierala 3446895f89fSAlexander Motin request->ip_ahs_mbuf = *r; 3456895f89fSAlexander Motin *r = m_split(request->ip_ahs_mbuf, request->ip_ahs_len, M_WAITOK); 3466895f89fSAlexander Motin *rs -= request->ip_ahs_len; 347321b17ecSEdward Tomasz Napierala } 348321b17ecSEdward Tomasz Napierala 349321b17ecSEdward Tomasz Napierala static uint32_t 350321b17ecSEdward Tomasz Napierala icl_mbuf_to_crc32c(const struct mbuf *m0) 351321b17ecSEdward Tomasz Napierala { 352321b17ecSEdward Tomasz Napierala uint32_t digest = 0xffffffff; 353321b17ecSEdward Tomasz Napierala const struct mbuf *m; 354321b17ecSEdward Tomasz Napierala 355321b17ecSEdward Tomasz Napierala for (m = m0; m != NULL; m = m->m_next) 356321b17ecSEdward Tomasz Napierala digest = calculate_crc32c(digest, 357321b17ecSEdward Tomasz Napierala mtod(m, const void *), m->m_len); 358321b17ecSEdward Tomasz Napierala 359321b17ecSEdward Tomasz Napierala digest = digest ^ 0xffffffff; 360321b17ecSEdward Tomasz Napierala 361321b17ecSEdward Tomasz Napierala return (digest); 362321b17ecSEdward Tomasz Napierala } 363321b17ecSEdward Tomasz Napierala 364321b17ecSEdward Tomasz Napierala static int 3656895f89fSAlexander Motin icl_pdu_check_header_digest(struct icl_pdu *request, struct mbuf **r, size_t *rs) 366321b17ecSEdward Tomasz Napierala { 367321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 368321b17ecSEdward Tomasz Napierala 369321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_header_crc32c == false) 370321b17ecSEdward Tomasz Napierala return (0); 371321b17ecSEdward Tomasz Napierala 372d0d587c7SAlexander Motin CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); 3736895f89fSAlexander Motin icl_soft_receive_buf(r, rs, &received_digest, ISCSI_HEADER_DIGEST_SIZE); 374321b17ecSEdward Tomasz Napierala 375875ac6cfSAlexander Motin /* Temporary attach AHS to BHS to calculate header digest. */ 376875ac6cfSAlexander Motin request->ip_bhs_mbuf->m_next = request->ip_ahs_mbuf; 377321b17ecSEdward Tomasz Napierala valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 378875ac6cfSAlexander Motin request->ip_bhs_mbuf->m_next = NULL; 379321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 380321b17ecSEdward Tomasz Napierala ICL_WARN("header digest check failed; got 0x%x, " 381321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 382321b17ecSEdward Tomasz Napierala return (-1); 383321b17ecSEdward Tomasz Napierala } 384321b17ecSEdward Tomasz Napierala 385321b17ecSEdward Tomasz Napierala return (0); 386321b17ecSEdward Tomasz Napierala } 387321b17ecSEdward Tomasz Napierala 388321b17ecSEdward Tomasz Napierala /* 389321b17ecSEdward Tomasz Napierala * Return the number of bytes that should be waiting in the receive socket 390321b17ecSEdward Tomasz Napierala * before icl_pdu_receive_data_segment() gets called. 391321b17ecSEdward Tomasz Napierala */ 392321b17ecSEdward Tomasz Napierala static size_t 393321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(const struct icl_pdu *request) 394321b17ecSEdward Tomasz Napierala { 395321b17ecSEdward Tomasz Napierala size_t len; 396321b17ecSEdward Tomasz Napierala 397321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 398321b17ecSEdward Tomasz Napierala if (len == 0) 399321b17ecSEdward Tomasz Napierala return (0); 400321b17ecSEdward Tomasz Napierala 401321b17ecSEdward Tomasz Napierala /* 402321b17ecSEdward Tomasz Napierala * Account for the parts of data segment already read from 403321b17ecSEdward Tomasz Napierala * the socket buffer. 404321b17ecSEdward Tomasz Napierala */ 405321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 406321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 407321b17ecSEdward Tomasz Napierala 408321b17ecSEdward Tomasz Napierala /* 409321b17ecSEdward Tomasz Napierala * Don't always wait for the full data segment to be delivered 410321b17ecSEdward Tomasz Napierala * to the socket; this might badly affect performance due to 411321b17ecSEdward Tomasz Napierala * TCP window scaling. 412321b17ecSEdward Tomasz Napierala */ 413321b17ecSEdward Tomasz Napierala if (len > partial_receive_len) { 414321b17ecSEdward Tomasz Napierala #if 0 415321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data, limiting to %zd", 416321b17ecSEdward Tomasz Napierala len, partial_receive_len)); 417321b17ecSEdward Tomasz Napierala #endif 418321b17ecSEdward Tomasz Napierala len = partial_receive_len; 419321b17ecSEdward Tomasz Napierala 420321b17ecSEdward Tomasz Napierala return (len); 421321b17ecSEdward Tomasz Napierala } 422321b17ecSEdward Tomasz Napierala 423321b17ecSEdward Tomasz Napierala /* 424321b17ecSEdward Tomasz Napierala * Account for padding. Note that due to the way code is written, 425321b17ecSEdward Tomasz Napierala * the icl_pdu_receive_data_segment() must always receive padding 426321b17ecSEdward Tomasz Napierala * along with the last part of data segment, because it would be 427321b17ecSEdward Tomasz Napierala * impossible to tell whether we've already received the full data 428321b17ecSEdward Tomasz Napierala * segment including padding, or without it. 429321b17ecSEdward Tomasz Napierala */ 430321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 431321b17ecSEdward Tomasz Napierala len += 4 - (len % 4); 432321b17ecSEdward Tomasz Napierala 433321b17ecSEdward Tomasz Napierala #if 0 434321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data", len)); 435321b17ecSEdward Tomasz Napierala #endif 436321b17ecSEdward Tomasz Napierala 437321b17ecSEdward Tomasz Napierala return (len); 438321b17ecSEdward Tomasz Napierala } 439321b17ecSEdward Tomasz Napierala 440321b17ecSEdward Tomasz Napierala static int 4416895f89fSAlexander Motin icl_pdu_receive_data_segment(struct icl_pdu *request, struct mbuf **r, 4426895f89fSAlexander Motin size_t *rs, bool *more_neededp) 443321b17ecSEdward Tomasz Napierala { 444321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 445321b17ecSEdward Tomasz Napierala size_t len, padding = 0; 446321b17ecSEdward Tomasz Napierala struct mbuf *m; 447321b17ecSEdward Tomasz Napierala 448321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 449321b17ecSEdward Tomasz Napierala 450321b17ecSEdward Tomasz Napierala *more_neededp = false; 451321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 452321b17ecSEdward Tomasz Napierala 453321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 454321b17ecSEdward Tomasz Napierala if (len == 0) 455321b17ecSEdward Tomasz Napierala return (0); 456321b17ecSEdward Tomasz Napierala 457321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 458321b17ecSEdward Tomasz Napierala padding = 4 - (len % 4); 459321b17ecSEdward Tomasz Napierala 460321b17ecSEdward Tomasz Napierala /* 461321b17ecSEdward Tomasz Napierala * Account for already received parts of data segment. 462321b17ecSEdward Tomasz Napierala */ 463321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 464321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 465321b17ecSEdward Tomasz Napierala 4666895f89fSAlexander Motin if (len + padding > *rs) { 467321b17ecSEdward Tomasz Napierala /* 468321b17ecSEdward Tomasz Napierala * Not enough data in the socket buffer. Receive as much 469321b17ecSEdward Tomasz Napierala * as we can. Don't receive padding, since, obviously, it's 470321b17ecSEdward Tomasz Napierala * not the end of data segment yet. 471321b17ecSEdward Tomasz Napierala */ 472321b17ecSEdward Tomasz Napierala #if 0 473321b17ecSEdward Tomasz Napierala ICL_DEBUG("limited from %zd to %zd", 4746895f89fSAlexander Motin len + padding, *rs - padding)); 475321b17ecSEdward Tomasz Napierala #endif 4766895f89fSAlexander Motin len = *rs - padding; 477321b17ecSEdward Tomasz Napierala *more_neededp = true; 478321b17ecSEdward Tomasz Napierala padding = 0; 479321b17ecSEdward Tomasz Napierala } 480321b17ecSEdward Tomasz Napierala 481321b17ecSEdward Tomasz Napierala /* 482321b17ecSEdward Tomasz Napierala * Must not try to receive padding without at least one byte 483321b17ecSEdward Tomasz Napierala * of actual data segment. 484321b17ecSEdward Tomasz Napierala */ 485321b17ecSEdward Tomasz Napierala if (len > 0) { 4866895f89fSAlexander Motin m = *r; 4876895f89fSAlexander Motin *r = m_split(m, len + padding, M_WAITOK); 4886895f89fSAlexander Motin *rs -= len + padding; 489321b17ecSEdward Tomasz Napierala 490321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) 491321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = m; 492321b17ecSEdward Tomasz Napierala else 493321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, m); 494321b17ecSEdward Tomasz Napierala 495321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 496321b17ecSEdward Tomasz Napierala } else 497321b17ecSEdward Tomasz Napierala ICL_DEBUG("len 0"); 498321b17ecSEdward Tomasz Napierala 499321b17ecSEdward Tomasz Napierala if (*more_neededp) 500321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 501321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(request); 502321b17ecSEdward Tomasz Napierala 503321b17ecSEdward Tomasz Napierala return (0); 504321b17ecSEdward Tomasz Napierala } 505321b17ecSEdward Tomasz Napierala 506321b17ecSEdward Tomasz Napierala static int 5076895f89fSAlexander Motin icl_pdu_check_data_digest(struct icl_pdu *request, struct mbuf **r, size_t *rs) 508321b17ecSEdward Tomasz Napierala { 509321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 510321b17ecSEdward Tomasz Napierala 511321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_data_crc32c == false) 512321b17ecSEdward Tomasz Napierala return (0); 513321b17ecSEdward Tomasz Napierala 514321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0) 515321b17ecSEdward Tomasz Napierala return (0); 516321b17ecSEdward Tomasz Napierala 517d0d587c7SAlexander Motin CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); 5186895f89fSAlexander Motin icl_soft_receive_buf(r, rs, &received_digest, ISCSI_DATA_DIGEST_SIZE); 519321b17ecSEdward Tomasz Napierala 520321b17ecSEdward Tomasz Napierala /* 521321b17ecSEdward Tomasz Napierala * Note that ip_data_mbuf also contains padding; since digest 522321b17ecSEdward Tomasz Napierala * calculation is supposed to include that, we iterate over 523321b17ecSEdward Tomasz Napierala * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. 524321b17ecSEdward Tomasz Napierala */ 525321b17ecSEdward Tomasz Napierala valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 526321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 527321b17ecSEdward Tomasz Napierala ICL_WARN("data digest check failed; got 0x%x, " 528321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 529321b17ecSEdward Tomasz Napierala return (-1); 530321b17ecSEdward Tomasz Napierala } 531321b17ecSEdward Tomasz Napierala 532321b17ecSEdward Tomasz Napierala return (0); 533321b17ecSEdward Tomasz Napierala } 534321b17ecSEdward Tomasz Napierala 535321b17ecSEdward Tomasz Napierala /* 536321b17ecSEdward Tomasz Napierala * Somewhat contrary to the name, this attempts to receive only one 537321b17ecSEdward Tomasz Napierala * "part" of PDU at a time; call it repeatedly until it returns non-NULL. 538321b17ecSEdward Tomasz Napierala */ 539321b17ecSEdward Tomasz Napierala static struct icl_pdu * 5406895f89fSAlexander Motin icl_conn_receive_pdu(struct icl_conn *ic, struct mbuf **r, size_t *rs) 541321b17ecSEdward Tomasz Napierala { 542321b17ecSEdward Tomasz Napierala struct icl_pdu *request; 543321b17ecSEdward Tomasz Napierala size_t len; 5446895f89fSAlexander Motin int error = 0; 545321b17ecSEdward Tomasz Napierala bool more_needed; 546321b17ecSEdward Tomasz Napierala 547321b17ecSEdward Tomasz Napierala if (ic->ic_receive_state == ICL_CONN_STATE_BHS) { 548321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_receive_pdu == NULL, 549321b17ecSEdward Tomasz Napierala ("ic->ic_receive_pdu != NULL")); 550d0d587c7SAlexander Motin request = icl_soft_conn_new_pdu(ic, M_NOWAIT); 551321b17ecSEdward Tomasz Napierala if (request == NULL) { 552321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to allocate PDU; " 553321b17ecSEdward Tomasz Napierala "dropping connection"); 554321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 555321b17ecSEdward Tomasz Napierala return (NULL); 556321b17ecSEdward Tomasz Napierala } 557321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = request; 558321b17ecSEdward Tomasz Napierala } else { 559321b17ecSEdward Tomasz Napierala KASSERT(ic->ic_receive_pdu != NULL, 560321b17ecSEdward Tomasz Napierala ("ic->ic_receive_pdu == NULL")); 561321b17ecSEdward Tomasz Napierala request = ic->ic_receive_pdu; 562321b17ecSEdward Tomasz Napierala } 563321b17ecSEdward Tomasz Napierala 564321b17ecSEdward Tomasz Napierala switch (ic->ic_receive_state) { 565321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_BHS: 566321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving BHS"); 5676895f89fSAlexander Motin icl_soft_receive_buf(r, rs, request->ip_bhs, 5686895f89fSAlexander Motin sizeof(struct iscsi_bhs)); 569321b17ecSEdward Tomasz Napierala 570321b17ecSEdward Tomasz Napierala /* 571321b17ecSEdward Tomasz Napierala * We don't enforce any limit for AHS length; 572321b17ecSEdward Tomasz Napierala * its length is stored in 8 bit field. 573321b17ecSEdward Tomasz Napierala */ 574321b17ecSEdward Tomasz Napierala 575321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 576321b17ecSEdward Tomasz Napierala if (len > ic->ic_max_data_segment_length) { 577321b17ecSEdward Tomasz Napierala ICL_WARN("received data segment " 578b75168edSAlexander Motin "length %zd is larger than negotiated; " 579b75168edSAlexander Motin "dropping connection", len); 580321b17ecSEdward Tomasz Napierala error = EINVAL; 581321b17ecSEdward Tomasz Napierala break; 582321b17ecSEdward Tomasz Napierala } 583321b17ecSEdward Tomasz Napierala 584321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_AHS; 585321b17ecSEdward Tomasz Napierala ic->ic_receive_len = icl_pdu_ahs_length(request); 586321b17ecSEdward Tomasz Napierala break; 587321b17ecSEdward Tomasz Napierala 588321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_AHS: 589321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving AHS"); 5906895f89fSAlexander Motin icl_pdu_receive_ahs(request, r, rs); 591321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST; 592321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c == false) 593321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 594321b17ecSEdward Tomasz Napierala else 595321b17ecSEdward Tomasz Napierala ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE; 596321b17ecSEdward Tomasz Napierala break; 597321b17ecSEdward Tomasz Napierala 598321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_HEADER_DIGEST: 599321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving header digest"); 6006895f89fSAlexander Motin error = icl_pdu_check_header_digest(request, r, rs); 601321b17ecSEdward Tomasz Napierala if (error != 0) { 602321b17ecSEdward Tomasz Napierala ICL_DEBUG("header digest failed; " 603321b17ecSEdward Tomasz Napierala "dropping connection"); 604321b17ecSEdward Tomasz Napierala break; 605321b17ecSEdward Tomasz Napierala } 606321b17ecSEdward Tomasz Napierala 607321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_DATA; 608321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 609321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(request); 610321b17ecSEdward Tomasz Napierala break; 611321b17ecSEdward Tomasz Napierala 612321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA: 613321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data segment"); 6146895f89fSAlexander Motin error = icl_pdu_receive_data_segment(request, r, rs, 615321b17ecSEdward Tomasz Napierala &more_needed); 616321b17ecSEdward Tomasz Napierala if (error != 0) { 617321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data segment;" 618321b17ecSEdward Tomasz Napierala "dropping connection"); 619321b17ecSEdward Tomasz Napierala break; 620321b17ecSEdward Tomasz Napierala } 621321b17ecSEdward Tomasz Napierala 622321b17ecSEdward Tomasz Napierala if (more_needed) 623321b17ecSEdward Tomasz Napierala break; 624321b17ecSEdward Tomasz Napierala 625321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST; 626321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) 627321b17ecSEdward Tomasz Napierala ic->ic_receive_len = 0; 628321b17ecSEdward Tomasz Napierala else 629321b17ecSEdward Tomasz Napierala ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE; 630321b17ecSEdward Tomasz Napierala break; 631321b17ecSEdward Tomasz Napierala 632321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA_DIGEST: 633321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data digest"); 6346895f89fSAlexander Motin error = icl_pdu_check_data_digest(request, r, rs); 635321b17ecSEdward Tomasz Napierala if (error != 0) { 636321b17ecSEdward Tomasz Napierala ICL_DEBUG("data digest failed; " 637321b17ecSEdward Tomasz Napierala "dropping connection"); 638321b17ecSEdward Tomasz Napierala break; 639321b17ecSEdward Tomasz Napierala } 640321b17ecSEdward Tomasz Napierala 641321b17ecSEdward Tomasz Napierala /* 642321b17ecSEdward Tomasz Napierala * We've received complete PDU; reset the receive state machine 643321b17ecSEdward Tomasz Napierala * and return the PDU. 644321b17ecSEdward Tomasz Napierala */ 645321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_BHS; 646321b17ecSEdward Tomasz Napierala ic->ic_receive_len = sizeof(struct iscsi_bhs); 647321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = NULL; 648321b17ecSEdward Tomasz Napierala return (request); 649321b17ecSEdward Tomasz Napierala 650321b17ecSEdward Tomasz Napierala default: 651321b17ecSEdward Tomasz Napierala panic("invalid ic_receive_state %d\n", ic->ic_receive_state); 652321b17ecSEdward Tomasz Napierala } 653321b17ecSEdward Tomasz Napierala 654321b17ecSEdward Tomasz Napierala if (error != 0) { 655321b17ecSEdward Tomasz Napierala /* 656321b17ecSEdward Tomasz Napierala * Don't free the PDU; it's pointed to by ic->ic_receive_pdu 6575aabcd7cSEdward Tomasz Napierala * and will get freed in icl_soft_conn_close(). 658321b17ecSEdward Tomasz Napierala */ 659321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 660321b17ecSEdward Tomasz Napierala } 661321b17ecSEdward Tomasz Napierala 662321b17ecSEdward Tomasz Napierala return (NULL); 663321b17ecSEdward Tomasz Napierala } 664321b17ecSEdward Tomasz Napierala 665321b17ecSEdward Tomasz Napierala static void 6666895f89fSAlexander Motin icl_conn_receive_pdus(struct icl_conn *ic, struct mbuf **r, size_t *rs) 667321b17ecSEdward Tomasz Napierala { 668321b17ecSEdward Tomasz Napierala struct icl_pdu *response; 669321b17ecSEdward Tomasz Napierala 670321b17ecSEdward Tomasz Napierala for (;;) { 671321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) 672321b17ecSEdward Tomasz Napierala return; 673321b17ecSEdward Tomasz Napierala 674321b17ecSEdward Tomasz Napierala /* 675321b17ecSEdward Tomasz Napierala * Loop until we have a complete PDU or there is not enough 676321b17ecSEdward Tomasz Napierala * data in the socket buffer. 677321b17ecSEdward Tomasz Napierala */ 6786895f89fSAlexander Motin if (*rs < ic->ic_receive_len) { 679321b17ecSEdward Tomasz Napierala #if 0 6806895f89fSAlexander Motin ICL_DEBUG("not enough data; have %zd, need %zd", 6816895f89fSAlexander Motin *rs, ic->ic_receive_len); 682321b17ecSEdward Tomasz Napierala #endif 683321b17ecSEdward Tomasz Napierala return; 684321b17ecSEdward Tomasz Napierala } 685321b17ecSEdward Tomasz Napierala 6866895f89fSAlexander Motin response = icl_conn_receive_pdu(ic, r, rs); 687321b17ecSEdward Tomasz Napierala if (response == NULL) 688321b17ecSEdward Tomasz Napierala continue; 689321b17ecSEdward Tomasz Napierala 690321b17ecSEdward Tomasz Napierala if (response->ip_ahs_len > 0) { 691321b17ecSEdward Tomasz Napierala ICL_WARN("received PDU with unsupported " 692321b17ecSEdward Tomasz Napierala "AHS; opcode 0x%x; dropping connection", 693321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_opcode); 69482f7fa7aSAlexander Motin icl_soft_conn_pdu_free(ic, response); 695321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 696321b17ecSEdward Tomasz Napierala return; 697321b17ecSEdward Tomasz Napierala } 698321b17ecSEdward Tomasz Napierala 699321b17ecSEdward Tomasz Napierala (ic->ic_receive)(response); 700321b17ecSEdward Tomasz Napierala } 701321b17ecSEdward Tomasz Napierala } 702321b17ecSEdward Tomasz Napierala 703321b17ecSEdward Tomasz Napierala static void 704321b17ecSEdward Tomasz Napierala icl_receive_thread(void *arg) 705321b17ecSEdward Tomasz Napierala { 706321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 7076895f89fSAlexander Motin size_t available, read = 0; 708321b17ecSEdward Tomasz Napierala struct socket *so; 7096895f89fSAlexander Motin struct mbuf *m, *r = NULL; 7106895f89fSAlexander Motin struct uio uio; 7116895f89fSAlexander Motin int error, flags; 712321b17ecSEdward Tomasz Napierala 713321b17ecSEdward Tomasz Napierala ic = arg; 714321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 715321b17ecSEdward Tomasz Napierala 716321b17ecSEdward Tomasz Napierala for (;;) { 7176895f89fSAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 718321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 7196895f89fSAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 720321b17ecSEdward Tomasz Napierala break; 721321b17ecSEdward Tomasz Napierala } 722321b17ecSEdward Tomasz Napierala 723321b17ecSEdward Tomasz Napierala /* 724321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 725321b17ecSEdward Tomasz Napierala * soreadable() in icl_soupcall_receive() 726266078c6SPedro F. Giffuni * to avoid unnecessary wakeups until there 727321b17ecSEdward Tomasz Napierala * is enough data received to read the PDU. 728321b17ecSEdward Tomasz Napierala */ 729321b17ecSEdward Tomasz Napierala available = sbavail(&so->so_rcv); 7306895f89fSAlexander Motin if (read + available < ic->ic_receive_len) { 7316895f89fSAlexander Motin so->so_rcv.sb_lowat = ic->ic_receive_len - read; 732321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx); 733321b17ecSEdward Tomasz Napierala so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; 7346895f89fSAlexander Motin available = sbavail(&so->so_rcv); 7356895f89fSAlexander Motin } 736321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_rcv); 737321b17ecSEdward Tomasz Napierala 7386895f89fSAlexander Motin if (available == 0) { 7396895f89fSAlexander Motin if (so->so_error != 0) { 7406895f89fSAlexander Motin ICL_DEBUG("connection error %d; " 7416895f89fSAlexander Motin "dropping connection", so->so_error); 7426895f89fSAlexander Motin icl_conn_fail(ic); 7436895f89fSAlexander Motin break; 744321b17ecSEdward Tomasz Napierala } 7456895f89fSAlexander Motin continue; 7466895f89fSAlexander Motin } 7476895f89fSAlexander Motin 7486895f89fSAlexander Motin memset(&uio, 0, sizeof(uio)); 7496895f89fSAlexander Motin uio.uio_resid = available; 7506895f89fSAlexander Motin flags = MSG_DONTWAIT; 7516895f89fSAlexander Motin error = soreceive(so, NULL, &uio, &m, NULL, &flags); 7526895f89fSAlexander Motin if (error != 0) { 7536895f89fSAlexander Motin ICL_DEBUG("soreceive error %d", error); 7546895f89fSAlexander Motin break; 7556895f89fSAlexander Motin } 7566895f89fSAlexander Motin if (uio.uio_resid != 0) { 7576895f89fSAlexander Motin m_freem(m); 7586895f89fSAlexander Motin ICL_DEBUG("short read"); 7596895f89fSAlexander Motin break; 7606895f89fSAlexander Motin } 7616895f89fSAlexander Motin if (r) 7626895f89fSAlexander Motin m_cat(r, m); 7636895f89fSAlexander Motin else 7646895f89fSAlexander Motin r = m; 7656895f89fSAlexander Motin read += available; 7666895f89fSAlexander Motin 7676895f89fSAlexander Motin icl_conn_receive_pdus(ic, &r, &read); 7686895f89fSAlexander Motin } 7696895f89fSAlexander Motin 7706895f89fSAlexander Motin if (r) 7716895f89fSAlexander Motin m_freem(r); 772321b17ecSEdward Tomasz Napierala 773321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 774321b17ecSEdward Tomasz Napierala ic->ic_receive_running = false; 775321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 776321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 777321b17ecSEdward Tomasz Napierala kthread_exit(); 778321b17ecSEdward Tomasz Napierala } 779321b17ecSEdward Tomasz Napierala 780321b17ecSEdward Tomasz Napierala static int 781321b17ecSEdward Tomasz Napierala icl_soupcall_receive(struct socket *so, void *arg, int waitflag) 782321b17ecSEdward Tomasz Napierala { 783321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 784321b17ecSEdward Tomasz Napierala 785321b17ecSEdward Tomasz Napierala if (!soreadable(so)) 786321b17ecSEdward Tomasz Napierala return (SU_OK); 787321b17ecSEdward Tomasz Napierala 788321b17ecSEdward Tomasz Napierala ic = arg; 789321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_receive_cv); 790321b17ecSEdward Tomasz Napierala return (SU_OK); 791321b17ecSEdward Tomasz Napierala } 792321b17ecSEdward Tomasz Napierala 793321b17ecSEdward Tomasz Napierala static int 794321b17ecSEdward Tomasz Napierala icl_pdu_finalize(struct icl_pdu *request) 795321b17ecSEdward Tomasz Napierala { 796321b17ecSEdward Tomasz Napierala size_t padding, pdu_len; 797321b17ecSEdward Tomasz Napierala uint32_t digest, zero = 0; 798321b17ecSEdward Tomasz Napierala int ok; 799321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 800321b17ecSEdward Tomasz Napierala 801321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 802321b17ecSEdward Tomasz Napierala 803321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(request, request->ip_data_len); 804321b17ecSEdward Tomasz Napierala 805321b17ecSEdward Tomasz Napierala pdu_len = icl_pdu_size(request); 806321b17ecSEdward Tomasz Napierala 807321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c) { 808321b17ecSEdward Tomasz Napierala digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf); 809321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_bhs_mbuf, sizeof(digest), 810321b17ecSEdward Tomasz Napierala (void *)&digest); 811321b17ecSEdward Tomasz Napierala if (ok != 1) { 812321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append header digest"); 813321b17ecSEdward Tomasz Napierala return (1); 814321b17ecSEdward Tomasz Napierala } 815321b17ecSEdward Tomasz Napierala } 816321b17ecSEdward Tomasz Napierala 817321b17ecSEdward Tomasz Napierala if (request->ip_data_len != 0) { 818321b17ecSEdward Tomasz Napierala padding = icl_pdu_padding(request); 819321b17ecSEdward Tomasz Napierala if (padding > 0) { 820321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, padding, 821321b17ecSEdward Tomasz Napierala (void *)&zero); 822321b17ecSEdward Tomasz Napierala if (ok != 1) { 823321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append padding"); 824321b17ecSEdward Tomasz Napierala return (1); 825321b17ecSEdward Tomasz Napierala } 826321b17ecSEdward Tomasz Napierala } 827321b17ecSEdward Tomasz Napierala 828321b17ecSEdward Tomasz Napierala if (ic->ic_data_crc32c) { 829321b17ecSEdward Tomasz Napierala digest = icl_mbuf_to_crc32c(request->ip_data_mbuf); 830321b17ecSEdward Tomasz Napierala 831321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, sizeof(digest), 832321b17ecSEdward Tomasz Napierala (void *)&digest); 833321b17ecSEdward Tomasz Napierala if (ok != 1) { 834321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append data digest"); 835321b17ecSEdward Tomasz Napierala return (1); 836321b17ecSEdward Tomasz Napierala } 837321b17ecSEdward Tomasz Napierala } 838321b17ecSEdward Tomasz Napierala 839321b17ecSEdward Tomasz Napierala m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); 840321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = NULL; 841321b17ecSEdward Tomasz Napierala } 842321b17ecSEdward Tomasz Napierala 843321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; 844321b17ecSEdward Tomasz Napierala 845321b17ecSEdward Tomasz Napierala return (0); 846321b17ecSEdward Tomasz Napierala } 847321b17ecSEdward Tomasz Napierala 848321b17ecSEdward Tomasz Napierala static void 849321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue) 850321b17ecSEdward Tomasz Napierala { 851321b17ecSEdward Tomasz Napierala struct icl_pdu *request, *request2; 852b85a67f5SAlexander Motin struct mbuf *m; 853321b17ecSEdward Tomasz Napierala struct socket *so; 854605703b5SAlexander Motin long available, size, size2; 855321b17ecSEdward Tomasz Napierala int coalesced, error; 856321b17ecSEdward Tomasz Napierala 857321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 858321b17ecSEdward Tomasz Napierala 859321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 860321b17ecSEdward Tomasz Napierala 861321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 862321b17ecSEdward Tomasz Napierala /* 863321b17ecSEdward Tomasz Napierala * Check how much space do we have for transmit. We can't just 864321b17ecSEdward Tomasz Napierala * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, 865321b17ecSEdward Tomasz Napierala * as it always frees the mbuf chain passed to it, even in case 866321b17ecSEdward Tomasz Napierala * of error. 867321b17ecSEdward Tomasz Napierala */ 868321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 869321b17ecSEdward Tomasz Napierala 870321b17ecSEdward Tomasz Napierala /* 871321b17ecSEdward Tomasz Napierala * Notify the socket upcall that we don't need wakeups 872321b17ecSEdward Tomasz Napierala * for the time being. 873321b17ecSEdward Tomasz Napierala */ 874321b17ecSEdward Tomasz Napierala so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; 875321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 876321b17ecSEdward Tomasz Napierala 877321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(queue)) { 878321b17ecSEdward Tomasz Napierala request = STAILQ_FIRST(queue); 879321b17ecSEdward Tomasz Napierala size = icl_pdu_size(request); 880321b17ecSEdward Tomasz Napierala if (available < size) { 881321b17ecSEdward Tomasz Napierala /* 882321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 883321b17ecSEdward Tomasz Napierala * sowriteable() in icl_soupcall_send() 884266078c6SPedro F. Giffuni * to avoid unnecessary wakeups until there 885321b17ecSEdward Tomasz Napierala * is enough space for the PDU to fit. 886321b17ecSEdward Tomasz Napierala */ 887321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 888321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 889321b17ecSEdward Tomasz Napierala if (available < size) { 890321b17ecSEdward Tomasz Napierala #if 1 891321b17ecSEdward Tomasz Napierala ICL_DEBUG("no space to send; " 892605703b5SAlexander Motin "have %ld, need %ld", 893321b17ecSEdward Tomasz Napierala available, size); 894321b17ecSEdward Tomasz Napierala #endif 8951f29b46cSAlexander Motin so->so_snd.sb_lowat = max(size, 8961f29b46cSAlexander Motin so->so_snd.sb_hiwat / 8); 897321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 898321b17ecSEdward Tomasz Napierala return; 899321b17ecSEdward Tomasz Napierala } 900321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 901321b17ecSEdward Tomasz Napierala } 902321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 903321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request); 904321b17ecSEdward Tomasz Napierala if (error != 0) { 905321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 906321b17ecSEdward Tomasz Napierala "dropping connection"); 9079a4510acSAlexander Motin icl_soft_pdu_done(request, EIO); 908321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 909321b17ecSEdward Tomasz Napierala return; 910321b17ecSEdward Tomasz Napierala } 911321b17ecSEdward Tomasz Napierala if (coalesce) { 912b85a67f5SAlexander Motin m = request->ip_bhs_mbuf; 913b85a67f5SAlexander Motin for (coalesced = 1; ; coalesced++) { 914321b17ecSEdward Tomasz Napierala request2 = STAILQ_FIRST(queue); 915321b17ecSEdward Tomasz Napierala if (request2 == NULL) 916321b17ecSEdward Tomasz Napierala break; 917321b17ecSEdward Tomasz Napierala size2 = icl_pdu_size(request2); 918321b17ecSEdward Tomasz Napierala if (available < size + size2) 919321b17ecSEdward Tomasz Napierala break; 920321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 921321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request2); 922321b17ecSEdward Tomasz Napierala if (error != 0) { 923321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 924321b17ecSEdward Tomasz Napierala "dropping connection"); 9259a4510acSAlexander Motin icl_soft_pdu_done(request, EIO); 9269a4510acSAlexander Motin icl_soft_pdu_done(request2, EIO); 927321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 928321b17ecSEdward Tomasz Napierala return; 929321b17ecSEdward Tomasz Napierala } 930b85a67f5SAlexander Motin while (m->m_next) 931b85a67f5SAlexander Motin m = m->m_next; 932b85a67f5SAlexander Motin m_cat(m, request2->ip_bhs_mbuf); 933321b17ecSEdward Tomasz Napierala request2->ip_bhs_mbuf = NULL; 934321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len += size2; 935321b17ecSEdward Tomasz Napierala size += size2; 9369a4510acSAlexander Motin icl_soft_pdu_done(request2, 0); 937321b17ecSEdward Tomasz Napierala } 938321b17ecSEdward Tomasz Napierala #if 0 939321b17ecSEdward Tomasz Napierala if (coalesced > 1) { 940605703b5SAlexander Motin ICL_DEBUG("coalesced %d PDUs into %ld bytes", 941321b17ecSEdward Tomasz Napierala coalesced, size); 942321b17ecSEdward Tomasz Napierala } 943321b17ecSEdward Tomasz Napierala #endif 944321b17ecSEdward Tomasz Napierala } 945321b17ecSEdward Tomasz Napierala available -= size; 946321b17ecSEdward Tomasz Napierala error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, 947321b17ecSEdward Tomasz Napierala NULL, MSG_DONTWAIT, curthread); 948321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ 949321b17ecSEdward Tomasz Napierala if (error != 0) { 950321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to send PDU, error %d; " 951321b17ecSEdward Tomasz Napierala "dropping connection", error); 9529a4510acSAlexander Motin icl_soft_pdu_done(request, error); 953321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 954321b17ecSEdward Tomasz Napierala return; 955321b17ecSEdward Tomasz Napierala } 9569a4510acSAlexander Motin icl_soft_pdu_done(request, 0); 957321b17ecSEdward Tomasz Napierala } 958321b17ecSEdward Tomasz Napierala } 959321b17ecSEdward Tomasz Napierala 960321b17ecSEdward Tomasz Napierala static void 961321b17ecSEdward Tomasz Napierala icl_send_thread(void *arg) 962321b17ecSEdward Tomasz Napierala { 963321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 964321b17ecSEdward Tomasz Napierala struct icl_pdu_stailq queue; 965321b17ecSEdward Tomasz Napierala 966321b17ecSEdward Tomasz Napierala ic = arg; 967321b17ecSEdward Tomasz Napierala 968321b17ecSEdward Tomasz Napierala STAILQ_INIT(&queue); 969321b17ecSEdward Tomasz Napierala 970321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 971321b17ecSEdward Tomasz Napierala for (;;) { 972321b17ecSEdward Tomasz Napierala for (;;) { 973321b17ecSEdward Tomasz Napierala /* 974*df3747c6SAlexander Motin * Populate the local queue from the main one. 975*df3747c6SAlexander Motin * This way the icl_conn_send_pdus() can go through 976*df3747c6SAlexander Motin * all the queued PDUs without holding any locks. 977321b17ecSEdward Tomasz Napierala */ 978*df3747c6SAlexander Motin STAILQ_CONCAT(&queue, &ic->ic_to_send); 979321b17ecSEdward Tomasz Napierala 980321b17ecSEdward Tomasz Napierala ic->ic_check_send_space = false; 981321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 982321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(ic, &queue); 983321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 984321b17ecSEdward Tomasz Napierala 985321b17ecSEdward Tomasz Napierala /* 986321b17ecSEdward Tomasz Napierala * The icl_soupcall_send() was called since the last 987321b17ecSEdward Tomasz Napierala * call to sbspace(); go around; 988321b17ecSEdward Tomasz Napierala */ 989321b17ecSEdward Tomasz Napierala if (ic->ic_check_send_space) 990321b17ecSEdward Tomasz Napierala continue; 991321b17ecSEdward Tomasz Napierala 992321b17ecSEdward Tomasz Napierala /* 993321b17ecSEdward Tomasz Napierala * Local queue is empty, but we still have PDUs 994321b17ecSEdward Tomasz Napierala * in the main one; go around. 995321b17ecSEdward Tomasz Napierala */ 996321b17ecSEdward Tomasz Napierala if (STAILQ_EMPTY(&queue) && 997321b17ecSEdward Tomasz Napierala !STAILQ_EMPTY(&ic->ic_to_send)) 998321b17ecSEdward Tomasz Napierala continue; 999321b17ecSEdward Tomasz Napierala 1000321b17ecSEdward Tomasz Napierala /* 1001321b17ecSEdward Tomasz Napierala * There might be some stuff in the local queue, 1002321b17ecSEdward Tomasz Napierala * which didn't get sent due to not having enough send 1003321b17ecSEdward Tomasz Napierala * space. Wait for socket upcall. 1004321b17ecSEdward Tomasz Napierala */ 1005321b17ecSEdward Tomasz Napierala break; 1006321b17ecSEdward Tomasz Napierala } 1007321b17ecSEdward Tomasz Napierala 1008321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 1009321b17ecSEdward Tomasz Napierala //ICL_DEBUG("terminating"); 1010321b17ecSEdward Tomasz Napierala break; 1011321b17ecSEdward Tomasz Napierala } 1012321b17ecSEdward Tomasz Napierala 1013321b17ecSEdward Tomasz Napierala cv_wait(&ic->ic_send_cv, ic->ic_lock); 1014321b17ecSEdward Tomasz Napierala } 1015321b17ecSEdward Tomasz Napierala 1016321b17ecSEdward Tomasz Napierala /* 1017321b17ecSEdward Tomasz Napierala * We're exiting; move PDUs back to the main queue, so they can 1018321b17ecSEdward Tomasz Napierala * get freed properly. At this point ordering doesn't matter. 1019321b17ecSEdward Tomasz Napierala */ 1020321b17ecSEdward Tomasz Napierala STAILQ_CONCAT(&ic->ic_to_send, &queue); 1021321b17ecSEdward Tomasz Napierala 1022321b17ecSEdward Tomasz Napierala ic->ic_send_running = false; 1023321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1024321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1025321b17ecSEdward Tomasz Napierala kthread_exit(); 1026321b17ecSEdward Tomasz Napierala } 1027321b17ecSEdward Tomasz Napierala 1028321b17ecSEdward Tomasz Napierala static int 1029321b17ecSEdward Tomasz Napierala icl_soupcall_send(struct socket *so, void *arg, int waitflag) 1030321b17ecSEdward Tomasz Napierala { 1031321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1032321b17ecSEdward Tomasz Napierala 1033321b17ecSEdward Tomasz Napierala if (!sowriteable(so)) 1034321b17ecSEdward Tomasz Napierala return (SU_OK); 1035321b17ecSEdward Tomasz Napierala 1036321b17ecSEdward Tomasz Napierala ic = arg; 1037321b17ecSEdward Tomasz Napierala 1038321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1039321b17ecSEdward Tomasz Napierala ic->ic_check_send_space = true; 1040321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1041321b17ecSEdward Tomasz Napierala 1042321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1043321b17ecSEdward Tomasz Napierala 1044321b17ecSEdward Tomasz Napierala return (SU_OK); 1045321b17ecSEdward Tomasz Napierala } 1046321b17ecSEdward Tomasz Napierala 1047321b17ecSEdward Tomasz Napierala static int 104882f7fa7aSAlexander Motin icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 104982f7fa7aSAlexander Motin const void *addr, size_t len, int flags) 1050321b17ecSEdward Tomasz Napierala { 10519a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)request; 1052321b17ecSEdward Tomasz Napierala struct mbuf *mb, *newmb; 1053321b17ecSEdward Tomasz Napierala size_t copylen, off = 0; 1054321b17ecSEdward Tomasz Napierala 1055321b17ecSEdward Tomasz Napierala KASSERT(len > 0, ("len == 0")); 1056321b17ecSEdward Tomasz Napierala 10579a4510acSAlexander Motin if (flags & ICL_NOCOPY) { 10589a4510acSAlexander Motin newmb = m_get(flags & ~ICL_NOCOPY, MT_DATA); 10599a4510acSAlexander Motin if (newmb == NULL) { 10609a4510acSAlexander Motin ICL_WARN("failed to allocate mbuf"); 10619a4510acSAlexander Motin return (ENOMEM); 10629a4510acSAlexander Motin } 10639a4510acSAlexander Motin 10649a4510acSAlexander Motin newmb->m_flags |= M_RDONLY; 10659a4510acSAlexander Motin m_extaddref(newmb, __DECONST(char *, addr), len, &isp->ref_cnt, 10669a4510acSAlexander Motin icl_soft_mbuf_done, isp, NULL); 10679a4510acSAlexander Motin newmb->m_len = len; 10689a4510acSAlexander Motin } else { 1069898fd11fSAlexander Motin newmb = m_getm2(NULL, len, flags, MT_DATA, 0); 1070321b17ecSEdward Tomasz Napierala if (newmb == NULL) { 1071321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate mbuf for %zd bytes", len); 1072321b17ecSEdward Tomasz Napierala return (ENOMEM); 1073321b17ecSEdward Tomasz Napierala } 1074321b17ecSEdward Tomasz Napierala 1075321b17ecSEdward Tomasz Napierala for (mb = newmb; mb != NULL; mb = mb->m_next) { 1076321b17ecSEdward Tomasz Napierala copylen = min(M_TRAILINGSPACE(mb), len - off); 1077321b17ecSEdward Tomasz Napierala memcpy(mtod(mb, char *), (const char *)addr + off, copylen); 1078321b17ecSEdward Tomasz Napierala mb->m_len = copylen; 1079321b17ecSEdward Tomasz Napierala off += copylen; 1080321b17ecSEdward Tomasz Napierala } 1081321b17ecSEdward Tomasz Napierala KASSERT(off == len, ("%s: off != len", __func__)); 10829a4510acSAlexander Motin } 1083321b17ecSEdward Tomasz Napierala 1084321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) { 1085321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = newmb; 1086321b17ecSEdward Tomasz Napierala request->ip_data_len = len; 1087321b17ecSEdward Tomasz Napierala } else { 1088321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, newmb); 1089321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 1090321b17ecSEdward Tomasz Napierala } 1091321b17ecSEdward Tomasz Napierala 1092321b17ecSEdward Tomasz Napierala return (0); 1093321b17ecSEdward Tomasz Napierala } 1094321b17ecSEdward Tomasz Napierala 1095321b17ecSEdward Tomasz Napierala void 1096321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1097321b17ecSEdward Tomasz Napierala size_t off, void *addr, size_t len) 1098321b17ecSEdward Tomasz Napierala { 1099321b17ecSEdward Tomasz Napierala 110082f7fa7aSAlexander Motin m_copydata(ip->ip_data_mbuf, off, len, addr); 1101321b17ecSEdward Tomasz Napierala } 1102321b17ecSEdward Tomasz Napierala 1103321b17ecSEdward Tomasz Napierala static void 11049a4510acSAlexander Motin icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 1105321b17ecSEdward Tomasz Napierala { 1106321b17ecSEdward Tomasz Napierala 11079a4510acSAlexander Motin icl_soft_conn_pdu_queue_cb(ic, ip, NULL); 11089a4510acSAlexander Motin } 11099a4510acSAlexander Motin 11109a4510acSAlexander Motin static void 11119a4510acSAlexander Motin icl_soft_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip, 11129a4510acSAlexander Motin icl_pdu_cb cb) 11139a4510acSAlexander Motin { 11149a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 1115321b17ecSEdward Tomasz Napierala 1116321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT(ic); 11179a4510acSAlexander Motin isp->ref_cnt++; 11189a4510acSAlexander Motin isp->cb = cb; 1119321b17ecSEdward Tomasz Napierala 1120321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1121321b17ecSEdward Tomasz Napierala ICL_DEBUG("icl_pdu_queue on closed connection"); 11229a4510acSAlexander Motin icl_soft_pdu_done(ip, ENOTCONN); 1123321b17ecSEdward Tomasz Napierala return; 1124321b17ecSEdward Tomasz Napierala } 1125321b17ecSEdward Tomasz Napierala 1126321b17ecSEdward Tomasz Napierala if (!STAILQ_EMPTY(&ic->ic_to_send)) { 1127321b17ecSEdward Tomasz Napierala STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1128321b17ecSEdward Tomasz Napierala /* 1129321b17ecSEdward Tomasz Napierala * If the queue is not empty, someone else had already 1130321b17ecSEdward Tomasz Napierala * signaled the send thread; no need to do that again, 1131321b17ecSEdward Tomasz Napierala * just return. 1132321b17ecSEdward Tomasz Napierala */ 1133321b17ecSEdward Tomasz Napierala return; 1134321b17ecSEdward Tomasz Napierala } 1135321b17ecSEdward Tomasz Napierala 1136321b17ecSEdward Tomasz Napierala STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next); 1137321b17ecSEdward Tomasz Napierala cv_signal(&ic->ic_send_cv); 1138321b17ecSEdward Tomasz Napierala } 1139321b17ecSEdward Tomasz Napierala 1140321b17ecSEdward Tomasz Napierala static struct icl_conn * 1141321b17ecSEdward Tomasz Napierala icl_soft_new_conn(const char *name, struct mtx *lock) 1142321b17ecSEdward Tomasz Napierala { 1143321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1144321b17ecSEdward Tomasz Napierala 1145321b17ecSEdward Tomasz Napierala refcount_acquire(&icl_ncons); 1146321b17ecSEdward Tomasz Napierala 1147321b17ecSEdward Tomasz Napierala ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO); 1148321b17ecSEdward Tomasz Napierala 1149321b17ecSEdward Tomasz Napierala STAILQ_INIT(&ic->ic_to_send); 1150321b17ecSEdward Tomasz Napierala ic->ic_lock = lock; 1151321b17ecSEdward Tomasz Napierala cv_init(&ic->ic_send_cv, "icl_tx"); 1152321b17ecSEdward Tomasz Napierala cv_init(&ic->ic_receive_cv, "icl_rx"); 1153321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 1154321b17ecSEdward Tomasz Napierala refcount_init(&ic->ic_outstanding_pdus, 0); 1155321b17ecSEdward Tomasz Napierala #endif 1156b75168edSAlexander Motin ic->ic_max_data_segment_length = max_data_segment_length; 1157321b17ecSEdward Tomasz Napierala ic->ic_name = name; 1158d4b195d3SEdward Tomasz Napierala ic->ic_offload = "None"; 11597deb68abSEdward Tomasz Napierala ic->ic_unmapped = false; 1160321b17ecSEdward Tomasz Napierala 1161321b17ecSEdward Tomasz Napierala return (ic); 1162321b17ecSEdward Tomasz Napierala } 1163321b17ecSEdward Tomasz Napierala 1164321b17ecSEdward Tomasz Napierala void 1165321b17ecSEdward Tomasz Napierala icl_soft_conn_free(struct icl_conn *ic) 1166321b17ecSEdward Tomasz Napierala { 1167321b17ecSEdward Tomasz Napierala 116822d3bb26SEdward Tomasz Napierala #ifdef DIAGNOSTIC 116922d3bb26SEdward Tomasz Napierala KASSERT(ic->ic_outstanding_pdus == 0, 117022d3bb26SEdward Tomasz Napierala ("destroying session with %d outstanding PDUs", 117122d3bb26SEdward Tomasz Napierala ic->ic_outstanding_pdus)); 117222d3bb26SEdward Tomasz Napierala #endif 1173321b17ecSEdward Tomasz Napierala cv_destroy(&ic->ic_send_cv); 1174321b17ecSEdward Tomasz Napierala cv_destroy(&ic->ic_receive_cv); 1175321b17ecSEdward Tomasz Napierala kobj_delete((struct kobj *)ic, M_ICL_SOFT); 1176321b17ecSEdward Tomasz Napierala refcount_release(&icl_ncons); 1177321b17ecSEdward Tomasz Napierala } 1178321b17ecSEdward Tomasz Napierala 1179321b17ecSEdward Tomasz Napierala static int 1180321b17ecSEdward Tomasz Napierala icl_conn_start(struct icl_conn *ic) 1181321b17ecSEdward Tomasz Napierala { 1182321b17ecSEdward Tomasz Napierala size_t minspace; 1183321b17ecSEdward Tomasz Napierala struct sockopt opt; 1184321b17ecSEdward Tomasz Napierala int error, one = 1; 1185321b17ecSEdward Tomasz Napierala 1186321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1187321b17ecSEdward Tomasz Napierala 1188321b17ecSEdward Tomasz Napierala /* 1189321b17ecSEdward Tomasz Napierala * XXX: Ugly hack. 1190321b17ecSEdward Tomasz Napierala */ 1191321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1192321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1193321b17ecSEdward Tomasz Napierala return (EINVAL); 1194321b17ecSEdward Tomasz Napierala } 1195321b17ecSEdward Tomasz Napierala 1196321b17ecSEdward Tomasz Napierala ic->ic_receive_state = ICL_CONN_STATE_BHS; 1197321b17ecSEdward Tomasz Napierala ic->ic_receive_len = sizeof(struct iscsi_bhs); 1198321b17ecSEdward Tomasz Napierala ic->ic_disconnecting = false; 1199321b17ecSEdward Tomasz Napierala 1200321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1201321b17ecSEdward Tomasz Napierala 1202321b17ecSEdward Tomasz Napierala /* 1203321b17ecSEdward Tomasz Napierala * For sendspace, this is required because the current code cannot 1204321b17ecSEdward Tomasz Napierala * send a PDU in pieces; thus, the minimum buffer size is equal 1205321b17ecSEdward Tomasz Napierala * to the maximum PDU size. "+4" is to account for possible padding. 1206321b17ecSEdward Tomasz Napierala */ 1207321b17ecSEdward Tomasz Napierala minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length + 1208321b17ecSEdward Tomasz Napierala ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1209321b17ecSEdward Tomasz Napierala if (sendspace < minspace) { 1210321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.sendspace too low; must be at least %zd", 1211321b17ecSEdward Tomasz Napierala minspace); 1212321b17ecSEdward Tomasz Napierala sendspace = minspace; 1213321b17ecSEdward Tomasz Napierala } 1214321b17ecSEdward Tomasz Napierala if (recvspace < minspace) { 1215321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.recvspace too low; must be at least %zd", 1216321b17ecSEdward Tomasz Napierala minspace); 1217321b17ecSEdward Tomasz Napierala recvspace = minspace; 1218321b17ecSEdward Tomasz Napierala } 1219321b17ecSEdward Tomasz Napierala 1220321b17ecSEdward Tomasz Napierala error = soreserve(ic->ic_socket, sendspace, recvspace); 1221321b17ecSEdward Tomasz Napierala if (error != 0) { 1222321b17ecSEdward Tomasz Napierala ICL_WARN("soreserve failed with error %d", error); 12235aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(ic); 1224321b17ecSEdward Tomasz Napierala return (error); 1225321b17ecSEdward Tomasz Napierala } 1226321b17ecSEdward Tomasz Napierala ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; 1227321b17ecSEdward Tomasz Napierala ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; 1228321b17ecSEdward Tomasz Napierala 1229321b17ecSEdward Tomasz Napierala /* 1230321b17ecSEdward Tomasz Napierala * Disable Nagle. 1231321b17ecSEdward Tomasz Napierala */ 1232321b17ecSEdward Tomasz Napierala bzero(&opt, sizeof(opt)); 1233321b17ecSEdward Tomasz Napierala opt.sopt_dir = SOPT_SET; 1234321b17ecSEdward Tomasz Napierala opt.sopt_level = IPPROTO_TCP; 1235321b17ecSEdward Tomasz Napierala opt.sopt_name = TCP_NODELAY; 1236321b17ecSEdward Tomasz Napierala opt.sopt_val = &one; 1237321b17ecSEdward Tomasz Napierala opt.sopt_valsize = sizeof(one); 1238321b17ecSEdward Tomasz Napierala error = sosetopt(ic->ic_socket, &opt); 1239321b17ecSEdward Tomasz Napierala if (error != 0) { 1240321b17ecSEdward Tomasz Napierala ICL_WARN("disabling TCP_NODELAY failed with error %d", error); 12415aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(ic); 1242321b17ecSEdward Tomasz Napierala return (error); 1243321b17ecSEdward Tomasz Napierala } 1244321b17ecSEdward Tomasz Napierala 1245321b17ecSEdward Tomasz Napierala /* 1246321b17ecSEdward Tomasz Napierala * Register socket upcall, to get notified about incoming PDUs 1247321b17ecSEdward Tomasz Napierala * and free space to send outgoing ones. 1248321b17ecSEdward Tomasz Napierala */ 1249321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_snd); 1250321b17ecSEdward Tomasz Napierala soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic); 1251321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1252321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 1253321b17ecSEdward Tomasz Napierala soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic); 1254321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1255321b17ecSEdward Tomasz Napierala 12565b157f21SAlexander Motin /* 12575b157f21SAlexander Motin * Start threads. 12585b157f21SAlexander Motin */ 12595b157f21SAlexander Motin ICL_CONN_LOCK(ic); 12605b157f21SAlexander Motin ic->ic_send_running = ic->ic_receive_running = true; 12615b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 12625b157f21SAlexander Motin error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", 12635b157f21SAlexander Motin ic->ic_name); 12645b157f21SAlexander Motin if (error != 0) { 12655b157f21SAlexander Motin ICL_WARN("kthread_add(9) failed with error %d", error); 12665b157f21SAlexander Motin ICL_CONN_LOCK(ic); 12675b157f21SAlexander Motin ic->ic_send_running = ic->ic_receive_running = false; 12685b157f21SAlexander Motin cv_signal(&ic->ic_send_cv); 12695b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 12705b157f21SAlexander Motin icl_soft_conn_close(ic); 12715b157f21SAlexander Motin return (error); 12725b157f21SAlexander Motin } 12735b157f21SAlexander Motin error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", 12745b157f21SAlexander Motin ic->ic_name); 12755b157f21SAlexander Motin if (error != 0) { 12765b157f21SAlexander Motin ICL_WARN("kthread_add(9) failed with error %d", error); 12775b157f21SAlexander Motin ICL_CONN_LOCK(ic); 12785b157f21SAlexander Motin ic->ic_receive_running = false; 12795b157f21SAlexander Motin cv_signal(&ic->ic_send_cv); 12805b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 12815b157f21SAlexander Motin icl_soft_conn_close(ic); 12825b157f21SAlexander Motin return (error); 12835b157f21SAlexander Motin } 12845b157f21SAlexander Motin 1285321b17ecSEdward Tomasz Napierala return (0); 1286321b17ecSEdward Tomasz Napierala } 1287321b17ecSEdward Tomasz Napierala 1288321b17ecSEdward Tomasz Napierala int 1289321b17ecSEdward Tomasz Napierala icl_soft_conn_handoff(struct icl_conn *ic, int fd) 1290321b17ecSEdward Tomasz Napierala { 1291321b17ecSEdward Tomasz Napierala struct file *fp; 1292321b17ecSEdward Tomasz Napierala struct socket *so; 1293321b17ecSEdward Tomasz Napierala cap_rights_t rights; 1294321b17ecSEdward Tomasz Napierala int error; 1295321b17ecSEdward Tomasz Napierala 1296321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1297321b17ecSEdward Tomasz Napierala 1298906a424bSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1299906a424bSEdward Tomasz Napierala /* 1300906a424bSEdward Tomasz Napierala * We're transitioning to Full Feature phase, and we don't 1301906a424bSEdward Tomasz Napierala * really care. 1302906a424bSEdward Tomasz Napierala */ 1303906a424bSEdward Tomasz Napierala if (fd == 0) { 1304906a424bSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1305906a424bSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1306906a424bSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1307906a424bSEdward Tomasz Napierala ICL_WARN("proxy handoff without connect"); 1308906a424bSEdward Tomasz Napierala return (EINVAL); 1309906a424bSEdward Tomasz Napierala } 1310906a424bSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1311906a424bSEdward Tomasz Napierala return (0); 1312906a424bSEdward Tomasz Napierala } 1313906a424bSEdward Tomasz Napierala #endif 1314906a424bSEdward Tomasz Napierala 1315321b17ecSEdward Tomasz Napierala /* 1316321b17ecSEdward Tomasz Napierala * Steal the socket from userland. 1317321b17ecSEdward Tomasz Napierala */ 1318321b17ecSEdward Tomasz Napierala error = fget(curthread, fd, 13196b3a9a0fSMateusz Guzik cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); 1320321b17ecSEdward Tomasz Napierala if (error != 0) 1321321b17ecSEdward Tomasz Napierala return (error); 1322321b17ecSEdward Tomasz Napierala if (fp->f_type != DTYPE_SOCKET) { 1323321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1324321b17ecSEdward Tomasz Napierala return (EINVAL); 1325321b17ecSEdward Tomasz Napierala } 1326321b17ecSEdward Tomasz Napierala so = fp->f_data; 1327321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) { 1328321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1329321b17ecSEdward Tomasz Napierala return (EINVAL); 1330321b17ecSEdward Tomasz Napierala } 1331321b17ecSEdward Tomasz Napierala 1332321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1333321b17ecSEdward Tomasz Napierala 1334321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1335321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1336321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1337321b17ecSEdward Tomasz Napierala return (EBUSY); 1338321b17ecSEdward Tomasz Napierala } 1339321b17ecSEdward Tomasz Napierala 1340321b17ecSEdward Tomasz Napierala ic->ic_socket = fp->f_data; 1341321b17ecSEdward Tomasz Napierala fp->f_ops = &badfileops; 1342321b17ecSEdward Tomasz Napierala fp->f_data = NULL; 1343321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1344321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1345321b17ecSEdward Tomasz Napierala 1346321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1347321b17ecSEdward Tomasz Napierala 1348321b17ecSEdward Tomasz Napierala return (error); 1349321b17ecSEdward Tomasz Napierala } 1350321b17ecSEdward Tomasz Napierala 1351321b17ecSEdward Tomasz Napierala void 13525aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(struct icl_conn *ic) 1353321b17ecSEdward Tomasz Napierala { 1354321b17ecSEdward Tomasz Napierala struct icl_pdu *pdu; 13555b157f21SAlexander Motin struct socket *so; 1356321b17ecSEdward Tomasz Napierala 13575b157f21SAlexander Motin /* 13585b157f21SAlexander Motin * Wake up the threads, so they can properly terminate. 13596895f89fSAlexander Motin * Receive thread sleeps on so->so_rcv lock, send on ic->ic_lock. 13605b157f21SAlexander Motin */ 13616895f89fSAlexander Motin ICL_CONN_LOCK(ic); 13626895f89fSAlexander Motin if (!ic->ic_disconnecting) { 13636895f89fSAlexander Motin so = ic->ic_socket; 136406e9c710SAlexander Motin if (so) 13656895f89fSAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 13665b157f21SAlexander Motin ic->ic_disconnecting = true; 136706e9c710SAlexander Motin if (so) 13686895f89fSAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 13696895f89fSAlexander Motin } 13705b157f21SAlexander Motin while (ic->ic_receive_running || ic->ic_send_running) { 13715b157f21SAlexander Motin cv_signal(&ic->ic_receive_cv); 13725b157f21SAlexander Motin cv_signal(&ic->ic_send_cv); 13735b157f21SAlexander Motin cv_wait(&ic->ic_send_cv, ic->ic_lock); 13745b157f21SAlexander Motin } 13755b157f21SAlexander Motin 13765b157f21SAlexander Motin /* Some other thread could close the connection same time. */ 13775b157f21SAlexander Motin so = ic->ic_socket; 13785b157f21SAlexander Motin if (so == NULL) { 1379321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1380321b17ecSEdward Tomasz Napierala return; 1381321b17ecSEdward Tomasz Napierala } 13825b157f21SAlexander Motin ic->ic_socket = NULL; 1383321b17ecSEdward Tomasz Napierala 1384321b17ecSEdward Tomasz Napierala /* 1385321b17ecSEdward Tomasz Napierala * Deregister socket upcalls. 1386321b17ecSEdward Tomasz Napierala */ 1387321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 13885b157f21SAlexander Motin SOCKBUF_LOCK(&so->so_snd); 13895b157f21SAlexander Motin if (so->so_snd.sb_upcall != NULL) 13905b157f21SAlexander Motin soupcall_clear(so, SO_SND); 13915b157f21SAlexander Motin SOCKBUF_UNLOCK(&so->so_snd); 13925b157f21SAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 13935b157f21SAlexander Motin if (so->so_rcv.sb_upcall != NULL) 13945b157f21SAlexander Motin soupcall_clear(so, SO_RCV); 13955b157f21SAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 13965b157f21SAlexander Motin soclose(so); 1397321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1398321b17ecSEdward Tomasz Napierala 1399321b17ecSEdward Tomasz Napierala if (ic->ic_receive_pdu != NULL) { 1400321b17ecSEdward Tomasz Napierala //ICL_DEBUG("freeing partially received PDU"); 140182f7fa7aSAlexander Motin icl_soft_conn_pdu_free(ic, ic->ic_receive_pdu); 1402321b17ecSEdward Tomasz Napierala ic->ic_receive_pdu = NULL; 1403321b17ecSEdward Tomasz Napierala } 1404321b17ecSEdward Tomasz Napierala 1405321b17ecSEdward Tomasz Napierala /* 1406321b17ecSEdward Tomasz Napierala * Remove any outstanding PDUs from the send queue. 1407321b17ecSEdward Tomasz Napierala */ 1408321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(&ic->ic_to_send)) { 1409321b17ecSEdward Tomasz Napierala pdu = STAILQ_FIRST(&ic->ic_to_send); 1410321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next); 14119a4510acSAlexander Motin icl_soft_pdu_done(pdu, ENOTCONN); 1412321b17ecSEdward Tomasz Napierala } 1413321b17ecSEdward Tomasz Napierala 1414321b17ecSEdward Tomasz Napierala KASSERT(STAILQ_EMPTY(&ic->ic_to_send), 1415321b17ecSEdward Tomasz Napierala ("destroying session with non-empty send queue")); 1416321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1417321b17ecSEdward Tomasz Napierala } 1418321b17ecSEdward Tomasz Napierala 14197a03d007SEdward Tomasz Napierala int 1420604c023fSEdward Tomasz Napierala icl_soft_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 1421604c023fSEdward Tomasz Napierala struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp) 14227a03d007SEdward Tomasz Napierala { 14237a03d007SEdward Tomasz Napierala 14247a03d007SEdward Tomasz Napierala return (0); 14257a03d007SEdward Tomasz Napierala } 14267a03d007SEdward Tomasz Napierala 14277a03d007SEdward Tomasz Napierala void 14287a03d007SEdward Tomasz Napierala icl_soft_conn_task_done(struct icl_conn *ic, void *prv) 14297a03d007SEdward Tomasz Napierala { 14307a03d007SEdward Tomasz Napierala } 14317a03d007SEdward Tomasz Napierala 14327a03d007SEdward Tomasz Napierala int 14337a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, 14347a03d007SEdward Tomasz Napierala uint32_t *transfer_tag, void **prvp) 14357a03d007SEdward Tomasz Napierala { 14367a03d007SEdward Tomasz Napierala 14377a03d007SEdward Tomasz Napierala return (0); 14387a03d007SEdward Tomasz Napierala } 14397a03d007SEdward Tomasz Napierala 14407a03d007SEdward Tomasz Napierala void 14417a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv) 14427a03d007SEdward Tomasz Napierala { 14437a03d007SEdward Tomasz Napierala } 14447a03d007SEdward Tomasz Napierala 1445321b17ecSEdward Tomasz Napierala static int 144697b84d34SNavdeep Parhar icl_soft_limits(struct icl_drv_limits *idl) 1447321b17ecSEdward Tomasz Napierala { 1448321b17ecSEdward Tomasz Napierala 1449b75168edSAlexander Motin idl->idl_max_recv_data_segment_length = max_data_segment_length; 1450b75168edSAlexander Motin idl->idl_max_send_data_segment_length = max_data_segment_length; 1451b75168edSAlexander Motin idl->idl_max_burst_length = max_burst_length; 1452b75168edSAlexander Motin idl->idl_first_burst_length = first_burst_length; 1453321b17ecSEdward Tomasz Napierala 1454321b17ecSEdward Tomasz Napierala return (0); 1455321b17ecSEdward Tomasz Napierala } 1456321b17ecSEdward Tomasz Napierala 1457321b17ecSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1458321b17ecSEdward Tomasz Napierala int 1459f41492b0SEdward Tomasz Napierala icl_soft_conn_connect(struct icl_conn *ic, int domain, int socktype, 1460f41492b0SEdward Tomasz Napierala int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa) 1461f41492b0SEdward Tomasz Napierala { 1462f41492b0SEdward Tomasz Napierala 1463f41492b0SEdward Tomasz Napierala return (icl_soft_proxy_connect(ic, domain, socktype, protocol, 1464f41492b0SEdward Tomasz Napierala from_sa, to_sa)); 1465f41492b0SEdward Tomasz Napierala } 1466f41492b0SEdward Tomasz Napierala 1467f41492b0SEdward Tomasz Napierala int 1468f41492b0SEdward Tomasz Napierala icl_soft_handoff_sock(struct icl_conn *ic, struct socket *so) 1469321b17ecSEdward Tomasz Napierala { 1470321b17ecSEdward Tomasz Napierala int error; 1471321b17ecSEdward Tomasz Napierala 1472321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1473321b17ecSEdward Tomasz Napierala 1474321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) 1475321b17ecSEdward Tomasz Napierala return (EINVAL); 1476321b17ecSEdward Tomasz Napierala 1477321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1478321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1479321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1480321b17ecSEdward Tomasz Napierala return (EBUSY); 1481321b17ecSEdward Tomasz Napierala } 1482321b17ecSEdward Tomasz Napierala ic->ic_socket = so; 1483321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1484321b17ecSEdward Tomasz Napierala 1485321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1486321b17ecSEdward Tomasz Napierala 1487321b17ecSEdward Tomasz Napierala return (error); 1488321b17ecSEdward Tomasz Napierala } 1489321b17ecSEdward Tomasz Napierala #endif /* ICL_KERNEL_PROXY */ 1490321b17ecSEdward Tomasz Napierala 1491321b17ecSEdward Tomasz Napierala static int 1492321b17ecSEdward Tomasz Napierala icl_soft_load(void) 1493321b17ecSEdward Tomasz Napierala { 1494321b17ecSEdward Tomasz Napierala int error; 1495321b17ecSEdward Tomasz Napierala 14969a4510acSAlexander Motin icl_soft_pdu_zone = uma_zcreate("icl_soft_pdu", 14979a4510acSAlexander Motin sizeof(struct icl_soft_pdu), NULL, NULL, NULL, NULL, 1498321b17ecSEdward Tomasz Napierala UMA_ALIGN_PTR, 0); 1499321b17ecSEdward Tomasz Napierala refcount_init(&icl_ncons, 0); 1500321b17ecSEdward Tomasz Napierala 1501321b17ecSEdward Tomasz Napierala /* 1502321b17ecSEdward Tomasz Napierala * The reason we call this "none" is that to the user, 1503321b17ecSEdward Tomasz Napierala * it's known as "offload driver"; "offload driver: soft" 1504321b17ecSEdward Tomasz Napierala * doesn't make much sense. 1505321b17ecSEdward Tomasz Napierala */ 1506b8911594SEdward Tomasz Napierala error = icl_register("none", false, 0, 1507b8911594SEdward Tomasz Napierala icl_soft_limits, icl_soft_new_conn); 1508321b17ecSEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1509321b17ecSEdward Tomasz Napierala 1510b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0 1511b8911594SEdward Tomasz Napierala /* 1512b8911594SEdward Tomasz Napierala * Debugging aid for kernel proxy functionality. 1513b8911594SEdward Tomasz Napierala */ 1514b8911594SEdward Tomasz Napierala error = icl_register("proxytest", true, 0, 1515b8911594SEdward Tomasz Napierala icl_soft_limits, icl_soft_new_conn); 1516b8911594SEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1517b8911594SEdward Tomasz Napierala #endif 1518b8911594SEdward Tomasz Napierala 1519321b17ecSEdward Tomasz Napierala return (error); 1520321b17ecSEdward Tomasz Napierala } 1521321b17ecSEdward Tomasz Napierala 1522321b17ecSEdward Tomasz Napierala static int 1523321b17ecSEdward Tomasz Napierala icl_soft_unload(void) 1524321b17ecSEdward Tomasz Napierala { 1525321b17ecSEdward Tomasz Napierala 1526321b17ecSEdward Tomasz Napierala if (icl_ncons != 0) 1527321b17ecSEdward Tomasz Napierala return (EBUSY); 1528321b17ecSEdward Tomasz Napierala 1529b8911594SEdward Tomasz Napierala icl_unregister("none", false); 1530b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0 1531b8911594SEdward Tomasz Napierala icl_unregister("proxytest", true); 1532b8911594SEdward Tomasz Napierala #endif 1533321b17ecSEdward Tomasz Napierala 15349a4510acSAlexander Motin uma_zdestroy(icl_soft_pdu_zone); 1535321b17ecSEdward Tomasz Napierala 1536321b17ecSEdward Tomasz Napierala return (0); 1537321b17ecSEdward Tomasz Napierala } 1538321b17ecSEdward Tomasz Napierala 1539321b17ecSEdward Tomasz Napierala static int 1540321b17ecSEdward Tomasz Napierala icl_soft_modevent(module_t mod, int what, void *arg) 1541321b17ecSEdward Tomasz Napierala { 1542321b17ecSEdward Tomasz Napierala 1543321b17ecSEdward Tomasz Napierala switch (what) { 1544321b17ecSEdward Tomasz Napierala case MOD_LOAD: 1545321b17ecSEdward Tomasz Napierala return (icl_soft_load()); 1546321b17ecSEdward Tomasz Napierala case MOD_UNLOAD: 1547321b17ecSEdward Tomasz Napierala return (icl_soft_unload()); 1548321b17ecSEdward Tomasz Napierala default: 1549321b17ecSEdward Tomasz Napierala return (EINVAL); 1550321b17ecSEdward Tomasz Napierala } 1551321b17ecSEdward Tomasz Napierala } 1552321b17ecSEdward Tomasz Napierala 1553321b17ecSEdward Tomasz Napierala moduledata_t icl_soft_data = { 1554321b17ecSEdward Tomasz Napierala "icl_soft", 1555321b17ecSEdward Tomasz Napierala icl_soft_modevent, 1556321b17ecSEdward Tomasz Napierala 0 1557321b17ecSEdward Tomasz Napierala }; 1558321b17ecSEdward Tomasz Napierala 1559321b17ecSEdward Tomasz Napierala DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 1560321b17ecSEdward Tomasz Napierala MODULE_DEPEND(icl_soft, icl, 1, 1, 1); 1561872d2d92SEdward Tomasz Napierala MODULE_VERSION(icl_soft, 1); 1562