1321b17ecSEdward Tomasz Napierala /*- 243ee6e9dSEdward Tomasz Napierala * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 343ee6e9dSEdward Tomasz Napierala * 4321b17ecSEdward Tomasz Napierala * Copyright (c) 2012 The FreeBSD Foundation 5321b17ecSEdward Tomasz Napierala * 6321b17ecSEdward Tomasz Napierala * This software was developed by Edward Tomasz Napierala under sponsorship 7321b17ecSEdward Tomasz Napierala * from the FreeBSD Foundation. 8321b17ecSEdward Tomasz Napierala * 9321b17ecSEdward Tomasz Napierala * Redistribution and use in source and binary forms, with or without 10321b17ecSEdward Tomasz Napierala * modification, are permitted provided that the following conditions 11321b17ecSEdward Tomasz Napierala * are met: 12321b17ecSEdward Tomasz Napierala * 1. Redistributions of source code must retain the above copyright 13321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer. 14321b17ecSEdward Tomasz Napierala * 2. Redistributions in binary form must reproduce the above copyright 15321b17ecSEdward Tomasz Napierala * notice, this list of conditions and the following disclaimer in the 16321b17ecSEdward Tomasz Napierala * documentation and/or other materials provided with the distribution. 17321b17ecSEdward Tomasz Napierala * 18321b17ecSEdward Tomasz Napierala * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19321b17ecSEdward Tomasz Napierala * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20321b17ecSEdward Tomasz Napierala * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21321b17ecSEdward Tomasz Napierala * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22321b17ecSEdward Tomasz Napierala * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23321b17ecSEdward Tomasz Napierala * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24321b17ecSEdward Tomasz Napierala * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25321b17ecSEdward Tomasz Napierala * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26321b17ecSEdward Tomasz Napierala * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27321b17ecSEdward Tomasz Napierala * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28321b17ecSEdward Tomasz Napierala * SUCH DAMAGE. 29321b17ecSEdward Tomasz Napierala * 30321b17ecSEdward Tomasz Napierala */ 31321b17ecSEdward Tomasz Napierala 32321b17ecSEdward Tomasz Napierala /* 335aabcd7cSEdward Tomasz Napierala * Software implementation of iSCSI Common Layer kobj(9) interface. 34321b17ecSEdward Tomasz Napierala */ 35321b17ecSEdward Tomasz Napierala 36321b17ecSEdward Tomasz Napierala #include <sys/cdefs.h> 37321b17ecSEdward Tomasz Napierala __FBSDID("$FreeBSD$"); 38321b17ecSEdward Tomasz Napierala 39321b17ecSEdward Tomasz Napierala #include <sys/param.h> 40530e725dSJohn Baldwin #include <sys/bio.h> 41321b17ecSEdward Tomasz Napierala #include <sys/capsicum.h> 42321b17ecSEdward Tomasz Napierala #include <sys/condvar.h> 43321b17ecSEdward Tomasz Napierala #include <sys/conf.h> 44f89d2072SXin LI #include <sys/gsb_crc32.h> 45321b17ecSEdward Tomasz Napierala #include <sys/file.h> 46321b17ecSEdward Tomasz Napierala #include <sys/kernel.h> 47321b17ecSEdward Tomasz Napierala #include <sys/kthread.h> 48321b17ecSEdward Tomasz Napierala #include <sys/lock.h> 49321b17ecSEdward Tomasz Napierala #include <sys/mbuf.h> 50321b17ecSEdward Tomasz Napierala #include <sys/mutex.h> 51321b17ecSEdward Tomasz Napierala #include <sys/module.h> 52321b17ecSEdward Tomasz Napierala #include <sys/protosw.h> 53321b17ecSEdward Tomasz Napierala #include <sys/socket.h> 54321b17ecSEdward Tomasz Napierala #include <sys/socketvar.h> 55321b17ecSEdward Tomasz Napierala #include <sys/sysctl.h> 56321b17ecSEdward Tomasz Napierala #include <sys/systm.h> 57321b17ecSEdward Tomasz Napierala #include <sys/sx.h> 58321b17ecSEdward Tomasz Napierala #include <sys/uio.h> 59321b17ecSEdward Tomasz Napierala #include <vm/uma.h> 60530e725dSJohn Baldwin #include <vm/vm_page.h> 61321b17ecSEdward Tomasz Napierala #include <netinet/in.h> 62321b17ecSEdward Tomasz Napierala #include <netinet/tcp.h> 63321b17ecSEdward Tomasz Napierala 64321b17ecSEdward Tomasz Napierala #include <dev/iscsi/icl.h> 65321b17ecSEdward Tomasz Napierala #include <dev/iscsi/iscsi_proto.h> 66321b17ecSEdward Tomasz Napierala #include <icl_conn_if.h> 67321b17ecSEdward Tomasz Napierala 6887322a90SJohn Baldwin #define ICL_CONN_STATE_BHS 1 6987322a90SJohn Baldwin #define ICL_CONN_STATE_AHS 2 7087322a90SJohn Baldwin #define ICL_CONN_STATE_HEADER_DIGEST 3 7187322a90SJohn Baldwin #define ICL_CONN_STATE_DATA 4 7287322a90SJohn Baldwin #define ICL_CONN_STATE_DATA_DIGEST 5 7387322a90SJohn Baldwin 7487322a90SJohn Baldwin struct icl_soft_conn { 7587322a90SJohn Baldwin struct icl_conn ic; 7687322a90SJohn Baldwin 7787322a90SJohn Baldwin /* soft specific stuff goes here. */ 7887322a90SJohn Baldwin STAILQ_HEAD(, icl_pdu) to_send; 7987322a90SJohn Baldwin struct cv send_cv; 8087322a90SJohn Baldwin struct cv receive_cv; 8187322a90SJohn Baldwin struct icl_pdu *receive_pdu; 8287322a90SJohn Baldwin size_t receive_len; 8387322a90SJohn Baldwin int receive_state; 8487322a90SJohn Baldwin bool receive_running; 8587322a90SJohn Baldwin bool check_send_space; 8687322a90SJohn Baldwin bool send_running; 8787322a90SJohn Baldwin }; 8887322a90SJohn Baldwin 899a4510acSAlexander Motin struct icl_soft_pdu { 909a4510acSAlexander Motin struct icl_pdu ip; 919a4510acSAlexander Motin 929a4510acSAlexander Motin /* soft specific stuff goes here. */ 939a4510acSAlexander Motin u_int ref_cnt; 949a4510acSAlexander Motin icl_pdu_cb cb; 959a4510acSAlexander Motin int error; 969a4510acSAlexander Motin }; 979a4510acSAlexander Motin 98b75168edSAlexander Motin SYSCTL_NODE(_kern_icl, OID_AUTO, soft, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 99b75168edSAlexander Motin "Software iSCSI"); 100321b17ecSEdward Tomasz Napierala static int coalesce = 1; 101b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, coalesce, CTLFLAG_RWTUN, 102321b17ecSEdward Tomasz Napierala &coalesce, 0, "Try to coalesce PDUs before sending"); 103b75168edSAlexander Motin static int partial_receive_len = 256 * 1024; 104b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN, 105321b17ecSEdward Tomasz Napierala &partial_receive_len, 0, "Minimum read size for partially received " 106321b17ecSEdward Tomasz Napierala "data segment"); 107b75168edSAlexander Motin static int max_data_segment_length = 256 * 1024; 108b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, max_data_segment_length, CTLFLAG_RWTUN, 109b75168edSAlexander Motin &max_data_segment_length, 0, "Maximum data segment length"); 110b75168edSAlexander Motin static int first_burst_length = 1024 * 1024; 111b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, first_burst_length, CTLFLAG_RWTUN, 112b75168edSAlexander Motin &first_burst_length, 0, "First burst length"); 113b75168edSAlexander Motin static int max_burst_length = 1024 * 1024; 114b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, max_burst_length, CTLFLAG_RWTUN, 115b75168edSAlexander Motin &max_burst_length, 0, "Maximum burst length"); 116b75168edSAlexander Motin static int sendspace = 1536 * 1024; 117b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, sendspace, CTLFLAG_RWTUN, 118321b17ecSEdward Tomasz Napierala &sendspace, 0, "Default send socket buffer size"); 119b75168edSAlexander Motin static int recvspace = 1536 * 1024; 120b75168edSAlexander Motin SYSCTL_INT(_kern_icl_soft, OID_AUTO, recvspace, CTLFLAG_RWTUN, 121321b17ecSEdward Tomasz Napierala &recvspace, 0, "Default receive socket buffer size"); 122321b17ecSEdward Tomasz Napierala 123321b17ecSEdward Tomasz Napierala static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend"); 1249a4510acSAlexander Motin static uma_zone_t icl_soft_pdu_zone; 125321b17ecSEdward Tomasz Napierala 126321b17ecSEdward Tomasz Napierala static volatile u_int icl_ncons; 127321b17ecSEdward Tomasz Napierala 128321b17ecSEdward Tomasz Napierala STAILQ_HEAD(icl_pdu_stailq, icl_pdu); 129321b17ecSEdward Tomasz Napierala 130321b17ecSEdward Tomasz Napierala static icl_conn_new_pdu_t icl_soft_conn_new_pdu; 131321b17ecSEdward Tomasz Napierala static icl_conn_pdu_free_t icl_soft_conn_pdu_free; 132321b17ecSEdward Tomasz Napierala static icl_conn_pdu_data_segment_length_t 133321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length; 134530e725dSJohn Baldwin static icl_conn_pdu_append_bio_t icl_soft_conn_pdu_append_bio; 135321b17ecSEdward Tomasz Napierala static icl_conn_pdu_append_data_t icl_soft_conn_pdu_append_data; 136530e725dSJohn Baldwin static icl_conn_pdu_get_bio_t icl_soft_conn_pdu_get_bio; 137321b17ecSEdward Tomasz Napierala static icl_conn_pdu_get_data_t icl_soft_conn_pdu_get_data; 138321b17ecSEdward Tomasz Napierala static icl_conn_pdu_queue_t icl_soft_conn_pdu_queue; 1399a4510acSAlexander Motin static icl_conn_pdu_queue_cb_t icl_soft_conn_pdu_queue_cb; 140321b17ecSEdward Tomasz Napierala static icl_conn_handoff_t icl_soft_conn_handoff; 141321b17ecSEdward Tomasz Napierala static icl_conn_free_t icl_soft_conn_free; 142321b17ecSEdward Tomasz Napierala static icl_conn_close_t icl_soft_conn_close; 1437a03d007SEdward Tomasz Napierala static icl_conn_task_setup_t icl_soft_conn_task_setup; 1447a03d007SEdward Tomasz Napierala static icl_conn_task_done_t icl_soft_conn_task_done; 1457a03d007SEdward Tomasz Napierala static icl_conn_transfer_setup_t icl_soft_conn_transfer_setup; 1467a03d007SEdward Tomasz Napierala static icl_conn_transfer_done_t icl_soft_conn_transfer_done; 147f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 148f41492b0SEdward Tomasz Napierala static icl_conn_connect_t icl_soft_conn_connect; 149f41492b0SEdward Tomasz Napierala #endif 150321b17ecSEdward Tomasz Napierala 151321b17ecSEdward Tomasz Napierala static kobj_method_t icl_soft_methods[] = { 152321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu), 153321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free), 154321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_data_segment_length, 155321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length), 156530e725dSJohn Baldwin KOBJMETHOD(icl_conn_pdu_append_bio, icl_soft_conn_pdu_append_bio), 157321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data), 158530e725dSJohn Baldwin KOBJMETHOD(icl_conn_pdu_get_bio, icl_soft_conn_pdu_get_bio), 159321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data), 160321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue), 1619a4510acSAlexander Motin KOBJMETHOD(icl_conn_pdu_queue_cb, icl_soft_conn_pdu_queue_cb), 162321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff), 163321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_free, icl_soft_conn_free), 164321b17ecSEdward Tomasz Napierala KOBJMETHOD(icl_conn_close, icl_soft_conn_close), 1657a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup), 1667a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done), 1677a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup), 1687a03d007SEdward Tomasz Napierala KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done), 169f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 170f41492b0SEdward Tomasz Napierala KOBJMETHOD(icl_conn_connect, icl_soft_conn_connect), 171f41492b0SEdward Tomasz Napierala #endif 172321b17ecSEdward Tomasz Napierala { 0, 0 } 173321b17ecSEdward Tomasz Napierala }; 174321b17ecSEdward Tomasz Napierala 17587322a90SJohn Baldwin DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_soft_conn)); 176321b17ecSEdward Tomasz Napierala 177321b17ecSEdward Tomasz Napierala static void 178321b17ecSEdward Tomasz Napierala icl_conn_fail(struct icl_conn *ic) 179321b17ecSEdward Tomasz Napierala { 180321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) 181321b17ecSEdward Tomasz Napierala return; 182321b17ecSEdward Tomasz Napierala 183321b17ecSEdward Tomasz Napierala /* 184321b17ecSEdward Tomasz Napierala * XXX 185321b17ecSEdward Tomasz Napierala */ 186321b17ecSEdward Tomasz Napierala ic->ic_socket->so_error = EDOOFUS; 187321b17ecSEdward Tomasz Napierala (ic->ic_error)(ic); 188321b17ecSEdward Tomasz Napierala } 189321b17ecSEdward Tomasz Napierala 19082f7fa7aSAlexander Motin static void 19182f7fa7aSAlexander Motin icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 19282f7fa7aSAlexander Motin { 1939a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 19482f7fa7aSAlexander Motin 1959a4510acSAlexander Motin KASSERT(isp->ref_cnt == 0, ("freeing active PDU")); 19682f7fa7aSAlexander Motin m_freem(ip->ip_bhs_mbuf); 19782f7fa7aSAlexander Motin m_freem(ip->ip_ahs_mbuf); 19882f7fa7aSAlexander Motin m_freem(ip->ip_data_mbuf); 1999a4510acSAlexander Motin uma_zfree(icl_soft_pdu_zone, isp); 20082f7fa7aSAlexander Motin #ifdef DIAGNOSTIC 20182f7fa7aSAlexander Motin refcount_release(&ic->ic_outstanding_pdus); 20282f7fa7aSAlexander Motin #endif 20382f7fa7aSAlexander Motin } 20482f7fa7aSAlexander Motin 2059a4510acSAlexander Motin static void 2069a4510acSAlexander Motin icl_soft_pdu_call_cb(struct icl_pdu *ip) 2079a4510acSAlexander Motin { 2089a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 2099a4510acSAlexander Motin 2109a4510acSAlexander Motin if (isp->cb != NULL) 2119a4510acSAlexander Motin isp->cb(ip, isp->error); 2129a4510acSAlexander Motin #ifdef DIAGNOSTIC 2139a4510acSAlexander Motin refcount_release(&ip->ip_conn->ic_outstanding_pdus); 2149a4510acSAlexander Motin #endif 2159a4510acSAlexander Motin uma_zfree(icl_soft_pdu_zone, isp); 2169a4510acSAlexander Motin } 2179a4510acSAlexander Motin 2189a4510acSAlexander Motin static void 2199a4510acSAlexander Motin icl_soft_pdu_done(struct icl_pdu *ip, int error) 2209a4510acSAlexander Motin { 2219a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 2229a4510acSAlexander Motin 2239a4510acSAlexander Motin if (error != 0) 2249a4510acSAlexander Motin isp->error = error; 2259a4510acSAlexander Motin 2269a4510acSAlexander Motin m_freem(ip->ip_bhs_mbuf); 2279a4510acSAlexander Motin ip->ip_bhs_mbuf = NULL; 2289a4510acSAlexander Motin m_freem(ip->ip_ahs_mbuf); 2299a4510acSAlexander Motin ip->ip_ahs_mbuf = NULL; 2309a4510acSAlexander Motin m_freem(ip->ip_data_mbuf); 2319a4510acSAlexander Motin ip->ip_data_mbuf = NULL; 2329a4510acSAlexander Motin 2339a4510acSAlexander Motin if (atomic_fetchadd_int(&isp->ref_cnt, -1) == 1) 2349a4510acSAlexander Motin icl_soft_pdu_call_cb(ip); 2359a4510acSAlexander Motin } 2369a4510acSAlexander Motin 2379a4510acSAlexander Motin static void 2389a4510acSAlexander Motin icl_soft_mbuf_done(struct mbuf *mb) 2399a4510acSAlexander Motin { 2409a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)mb->m_ext.ext_arg1; 2419a4510acSAlexander Motin 2429a4510acSAlexander Motin icl_soft_pdu_call_cb(&isp->ip); 2439a4510acSAlexander Motin } 2449a4510acSAlexander Motin 24582f7fa7aSAlexander Motin /* 24682f7fa7aSAlexander Motin * Allocate icl_pdu with empty BHS to fill up by the caller. 24782f7fa7aSAlexander Motin */ 24882f7fa7aSAlexander Motin struct icl_pdu * 24982f7fa7aSAlexander Motin icl_soft_conn_new_pdu(struct icl_conn *ic, int flags) 250321b17ecSEdward Tomasz Napierala { 2519a4510acSAlexander Motin struct icl_soft_pdu *isp; 252321b17ecSEdward Tomasz Napierala struct icl_pdu *ip; 253321b17ecSEdward Tomasz Napierala 254321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 255321b17ecSEdward Tomasz Napierala refcount_acquire(&ic->ic_outstanding_pdus); 256321b17ecSEdward Tomasz Napierala #endif 2579a4510acSAlexander Motin isp = uma_zalloc(icl_soft_pdu_zone, flags | M_ZERO); 2589a4510acSAlexander Motin if (isp == NULL) { 2599a4510acSAlexander Motin ICL_WARN("failed to allocate soft PDU"); 260321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 261321b17ecSEdward Tomasz Napierala refcount_release(&ic->ic_outstanding_pdus); 262321b17ecSEdward Tomasz Napierala #endif 263321b17ecSEdward Tomasz Napierala return (NULL); 264321b17ecSEdward Tomasz Napierala } 2659a4510acSAlexander Motin ip = &isp->ip; 266321b17ecSEdward Tomasz Napierala ip->ip_conn = ic; 267321b17ecSEdward Tomasz Napierala 26833d9db92SAlexander Motin CTASSERT(sizeof(struct iscsi_bhs) <= MHLEN); 26933d9db92SAlexander Motin ip->ip_bhs_mbuf = m_gethdr(flags, MT_DATA); 270321b17ecSEdward Tomasz Napierala if (ip->ip_bhs_mbuf == NULL) { 271d0d587c7SAlexander Motin ICL_WARN("failed to allocate BHS mbuf"); 27282f7fa7aSAlexander Motin icl_soft_conn_pdu_free(ic, ip); 273321b17ecSEdward Tomasz Napierala return (NULL); 274321b17ecSEdward Tomasz Napierala } 275321b17ecSEdward Tomasz Napierala ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *); 276321b17ecSEdward Tomasz Napierala memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs)); 277321b17ecSEdward Tomasz Napierala ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs); 278321b17ecSEdward Tomasz Napierala 279321b17ecSEdward Tomasz Napierala return (ip); 280321b17ecSEdward Tomasz Napierala } 281321b17ecSEdward Tomasz Napierala 282321b17ecSEdward Tomasz Napierala static int 283321b17ecSEdward Tomasz Napierala icl_pdu_ahs_length(const struct icl_pdu *request) 284321b17ecSEdward Tomasz Napierala { 285321b17ecSEdward Tomasz Napierala 286321b17ecSEdward Tomasz Napierala return (request->ip_bhs->bhs_total_ahs_len * 4); 287321b17ecSEdward Tomasz Napierala } 288321b17ecSEdward Tomasz Napierala 289321b17ecSEdward Tomasz Napierala static size_t 290321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_length(const struct icl_pdu *request) 291321b17ecSEdward Tomasz Napierala { 292321b17ecSEdward Tomasz Napierala uint32_t len = 0; 293321b17ecSEdward Tomasz Napierala 294321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[0]; 295321b17ecSEdward Tomasz Napierala len <<= 8; 296321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[1]; 297321b17ecSEdward Tomasz Napierala len <<= 8; 298321b17ecSEdward Tomasz Napierala len += request->ip_bhs->bhs_data_segment_len[2]; 299321b17ecSEdward Tomasz Napierala 300321b17ecSEdward Tomasz Napierala return (len); 301321b17ecSEdward Tomasz Napierala } 302321b17ecSEdward Tomasz Napierala 303321b17ecSEdward Tomasz Napierala size_t 304321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic, 305321b17ecSEdward Tomasz Napierala const struct icl_pdu *request) 306321b17ecSEdward Tomasz Napierala { 307321b17ecSEdward Tomasz Napierala 308321b17ecSEdward Tomasz Napierala return (icl_pdu_data_segment_length(request)); 309321b17ecSEdward Tomasz Napierala } 310321b17ecSEdward Tomasz Napierala 311321b17ecSEdward Tomasz Napierala static void 312321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len) 313321b17ecSEdward Tomasz Napierala { 314321b17ecSEdward Tomasz Napierala 315321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[2] = len; 316321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[1] = len >> 8; 317321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_data_segment_len[0] = len >> 16; 318321b17ecSEdward Tomasz Napierala } 319321b17ecSEdward Tomasz Napierala 320321b17ecSEdward Tomasz Napierala static size_t 321321b17ecSEdward Tomasz Napierala icl_pdu_padding(const struct icl_pdu *ip) 322321b17ecSEdward Tomasz Napierala { 323321b17ecSEdward Tomasz Napierala 324321b17ecSEdward Tomasz Napierala if ((ip->ip_data_len % 4) != 0) 325321b17ecSEdward Tomasz Napierala return (4 - (ip->ip_data_len % 4)); 326321b17ecSEdward Tomasz Napierala 327321b17ecSEdward Tomasz Napierala return (0); 328321b17ecSEdward Tomasz Napierala } 329321b17ecSEdward Tomasz Napierala 330321b17ecSEdward Tomasz Napierala static size_t 331321b17ecSEdward Tomasz Napierala icl_pdu_size(const struct icl_pdu *response) 332321b17ecSEdward Tomasz Napierala { 333321b17ecSEdward Tomasz Napierala size_t len; 334321b17ecSEdward Tomasz Napierala 335321b17ecSEdward Tomasz Napierala KASSERT(response->ip_ahs_len == 0, ("responding with AHS")); 336321b17ecSEdward Tomasz Napierala 337321b17ecSEdward Tomasz Napierala len = sizeof(struct iscsi_bhs) + response->ip_data_len + 338321b17ecSEdward Tomasz Napierala icl_pdu_padding(response); 339321b17ecSEdward Tomasz Napierala if (response->ip_conn->ic_header_crc32c) 340321b17ecSEdward Tomasz Napierala len += ISCSI_HEADER_DIGEST_SIZE; 341321b17ecSEdward Tomasz Napierala if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c) 342321b17ecSEdward Tomasz Napierala len += ISCSI_DATA_DIGEST_SIZE; 343321b17ecSEdward Tomasz Napierala 344321b17ecSEdward Tomasz Napierala return (len); 345321b17ecSEdward Tomasz Napierala } 346321b17ecSEdward Tomasz Napierala 3476895f89fSAlexander Motin static void 3486895f89fSAlexander Motin icl_soft_receive_buf(struct mbuf **r, size_t *rs, void *buf, size_t s) 349321b17ecSEdward Tomasz Napierala { 350321b17ecSEdward Tomasz Napierala 3516895f89fSAlexander Motin m_copydata(*r, 0, s, buf); 3526895f89fSAlexander Motin m_adj(*r, s); 3536895f89fSAlexander Motin while ((*r) != NULL && (*r)->m_len == 0) 3546895f89fSAlexander Motin *r = m_free(*r); 3556895f89fSAlexander Motin *rs -= s; 356321b17ecSEdward Tomasz Napierala } 357321b17ecSEdward Tomasz Napierala 3586895f89fSAlexander Motin static void 3596895f89fSAlexander Motin icl_pdu_receive_ahs(struct icl_pdu *request, struct mbuf **r, size_t *rs) 360321b17ecSEdward Tomasz Napierala { 361321b17ecSEdward Tomasz Napierala 362321b17ecSEdward Tomasz Napierala request->ip_ahs_len = icl_pdu_ahs_length(request); 363321b17ecSEdward Tomasz Napierala if (request->ip_ahs_len == 0) 3646895f89fSAlexander Motin return; 365321b17ecSEdward Tomasz Napierala 3666895f89fSAlexander Motin request->ip_ahs_mbuf = *r; 3676895f89fSAlexander Motin *r = m_split(request->ip_ahs_mbuf, request->ip_ahs_len, M_WAITOK); 3686895f89fSAlexander Motin *rs -= request->ip_ahs_len; 369321b17ecSEdward Tomasz Napierala } 370321b17ecSEdward Tomasz Napierala 371530e725dSJohn Baldwin static int 372530e725dSJohn Baldwin mbuf_crc32c_helper(void *arg, void *data, u_int len) 373530e725dSJohn Baldwin { 374530e725dSJohn Baldwin uint32_t *digestp = arg; 375530e725dSJohn Baldwin 376530e725dSJohn Baldwin *digestp = calculate_crc32c(*digestp, data, len); 377530e725dSJohn Baldwin return (0); 378530e725dSJohn Baldwin } 379530e725dSJohn Baldwin 380321b17ecSEdward Tomasz Napierala static uint32_t 381530e725dSJohn Baldwin icl_mbuf_to_crc32c(struct mbuf *m0, size_t len) 382321b17ecSEdward Tomasz Napierala { 383321b17ecSEdward Tomasz Napierala uint32_t digest = 0xffffffff; 384321b17ecSEdward Tomasz Napierala 385530e725dSJohn Baldwin m_apply(m0, 0, len, mbuf_crc32c_helper, &digest); 386321b17ecSEdward Tomasz Napierala digest = digest ^ 0xffffffff; 387321b17ecSEdward Tomasz Napierala 388321b17ecSEdward Tomasz Napierala return (digest); 389321b17ecSEdward Tomasz Napierala } 390321b17ecSEdward Tomasz Napierala 391321b17ecSEdward Tomasz Napierala static int 3926895f89fSAlexander Motin icl_pdu_check_header_digest(struct icl_pdu *request, struct mbuf **r, size_t *rs) 393321b17ecSEdward Tomasz Napierala { 394321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 395321b17ecSEdward Tomasz Napierala 396321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_header_crc32c == false) 397321b17ecSEdward Tomasz Napierala return (0); 398321b17ecSEdward Tomasz Napierala 399d0d587c7SAlexander Motin CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE); 4006895f89fSAlexander Motin icl_soft_receive_buf(r, rs, &received_digest, ISCSI_HEADER_DIGEST_SIZE); 401321b17ecSEdward Tomasz Napierala 402875ac6cfSAlexander Motin /* Temporary attach AHS to BHS to calculate header digest. */ 403875ac6cfSAlexander Motin request->ip_bhs_mbuf->m_next = request->ip_ahs_mbuf; 404530e725dSJohn Baldwin valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf, ISCSI_BHS_SIZE); 405875ac6cfSAlexander Motin request->ip_bhs_mbuf->m_next = NULL; 406321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 407321b17ecSEdward Tomasz Napierala ICL_WARN("header digest check failed; got 0x%x, " 408321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 409321b17ecSEdward Tomasz Napierala return (-1); 410321b17ecSEdward Tomasz Napierala } 411321b17ecSEdward Tomasz Napierala 412321b17ecSEdward Tomasz Napierala return (0); 413321b17ecSEdward Tomasz Napierala } 414321b17ecSEdward Tomasz Napierala 415321b17ecSEdward Tomasz Napierala /* 416321b17ecSEdward Tomasz Napierala * Return the number of bytes that should be waiting in the receive socket 417321b17ecSEdward Tomasz Napierala * before icl_pdu_receive_data_segment() gets called. 418321b17ecSEdward Tomasz Napierala */ 419321b17ecSEdward Tomasz Napierala static size_t 420321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(const struct icl_pdu *request) 421321b17ecSEdward Tomasz Napierala { 422321b17ecSEdward Tomasz Napierala size_t len; 423321b17ecSEdward Tomasz Napierala 424321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 425321b17ecSEdward Tomasz Napierala if (len == 0) 426321b17ecSEdward Tomasz Napierala return (0); 427321b17ecSEdward Tomasz Napierala 428321b17ecSEdward Tomasz Napierala /* 429321b17ecSEdward Tomasz Napierala * Account for the parts of data segment already read from 430321b17ecSEdward Tomasz Napierala * the socket buffer. 431321b17ecSEdward Tomasz Napierala */ 432321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 433321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 434321b17ecSEdward Tomasz Napierala 435321b17ecSEdward Tomasz Napierala /* 436321b17ecSEdward Tomasz Napierala * Don't always wait for the full data segment to be delivered 437321b17ecSEdward Tomasz Napierala * to the socket; this might badly affect performance due to 438321b17ecSEdward Tomasz Napierala * TCP window scaling. 439321b17ecSEdward Tomasz Napierala */ 440321b17ecSEdward Tomasz Napierala if (len > partial_receive_len) { 441321b17ecSEdward Tomasz Napierala #if 0 442321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data, limiting to %zd", 443321b17ecSEdward Tomasz Napierala len, partial_receive_len)); 444321b17ecSEdward Tomasz Napierala #endif 445321b17ecSEdward Tomasz Napierala len = partial_receive_len; 446321b17ecSEdward Tomasz Napierala 447321b17ecSEdward Tomasz Napierala return (len); 448321b17ecSEdward Tomasz Napierala } 449321b17ecSEdward Tomasz Napierala 450321b17ecSEdward Tomasz Napierala /* 451321b17ecSEdward Tomasz Napierala * Account for padding. Note that due to the way code is written, 452321b17ecSEdward Tomasz Napierala * the icl_pdu_receive_data_segment() must always receive padding 453321b17ecSEdward Tomasz Napierala * along with the last part of data segment, because it would be 454321b17ecSEdward Tomasz Napierala * impossible to tell whether we've already received the full data 455321b17ecSEdward Tomasz Napierala * segment including padding, or without it. 456321b17ecSEdward Tomasz Napierala */ 457321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 458321b17ecSEdward Tomasz Napierala len += 4 - (len % 4); 459321b17ecSEdward Tomasz Napierala 460321b17ecSEdward Tomasz Napierala #if 0 461321b17ecSEdward Tomasz Napierala ICL_DEBUG("need %zd bytes of data", len)); 462321b17ecSEdward Tomasz Napierala #endif 463321b17ecSEdward Tomasz Napierala 464321b17ecSEdward Tomasz Napierala return (len); 465321b17ecSEdward Tomasz Napierala } 466321b17ecSEdward Tomasz Napierala 467321b17ecSEdward Tomasz Napierala static int 4686895f89fSAlexander Motin icl_pdu_receive_data_segment(struct icl_pdu *request, struct mbuf **r, 4696895f89fSAlexander Motin size_t *rs, bool *more_neededp) 470321b17ecSEdward Tomasz Napierala { 47187322a90SJohn Baldwin struct icl_soft_conn *isc; 472321b17ecSEdward Tomasz Napierala size_t len, padding = 0; 473321b17ecSEdward Tomasz Napierala struct mbuf *m; 474321b17ecSEdward Tomasz Napierala 47587322a90SJohn Baldwin isc = (struct icl_soft_conn *)request->ip_conn; 476321b17ecSEdward Tomasz Napierala 477321b17ecSEdward Tomasz Napierala *more_neededp = false; 47887322a90SJohn Baldwin isc->receive_len = 0; 479321b17ecSEdward Tomasz Napierala 480321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 481321b17ecSEdward Tomasz Napierala if (len == 0) 482321b17ecSEdward Tomasz Napierala return (0); 483321b17ecSEdward Tomasz Napierala 484321b17ecSEdward Tomasz Napierala if ((len % 4) != 0) 485321b17ecSEdward Tomasz Napierala padding = 4 - (len % 4); 486321b17ecSEdward Tomasz Napierala 487321b17ecSEdward Tomasz Napierala /* 488321b17ecSEdward Tomasz Napierala * Account for already received parts of data segment. 489321b17ecSEdward Tomasz Napierala */ 490321b17ecSEdward Tomasz Napierala KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len")); 491321b17ecSEdward Tomasz Napierala len -= request->ip_data_len; 492321b17ecSEdward Tomasz Napierala 4936895f89fSAlexander Motin if (len + padding > *rs) { 494321b17ecSEdward Tomasz Napierala /* 495321b17ecSEdward Tomasz Napierala * Not enough data in the socket buffer. Receive as much 496321b17ecSEdward Tomasz Napierala * as we can. Don't receive padding, since, obviously, it's 497321b17ecSEdward Tomasz Napierala * not the end of data segment yet. 498321b17ecSEdward Tomasz Napierala */ 499321b17ecSEdward Tomasz Napierala #if 0 500321b17ecSEdward Tomasz Napierala ICL_DEBUG("limited from %zd to %zd", 5016895f89fSAlexander Motin len + padding, *rs - padding)); 502321b17ecSEdward Tomasz Napierala #endif 5036895f89fSAlexander Motin len = *rs - padding; 504321b17ecSEdward Tomasz Napierala *more_neededp = true; 505321b17ecSEdward Tomasz Napierala padding = 0; 506321b17ecSEdward Tomasz Napierala } 507321b17ecSEdward Tomasz Napierala 508321b17ecSEdward Tomasz Napierala /* 509321b17ecSEdward Tomasz Napierala * Must not try to receive padding without at least one byte 510321b17ecSEdward Tomasz Napierala * of actual data segment. 511321b17ecSEdward Tomasz Napierala */ 512321b17ecSEdward Tomasz Napierala if (len > 0) { 5136895f89fSAlexander Motin m = *r; 5146895f89fSAlexander Motin *r = m_split(m, len + padding, M_WAITOK); 5156895f89fSAlexander Motin *rs -= len + padding; 516321b17ecSEdward Tomasz Napierala 517321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) 518321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = m; 519321b17ecSEdward Tomasz Napierala else 520321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, m); 521321b17ecSEdward Tomasz Napierala 522321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 523321b17ecSEdward Tomasz Napierala } else 524321b17ecSEdward Tomasz Napierala ICL_DEBUG("len 0"); 525321b17ecSEdward Tomasz Napierala 526321b17ecSEdward Tomasz Napierala if (*more_neededp) 52787322a90SJohn Baldwin isc->receive_len = icl_pdu_data_segment_receive_len(request); 528321b17ecSEdward Tomasz Napierala 529321b17ecSEdward Tomasz Napierala return (0); 530321b17ecSEdward Tomasz Napierala } 531321b17ecSEdward Tomasz Napierala 532321b17ecSEdward Tomasz Napierala static int 5336895f89fSAlexander Motin icl_pdu_check_data_digest(struct icl_pdu *request, struct mbuf **r, size_t *rs) 534321b17ecSEdward Tomasz Napierala { 535321b17ecSEdward Tomasz Napierala uint32_t received_digest, valid_digest; 536321b17ecSEdward Tomasz Napierala 537321b17ecSEdward Tomasz Napierala if (request->ip_conn->ic_data_crc32c == false) 538321b17ecSEdward Tomasz Napierala return (0); 539321b17ecSEdward Tomasz Napierala 540321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0) 541321b17ecSEdward Tomasz Napierala return (0); 542321b17ecSEdward Tomasz Napierala 543d0d587c7SAlexander Motin CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE); 5446895f89fSAlexander Motin icl_soft_receive_buf(r, rs, &received_digest, ISCSI_DATA_DIGEST_SIZE); 545321b17ecSEdward Tomasz Napierala 546321b17ecSEdward Tomasz Napierala /* 547321b17ecSEdward Tomasz Napierala * Note that ip_data_mbuf also contains padding; since digest 548321b17ecSEdward Tomasz Napierala * calculation is supposed to include that, we iterate over 549321b17ecSEdward Tomasz Napierala * the entire ip_data_mbuf chain, not just ip_data_len bytes of it. 550321b17ecSEdward Tomasz Napierala */ 551530e725dSJohn Baldwin valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf, 552530e725dSJohn Baldwin roundup2(request->ip_data_len, 4)); 553321b17ecSEdward Tomasz Napierala if (received_digest != valid_digest) { 554321b17ecSEdward Tomasz Napierala ICL_WARN("data digest check failed; got 0x%x, " 555321b17ecSEdward Tomasz Napierala "should be 0x%x", received_digest, valid_digest); 556321b17ecSEdward Tomasz Napierala return (-1); 557321b17ecSEdward Tomasz Napierala } 558321b17ecSEdward Tomasz Napierala 559321b17ecSEdward Tomasz Napierala return (0); 560321b17ecSEdward Tomasz Napierala } 561321b17ecSEdward Tomasz Napierala 562321b17ecSEdward Tomasz Napierala /* 563321b17ecSEdward Tomasz Napierala * Somewhat contrary to the name, this attempts to receive only one 564321b17ecSEdward Tomasz Napierala * "part" of PDU at a time; call it repeatedly until it returns non-NULL. 565321b17ecSEdward Tomasz Napierala */ 566321b17ecSEdward Tomasz Napierala static struct icl_pdu * 56787322a90SJohn Baldwin icl_conn_receive_pdu(struct icl_soft_conn *isc, struct mbuf **r, size_t *rs) 568321b17ecSEdward Tomasz Napierala { 56987322a90SJohn Baldwin struct icl_conn *ic = &isc->ic; 570321b17ecSEdward Tomasz Napierala struct icl_pdu *request; 571321b17ecSEdward Tomasz Napierala size_t len; 5726895f89fSAlexander Motin int error = 0; 573321b17ecSEdward Tomasz Napierala bool more_needed; 574321b17ecSEdward Tomasz Napierala 57587322a90SJohn Baldwin if (isc->receive_state == ICL_CONN_STATE_BHS) { 57687322a90SJohn Baldwin KASSERT(isc->receive_pdu == NULL, 57787322a90SJohn Baldwin ("isc->receive_pdu != NULL")); 578d0d587c7SAlexander Motin request = icl_soft_conn_new_pdu(ic, M_NOWAIT); 579321b17ecSEdward Tomasz Napierala if (request == NULL) { 580321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to allocate PDU; " 581321b17ecSEdward Tomasz Napierala "dropping connection"); 582321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 583321b17ecSEdward Tomasz Napierala return (NULL); 584321b17ecSEdward Tomasz Napierala } 58587322a90SJohn Baldwin isc->receive_pdu = request; 586321b17ecSEdward Tomasz Napierala } else { 58787322a90SJohn Baldwin KASSERT(isc->receive_pdu != NULL, 58887322a90SJohn Baldwin ("isc->receive_pdu == NULL")); 58987322a90SJohn Baldwin request = isc->receive_pdu; 590321b17ecSEdward Tomasz Napierala } 591321b17ecSEdward Tomasz Napierala 59287322a90SJohn Baldwin switch (isc->receive_state) { 593321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_BHS: 594321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving BHS"); 5956895f89fSAlexander Motin icl_soft_receive_buf(r, rs, request->ip_bhs, 5966895f89fSAlexander Motin sizeof(struct iscsi_bhs)); 597321b17ecSEdward Tomasz Napierala 598321b17ecSEdward Tomasz Napierala /* 599321b17ecSEdward Tomasz Napierala * We don't enforce any limit for AHS length; 600321b17ecSEdward Tomasz Napierala * its length is stored in 8 bit field. 601321b17ecSEdward Tomasz Napierala */ 602321b17ecSEdward Tomasz Napierala 603321b17ecSEdward Tomasz Napierala len = icl_pdu_data_segment_length(request); 6040cc7d64aSJohn Baldwin if (len > ic->ic_max_recv_data_segment_length) { 605321b17ecSEdward Tomasz Napierala ICL_WARN("received data segment " 606b75168edSAlexander Motin "length %zd is larger than negotiated; " 607b75168edSAlexander Motin "dropping connection", len); 608321b17ecSEdward Tomasz Napierala error = EINVAL; 609321b17ecSEdward Tomasz Napierala break; 610321b17ecSEdward Tomasz Napierala } 611321b17ecSEdward Tomasz Napierala 61287322a90SJohn Baldwin isc->receive_state = ICL_CONN_STATE_AHS; 61387322a90SJohn Baldwin isc->receive_len = icl_pdu_ahs_length(request); 614321b17ecSEdward Tomasz Napierala break; 615321b17ecSEdward Tomasz Napierala 616321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_AHS: 617321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving AHS"); 6186895f89fSAlexander Motin icl_pdu_receive_ahs(request, r, rs); 61987322a90SJohn Baldwin isc->receive_state = ICL_CONN_STATE_HEADER_DIGEST; 620321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c == false) 62187322a90SJohn Baldwin isc->receive_len = 0; 622321b17ecSEdward Tomasz Napierala else 62387322a90SJohn Baldwin isc->receive_len = ISCSI_HEADER_DIGEST_SIZE; 624321b17ecSEdward Tomasz Napierala break; 625321b17ecSEdward Tomasz Napierala 626321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_HEADER_DIGEST: 627321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving header digest"); 6286895f89fSAlexander Motin error = icl_pdu_check_header_digest(request, r, rs); 629321b17ecSEdward Tomasz Napierala if (error != 0) { 630321b17ecSEdward Tomasz Napierala ICL_DEBUG("header digest failed; " 631321b17ecSEdward Tomasz Napierala "dropping connection"); 632321b17ecSEdward Tomasz Napierala break; 633321b17ecSEdward Tomasz Napierala } 634321b17ecSEdward Tomasz Napierala 63587322a90SJohn Baldwin isc->receive_state = ICL_CONN_STATE_DATA; 63687322a90SJohn Baldwin isc->receive_len = icl_pdu_data_segment_receive_len(request); 637321b17ecSEdward Tomasz Napierala break; 638321b17ecSEdward Tomasz Napierala 639321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA: 640321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data segment"); 6416895f89fSAlexander Motin error = icl_pdu_receive_data_segment(request, r, rs, 642321b17ecSEdward Tomasz Napierala &more_needed); 643321b17ecSEdward Tomasz Napierala if (error != 0) { 644321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to receive data segment;" 645321b17ecSEdward Tomasz Napierala "dropping connection"); 646321b17ecSEdward Tomasz Napierala break; 647321b17ecSEdward Tomasz Napierala } 648321b17ecSEdward Tomasz Napierala 649321b17ecSEdward Tomasz Napierala if (more_needed) 650321b17ecSEdward Tomasz Napierala break; 651321b17ecSEdward Tomasz Napierala 65287322a90SJohn Baldwin isc->receive_state = ICL_CONN_STATE_DATA_DIGEST; 653321b17ecSEdward Tomasz Napierala if (request->ip_data_len == 0 || ic->ic_data_crc32c == false) 65487322a90SJohn Baldwin isc->receive_len = 0; 655321b17ecSEdward Tomasz Napierala else 65687322a90SJohn Baldwin isc->receive_len = ISCSI_DATA_DIGEST_SIZE; 657321b17ecSEdward Tomasz Napierala break; 658321b17ecSEdward Tomasz Napierala 659321b17ecSEdward Tomasz Napierala case ICL_CONN_STATE_DATA_DIGEST: 660321b17ecSEdward Tomasz Napierala //ICL_DEBUG("receiving data digest"); 6616895f89fSAlexander Motin error = icl_pdu_check_data_digest(request, r, rs); 662321b17ecSEdward Tomasz Napierala if (error != 0) { 663321b17ecSEdward Tomasz Napierala ICL_DEBUG("data digest failed; " 664321b17ecSEdward Tomasz Napierala "dropping connection"); 665321b17ecSEdward Tomasz Napierala break; 666321b17ecSEdward Tomasz Napierala } 667321b17ecSEdward Tomasz Napierala 668321b17ecSEdward Tomasz Napierala /* 669321b17ecSEdward Tomasz Napierala * We've received complete PDU; reset the receive state machine 670321b17ecSEdward Tomasz Napierala * and return the PDU. 671321b17ecSEdward Tomasz Napierala */ 67287322a90SJohn Baldwin isc->receive_state = ICL_CONN_STATE_BHS; 67387322a90SJohn Baldwin isc->receive_len = sizeof(struct iscsi_bhs); 67487322a90SJohn Baldwin isc->receive_pdu = NULL; 675321b17ecSEdward Tomasz Napierala return (request); 676321b17ecSEdward Tomasz Napierala 677321b17ecSEdward Tomasz Napierala default: 67887322a90SJohn Baldwin panic("invalid receive_state %d\n", isc->receive_state); 679321b17ecSEdward Tomasz Napierala } 680321b17ecSEdward Tomasz Napierala 681321b17ecSEdward Tomasz Napierala if (error != 0) { 682321b17ecSEdward Tomasz Napierala /* 68387322a90SJohn Baldwin * Don't free the PDU; it's pointed to by isc->receive_pdu 6845aabcd7cSEdward Tomasz Napierala * and will get freed in icl_soft_conn_close(). 685321b17ecSEdward Tomasz Napierala */ 686321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 687321b17ecSEdward Tomasz Napierala } 688321b17ecSEdward Tomasz Napierala 689321b17ecSEdward Tomasz Napierala return (NULL); 690321b17ecSEdward Tomasz Napierala } 691321b17ecSEdward Tomasz Napierala 692321b17ecSEdward Tomasz Napierala static void 69387322a90SJohn Baldwin icl_conn_receive_pdus(struct icl_soft_conn *isc, struct mbuf **r, size_t *rs) 694321b17ecSEdward Tomasz Napierala { 69587322a90SJohn Baldwin struct icl_conn *ic = &isc->ic; 696321b17ecSEdward Tomasz Napierala struct icl_pdu *response; 697321b17ecSEdward Tomasz Napierala 698321b17ecSEdward Tomasz Napierala for (;;) { 699321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) 700321b17ecSEdward Tomasz Napierala return; 701321b17ecSEdward Tomasz Napierala 702321b17ecSEdward Tomasz Napierala /* 703321b17ecSEdward Tomasz Napierala * Loop until we have a complete PDU or there is not enough 704321b17ecSEdward Tomasz Napierala * data in the socket buffer. 705321b17ecSEdward Tomasz Napierala */ 70687322a90SJohn Baldwin if (*rs < isc->receive_len) { 707321b17ecSEdward Tomasz Napierala #if 0 7086895f89fSAlexander Motin ICL_DEBUG("not enough data; have %zd, need %zd", 70987322a90SJohn Baldwin *rs, isc->receive_len); 710321b17ecSEdward Tomasz Napierala #endif 711321b17ecSEdward Tomasz Napierala return; 712321b17ecSEdward Tomasz Napierala } 713321b17ecSEdward Tomasz Napierala 71487322a90SJohn Baldwin response = icl_conn_receive_pdu(isc, r, rs); 715321b17ecSEdward Tomasz Napierala if (response == NULL) 716321b17ecSEdward Tomasz Napierala continue; 717321b17ecSEdward Tomasz Napierala 718321b17ecSEdward Tomasz Napierala if (response->ip_ahs_len > 0) { 719321b17ecSEdward Tomasz Napierala ICL_WARN("received PDU with unsupported " 720321b17ecSEdward Tomasz Napierala "AHS; opcode 0x%x; dropping connection", 721321b17ecSEdward Tomasz Napierala response->ip_bhs->bhs_opcode); 72282f7fa7aSAlexander Motin icl_soft_conn_pdu_free(ic, response); 723321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 724321b17ecSEdward Tomasz Napierala return; 725321b17ecSEdward Tomasz Napierala } 726321b17ecSEdward Tomasz Napierala 727321b17ecSEdward Tomasz Napierala (ic->ic_receive)(response); 728321b17ecSEdward Tomasz Napierala } 729321b17ecSEdward Tomasz Napierala } 730321b17ecSEdward Tomasz Napierala 731321b17ecSEdward Tomasz Napierala static void 732321b17ecSEdward Tomasz Napierala icl_receive_thread(void *arg) 733321b17ecSEdward Tomasz Napierala { 73487322a90SJohn Baldwin struct icl_soft_conn *isc = arg; 73587322a90SJohn Baldwin struct icl_conn *ic = &isc->ic; 7366895f89fSAlexander Motin size_t available, read = 0; 737321b17ecSEdward Tomasz Napierala struct socket *so; 7386895f89fSAlexander Motin struct mbuf *m, *r = NULL; 7396895f89fSAlexander Motin struct uio uio; 7406895f89fSAlexander Motin int error, flags; 741321b17ecSEdward Tomasz Napierala 742321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 743321b17ecSEdward Tomasz Napierala 744321b17ecSEdward Tomasz Napierala for (;;) { 7456895f89fSAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 746321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 7476895f89fSAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 748321b17ecSEdward Tomasz Napierala break; 749321b17ecSEdward Tomasz Napierala } 750321b17ecSEdward Tomasz Napierala 751321b17ecSEdward Tomasz Napierala /* 752321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 753321b17ecSEdward Tomasz Napierala * soreadable() in icl_soupcall_receive() 754266078c6SPedro F. Giffuni * to avoid unnecessary wakeups until there 755321b17ecSEdward Tomasz Napierala * is enough data received to read the PDU. 756321b17ecSEdward Tomasz Napierala */ 757321b17ecSEdward Tomasz Napierala available = sbavail(&so->so_rcv); 75887322a90SJohn Baldwin if (read + available < isc->receive_len) { 75987322a90SJohn Baldwin so->so_rcv.sb_lowat = isc->receive_len - read; 76087322a90SJohn Baldwin cv_wait(&isc->receive_cv, SOCKBUF_MTX(&so->so_rcv)); 761321b17ecSEdward Tomasz Napierala so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1; 7626895f89fSAlexander Motin available = sbavail(&so->so_rcv); 7636895f89fSAlexander Motin } 764321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_rcv); 765321b17ecSEdward Tomasz Napierala 7666895f89fSAlexander Motin if (available == 0) { 7676895f89fSAlexander Motin if (so->so_error != 0) { 7686895f89fSAlexander Motin ICL_DEBUG("connection error %d; " 7696895f89fSAlexander Motin "dropping connection", so->so_error); 7706895f89fSAlexander Motin icl_conn_fail(ic); 7716895f89fSAlexander Motin break; 772321b17ecSEdward Tomasz Napierala } 7736895f89fSAlexander Motin continue; 7746895f89fSAlexander Motin } 7756895f89fSAlexander Motin 7766895f89fSAlexander Motin memset(&uio, 0, sizeof(uio)); 7776895f89fSAlexander Motin uio.uio_resid = available; 7786895f89fSAlexander Motin flags = MSG_DONTWAIT; 7796895f89fSAlexander Motin error = soreceive(so, NULL, &uio, &m, NULL, &flags); 7806895f89fSAlexander Motin if (error != 0) { 7816895f89fSAlexander Motin ICL_DEBUG("soreceive error %d", error); 7826895f89fSAlexander Motin break; 7836895f89fSAlexander Motin } 7846895f89fSAlexander Motin if (uio.uio_resid != 0) { 7856895f89fSAlexander Motin m_freem(m); 7866895f89fSAlexander Motin ICL_DEBUG("short read"); 7876895f89fSAlexander Motin break; 7886895f89fSAlexander Motin } 7896895f89fSAlexander Motin if (r) 7906895f89fSAlexander Motin m_cat(r, m); 7916895f89fSAlexander Motin else 7926895f89fSAlexander Motin r = m; 7936895f89fSAlexander Motin read += available; 7946895f89fSAlexander Motin 79587322a90SJohn Baldwin icl_conn_receive_pdus(isc, &r, &read); 7966895f89fSAlexander Motin } 7976895f89fSAlexander Motin 7986895f89fSAlexander Motin if (r) 7996895f89fSAlexander Motin m_freem(r); 800321b17ecSEdward Tomasz Napierala 801321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 80287322a90SJohn Baldwin isc->receive_running = false; 80387322a90SJohn Baldwin cv_signal(&isc->send_cv); 804321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 805321b17ecSEdward Tomasz Napierala kthread_exit(); 806321b17ecSEdward Tomasz Napierala } 807321b17ecSEdward Tomasz Napierala 808321b17ecSEdward Tomasz Napierala static int 809321b17ecSEdward Tomasz Napierala icl_soupcall_receive(struct socket *so, void *arg, int waitflag) 810321b17ecSEdward Tomasz Napierala { 81187322a90SJohn Baldwin struct icl_soft_conn *isc; 812321b17ecSEdward Tomasz Napierala 813321b17ecSEdward Tomasz Napierala if (!soreadable(so)) 814321b17ecSEdward Tomasz Napierala return (SU_OK); 815321b17ecSEdward Tomasz Napierala 81687322a90SJohn Baldwin isc = arg; 81787322a90SJohn Baldwin cv_signal(&isc->receive_cv); 818321b17ecSEdward Tomasz Napierala return (SU_OK); 819321b17ecSEdward Tomasz Napierala } 820321b17ecSEdward Tomasz Napierala 821321b17ecSEdward Tomasz Napierala static int 822321b17ecSEdward Tomasz Napierala icl_pdu_finalize(struct icl_pdu *request) 823321b17ecSEdward Tomasz Napierala { 824321b17ecSEdward Tomasz Napierala size_t padding, pdu_len; 825321b17ecSEdward Tomasz Napierala uint32_t digest, zero = 0; 826321b17ecSEdward Tomasz Napierala int ok; 827321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 828321b17ecSEdward Tomasz Napierala 829321b17ecSEdward Tomasz Napierala ic = request->ip_conn; 830321b17ecSEdward Tomasz Napierala 831321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(request, request->ip_data_len); 832321b17ecSEdward Tomasz Napierala 833321b17ecSEdward Tomasz Napierala pdu_len = icl_pdu_size(request); 834321b17ecSEdward Tomasz Napierala 835321b17ecSEdward Tomasz Napierala if (ic->ic_header_crc32c) { 836530e725dSJohn Baldwin digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf, 837530e725dSJohn Baldwin ISCSI_BHS_SIZE); 838321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_bhs_mbuf, sizeof(digest), 839321b17ecSEdward Tomasz Napierala (void *)&digest); 840321b17ecSEdward Tomasz Napierala if (ok != 1) { 841321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append header digest"); 842321b17ecSEdward Tomasz Napierala return (1); 843321b17ecSEdward Tomasz Napierala } 844321b17ecSEdward Tomasz Napierala } 845321b17ecSEdward Tomasz Napierala 846321b17ecSEdward Tomasz Napierala if (request->ip_data_len != 0) { 847321b17ecSEdward Tomasz Napierala padding = icl_pdu_padding(request); 848321b17ecSEdward Tomasz Napierala if (padding > 0) { 849321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, padding, 850321b17ecSEdward Tomasz Napierala (void *)&zero); 851321b17ecSEdward Tomasz Napierala if (ok != 1) { 852321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append padding"); 853321b17ecSEdward Tomasz Napierala return (1); 854321b17ecSEdward Tomasz Napierala } 855321b17ecSEdward Tomasz Napierala } 856321b17ecSEdward Tomasz Napierala 857321b17ecSEdward Tomasz Napierala if (ic->ic_data_crc32c) { 858530e725dSJohn Baldwin digest = icl_mbuf_to_crc32c(request->ip_data_mbuf, 859530e725dSJohn Baldwin roundup2(request->ip_data_len, 4)); 860321b17ecSEdward Tomasz Napierala 861321b17ecSEdward Tomasz Napierala ok = m_append(request->ip_data_mbuf, sizeof(digest), 862321b17ecSEdward Tomasz Napierala (void *)&digest); 863321b17ecSEdward Tomasz Napierala if (ok != 1) { 864321b17ecSEdward Tomasz Napierala ICL_WARN("failed to append data digest"); 865321b17ecSEdward Tomasz Napierala return (1); 866321b17ecSEdward Tomasz Napierala } 867321b17ecSEdward Tomasz Napierala } 868321b17ecSEdward Tomasz Napierala 869321b17ecSEdward Tomasz Napierala m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf); 870321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = NULL; 871321b17ecSEdward Tomasz Napierala } 872321b17ecSEdward Tomasz Napierala 873321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len = pdu_len; 874321b17ecSEdward Tomasz Napierala 875321b17ecSEdward Tomasz Napierala return (0); 876321b17ecSEdward Tomasz Napierala } 877321b17ecSEdward Tomasz Napierala 878321b17ecSEdward Tomasz Napierala static void 87987322a90SJohn Baldwin icl_conn_send_pdus(struct icl_soft_conn *isc, struct icl_pdu_stailq *queue) 880321b17ecSEdward Tomasz Napierala { 88187322a90SJohn Baldwin struct icl_conn *ic = &isc->ic; 882321b17ecSEdward Tomasz Napierala struct icl_pdu *request, *request2; 883b85a67f5SAlexander Motin struct mbuf *m; 884321b17ecSEdward Tomasz Napierala struct socket *so; 885605703b5SAlexander Motin long available, size, size2; 886*f4f84701SDimitry Andric #ifdef DEBUG_COALESCED 887*f4f84701SDimitry Andric int coalesced; 888*f4f84701SDimitry Andric #endif 889*f4f84701SDimitry Andric int error; 890321b17ecSEdward Tomasz Napierala 891321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 892321b17ecSEdward Tomasz Napierala 893321b17ecSEdward Tomasz Napierala so = ic->ic_socket; 894321b17ecSEdward Tomasz Napierala 895321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 896321b17ecSEdward Tomasz Napierala /* 897321b17ecSEdward Tomasz Napierala * Check how much space do we have for transmit. We can't just 898321b17ecSEdward Tomasz Napierala * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE, 899321b17ecSEdward Tomasz Napierala * as it always frees the mbuf chain passed to it, even in case 900321b17ecSEdward Tomasz Napierala * of error. 901321b17ecSEdward Tomasz Napierala */ 902321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 90387322a90SJohn Baldwin isc->check_send_space = false; 904321b17ecSEdward Tomasz Napierala 905321b17ecSEdward Tomasz Napierala /* 906321b17ecSEdward Tomasz Napierala * Notify the socket upcall that we don't need wakeups 907321b17ecSEdward Tomasz Napierala * for the time being. 908321b17ecSEdward Tomasz Napierala */ 909321b17ecSEdward Tomasz Napierala so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1; 910321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 911321b17ecSEdward Tomasz Napierala 912321b17ecSEdward Tomasz Napierala while (!STAILQ_EMPTY(queue)) { 913321b17ecSEdward Tomasz Napierala request = STAILQ_FIRST(queue); 914321b17ecSEdward Tomasz Napierala size = icl_pdu_size(request); 915321b17ecSEdward Tomasz Napierala if (available < size) { 916321b17ecSEdward Tomasz Napierala /* 917321b17ecSEdward Tomasz Napierala * Set the low watermark, to be checked by 918321b17ecSEdward Tomasz Napierala * sowriteable() in icl_soupcall_send() 919266078c6SPedro F. Giffuni * to avoid unnecessary wakeups until there 920321b17ecSEdward Tomasz Napierala * is enough space for the PDU to fit. 921321b17ecSEdward Tomasz Napierala */ 922321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&so->so_snd); 923321b17ecSEdward Tomasz Napierala available = sbspace(&so->so_snd); 924321b17ecSEdward Tomasz Napierala if (available < size) { 925321b17ecSEdward Tomasz Napierala #if 1 926321b17ecSEdward Tomasz Napierala ICL_DEBUG("no space to send; " 927605703b5SAlexander Motin "have %ld, need %ld", 928321b17ecSEdward Tomasz Napierala available, size); 929321b17ecSEdward Tomasz Napierala #endif 9301f29b46cSAlexander Motin so->so_snd.sb_lowat = max(size, 9311f29b46cSAlexander Motin so->so_snd.sb_hiwat / 8); 932321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 933321b17ecSEdward Tomasz Napierala return; 934321b17ecSEdward Tomasz Napierala } 935321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&so->so_snd); 936321b17ecSEdward Tomasz Napierala } 937321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 938321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request); 939321b17ecSEdward Tomasz Napierala if (error != 0) { 940321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 941321b17ecSEdward Tomasz Napierala "dropping connection"); 9429a4510acSAlexander Motin icl_soft_pdu_done(request, EIO); 943321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 944321b17ecSEdward Tomasz Napierala return; 945321b17ecSEdward Tomasz Napierala } 946321b17ecSEdward Tomasz Napierala if (coalesce) { 947b85a67f5SAlexander Motin m = request->ip_bhs_mbuf; 948*f4f84701SDimitry Andric for ( 949*f4f84701SDimitry Andric #ifdef DEBUG_COALESCED 950*f4f84701SDimitry Andric coalesced = 1 951*f4f84701SDimitry Andric #endif 952*f4f84701SDimitry Andric ; ; 953*f4f84701SDimitry Andric #ifdef DEBUG_COALESCED 954*f4f84701SDimitry Andric coalesced++ 955*f4f84701SDimitry Andric #endif 956*f4f84701SDimitry Andric ) { 957321b17ecSEdward Tomasz Napierala request2 = STAILQ_FIRST(queue); 958321b17ecSEdward Tomasz Napierala if (request2 == NULL) 959321b17ecSEdward Tomasz Napierala break; 960321b17ecSEdward Tomasz Napierala size2 = icl_pdu_size(request2); 961321b17ecSEdward Tomasz Napierala if (available < size + size2) 962321b17ecSEdward Tomasz Napierala break; 963321b17ecSEdward Tomasz Napierala STAILQ_REMOVE_HEAD(queue, ip_next); 964321b17ecSEdward Tomasz Napierala error = icl_pdu_finalize(request2); 965321b17ecSEdward Tomasz Napierala if (error != 0) { 966321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to finalize PDU; " 967321b17ecSEdward Tomasz Napierala "dropping connection"); 9689a4510acSAlexander Motin icl_soft_pdu_done(request, EIO); 9699a4510acSAlexander Motin icl_soft_pdu_done(request2, EIO); 970321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 971321b17ecSEdward Tomasz Napierala return; 972321b17ecSEdward Tomasz Napierala } 973b85a67f5SAlexander Motin while (m->m_next) 974b85a67f5SAlexander Motin m = m->m_next; 975b85a67f5SAlexander Motin m_cat(m, request2->ip_bhs_mbuf); 976321b17ecSEdward Tomasz Napierala request2->ip_bhs_mbuf = NULL; 977321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf->m_pkthdr.len += size2; 978321b17ecSEdward Tomasz Napierala size += size2; 9799a4510acSAlexander Motin icl_soft_pdu_done(request2, 0); 980321b17ecSEdward Tomasz Napierala } 981*f4f84701SDimitry Andric #ifdef DEBUG_COALESCED 982321b17ecSEdward Tomasz Napierala if (coalesced > 1) { 983605703b5SAlexander Motin ICL_DEBUG("coalesced %d PDUs into %ld bytes", 984321b17ecSEdward Tomasz Napierala coalesced, size); 985321b17ecSEdward Tomasz Napierala } 986321b17ecSEdward Tomasz Napierala #endif 987321b17ecSEdward Tomasz Napierala } 988321b17ecSEdward Tomasz Napierala available -= size; 989321b17ecSEdward Tomasz Napierala error = sosend(so, NULL, NULL, request->ip_bhs_mbuf, 990321b17ecSEdward Tomasz Napierala NULL, MSG_DONTWAIT, curthread); 991321b17ecSEdward Tomasz Napierala request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */ 992321b17ecSEdward Tomasz Napierala if (error != 0) { 993321b17ecSEdward Tomasz Napierala ICL_DEBUG("failed to send PDU, error %d; " 994321b17ecSEdward Tomasz Napierala "dropping connection", error); 9959a4510acSAlexander Motin icl_soft_pdu_done(request, error); 996321b17ecSEdward Tomasz Napierala icl_conn_fail(ic); 997321b17ecSEdward Tomasz Napierala return; 998321b17ecSEdward Tomasz Napierala } 9999a4510acSAlexander Motin icl_soft_pdu_done(request, 0); 1000321b17ecSEdward Tomasz Napierala } 1001321b17ecSEdward Tomasz Napierala } 1002321b17ecSEdward Tomasz Napierala 1003321b17ecSEdward Tomasz Napierala static void 1004321b17ecSEdward Tomasz Napierala icl_send_thread(void *arg) 1005321b17ecSEdward Tomasz Napierala { 100687322a90SJohn Baldwin struct icl_soft_conn *isc; 1007321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1008321b17ecSEdward Tomasz Napierala struct icl_pdu_stailq queue; 1009321b17ecSEdward Tomasz Napierala 101087322a90SJohn Baldwin isc = arg; 101187322a90SJohn Baldwin ic = &isc->ic; 1012321b17ecSEdward Tomasz Napierala 1013321b17ecSEdward Tomasz Napierala STAILQ_INIT(&queue); 1014321b17ecSEdward Tomasz Napierala 1015321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1016321b17ecSEdward Tomasz Napierala for (;;) { 1017321b17ecSEdward Tomasz Napierala for (;;) { 1018321b17ecSEdward Tomasz Napierala /* 1019df3747c6SAlexander Motin * Populate the local queue from the main one. 1020df3747c6SAlexander Motin * This way the icl_conn_send_pdus() can go through 1021df3747c6SAlexander Motin * all the queued PDUs without holding any locks. 1022321b17ecSEdward Tomasz Napierala */ 102387322a90SJohn Baldwin if (STAILQ_EMPTY(&queue) || isc->check_send_space) 102487322a90SJohn Baldwin STAILQ_CONCAT(&queue, &isc->to_send); 1025321b17ecSEdward Tomasz Napierala 1026321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 102787322a90SJohn Baldwin icl_conn_send_pdus(isc, &queue); 1028321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1029321b17ecSEdward Tomasz Napierala 1030321b17ecSEdward Tomasz Napierala /* 1031321b17ecSEdward Tomasz Napierala * The icl_soupcall_send() was called since the last 1032321b17ecSEdward Tomasz Napierala * call to sbspace(); go around; 1033321b17ecSEdward Tomasz Napierala */ 103487322a90SJohn Baldwin if (isc->check_send_space) 1035321b17ecSEdward Tomasz Napierala continue; 1036321b17ecSEdward Tomasz Napierala 1037321b17ecSEdward Tomasz Napierala /* 1038321b17ecSEdward Tomasz Napierala * Local queue is empty, but we still have PDUs 1039321b17ecSEdward Tomasz Napierala * in the main one; go around. 1040321b17ecSEdward Tomasz Napierala */ 1041321b17ecSEdward Tomasz Napierala if (STAILQ_EMPTY(&queue) && 104287322a90SJohn Baldwin !STAILQ_EMPTY(&isc->to_send)) 1043321b17ecSEdward Tomasz Napierala continue; 1044321b17ecSEdward Tomasz Napierala 1045321b17ecSEdward Tomasz Napierala /* 1046321b17ecSEdward Tomasz Napierala * There might be some stuff in the local queue, 1047321b17ecSEdward Tomasz Napierala * which didn't get sent due to not having enough send 1048321b17ecSEdward Tomasz Napierala * space. Wait for socket upcall. 1049321b17ecSEdward Tomasz Napierala */ 1050321b17ecSEdward Tomasz Napierala break; 1051321b17ecSEdward Tomasz Napierala } 1052321b17ecSEdward Tomasz Napierala 1053321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting) { 1054321b17ecSEdward Tomasz Napierala //ICL_DEBUG("terminating"); 1055321b17ecSEdward Tomasz Napierala break; 1056321b17ecSEdward Tomasz Napierala } 1057321b17ecSEdward Tomasz Napierala 105887322a90SJohn Baldwin cv_wait(&isc->send_cv, ic->ic_lock); 1059321b17ecSEdward Tomasz Napierala } 1060321b17ecSEdward Tomasz Napierala 1061321b17ecSEdward Tomasz Napierala /* 1062321b17ecSEdward Tomasz Napierala * We're exiting; move PDUs back to the main queue, so they can 1063321b17ecSEdward Tomasz Napierala * get freed properly. At this point ordering doesn't matter. 1064321b17ecSEdward Tomasz Napierala */ 106587322a90SJohn Baldwin STAILQ_CONCAT(&isc->to_send, &queue); 1066321b17ecSEdward Tomasz Napierala 106787322a90SJohn Baldwin isc->send_running = false; 106887322a90SJohn Baldwin cv_signal(&isc->send_cv); 1069321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1070321b17ecSEdward Tomasz Napierala kthread_exit(); 1071321b17ecSEdward Tomasz Napierala } 1072321b17ecSEdward Tomasz Napierala 1073321b17ecSEdward Tomasz Napierala static int 1074321b17ecSEdward Tomasz Napierala icl_soupcall_send(struct socket *so, void *arg, int waitflag) 1075321b17ecSEdward Tomasz Napierala { 107687322a90SJohn Baldwin struct icl_soft_conn *isc; 1077321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1078321b17ecSEdward Tomasz Napierala 1079321b17ecSEdward Tomasz Napierala if (!sowriteable(so)) 1080321b17ecSEdward Tomasz Napierala return (SU_OK); 1081321b17ecSEdward Tomasz Napierala 108287322a90SJohn Baldwin isc = arg; 108387322a90SJohn Baldwin ic = &isc->ic; 1084321b17ecSEdward Tomasz Napierala 1085321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 108687322a90SJohn Baldwin isc->check_send_space = true; 1087321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1088321b17ecSEdward Tomasz Napierala 108987322a90SJohn Baldwin cv_signal(&isc->send_cv); 1090321b17ecSEdward Tomasz Napierala 1091321b17ecSEdward Tomasz Napierala return (SU_OK); 1092321b17ecSEdward Tomasz Napierala } 1093321b17ecSEdward Tomasz Napierala 1094530e725dSJohn Baldwin static void 1095530e725dSJohn Baldwin icl_soft_free_mext_pg(struct mbuf *m) 1096530e725dSJohn Baldwin { 1097530e725dSJohn Baldwin struct icl_soft_pdu *isp; 1098530e725dSJohn Baldwin 1099530e725dSJohn Baldwin M_ASSERTEXTPG(m); 1100530e725dSJohn Baldwin 1101530e725dSJohn Baldwin /* 1102530e725dSJohn Baldwin * Nothing to do for the pages; they are owned by the PDU / 1103530e725dSJohn Baldwin * I/O request. 1104530e725dSJohn Baldwin */ 1105530e725dSJohn Baldwin 1106530e725dSJohn Baldwin /* Drop reference on the PDU. */ 1107530e725dSJohn Baldwin isp = m->m_ext.ext_arg1; 1108530e725dSJohn Baldwin if (atomic_fetchadd_int(&isp->ref_cnt, -1) == 1) 1109530e725dSJohn Baldwin icl_soft_pdu_call_cb(&isp->ip); 1110530e725dSJohn Baldwin } 1111530e725dSJohn Baldwin 1112530e725dSJohn Baldwin static int 1113530e725dSJohn Baldwin icl_soft_conn_pdu_append_bio(struct icl_conn *ic, struct icl_pdu *request, 1114530e725dSJohn Baldwin struct bio *bp, size_t offset, size_t len, int flags) 1115530e725dSJohn Baldwin { 1116530e725dSJohn Baldwin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)request; 1117530e725dSJohn Baldwin struct mbuf *m, *m_tail; 1118530e725dSJohn Baldwin vm_offset_t vaddr; 1119530e725dSJohn Baldwin size_t mtodo, page_offset, todo; 1120530e725dSJohn Baldwin int i; 1121530e725dSJohn Baldwin 1122530e725dSJohn Baldwin KASSERT(len > 0, ("len == 0")); 1123530e725dSJohn Baldwin 1124530e725dSJohn Baldwin m_tail = request->ip_data_mbuf; 1125530e725dSJohn Baldwin if (m_tail != NULL) 1126530e725dSJohn Baldwin for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) 1127530e725dSJohn Baldwin ; 1128530e725dSJohn Baldwin 1129530e725dSJohn Baldwin MPASS(bp->bio_flags & BIO_UNMAPPED); 1130530e725dSJohn Baldwin if (offset < PAGE_SIZE - bp->bio_ma_offset) { 1131530e725dSJohn Baldwin page_offset = bp->bio_ma_offset + offset; 1132530e725dSJohn Baldwin i = 0; 1133530e725dSJohn Baldwin } else { 1134530e725dSJohn Baldwin offset -= PAGE_SIZE - bp->bio_ma_offset; 1135530e725dSJohn Baldwin for (i = 1; offset >= PAGE_SIZE; i++) 1136530e725dSJohn Baldwin offset -= PAGE_SIZE; 1137530e725dSJohn Baldwin page_offset = offset; 1138530e725dSJohn Baldwin } 1139530e725dSJohn Baldwin 1140530e725dSJohn Baldwin if (flags & ICL_NOCOPY) { 1141530e725dSJohn Baldwin m = NULL; 1142530e725dSJohn Baldwin while (len > 0) { 1143530e725dSJohn Baldwin if (m == NULL) { 1144530e725dSJohn Baldwin m = mb_alloc_ext_pgs(flags & ~ICL_NOCOPY, 1145530e725dSJohn Baldwin icl_soft_free_mext_pg); 1146530e725dSJohn Baldwin if (__predict_false(m == NULL)) 1147530e725dSJohn Baldwin return (ENOMEM); 1148530e725dSJohn Baldwin atomic_add_int(&isp->ref_cnt, 1); 1149530e725dSJohn Baldwin m->m_ext.ext_arg1 = isp; 1150530e725dSJohn Baldwin m->m_epg_1st_off = page_offset; 1151530e725dSJohn Baldwin } 1152530e725dSJohn Baldwin 1153530e725dSJohn Baldwin todo = MIN(len, PAGE_SIZE - page_offset); 1154530e725dSJohn Baldwin 1155530e725dSJohn Baldwin m->m_epg_pa[m->m_epg_npgs] = 1156530e725dSJohn Baldwin VM_PAGE_TO_PHYS(bp->bio_ma[i]); 1157530e725dSJohn Baldwin m->m_epg_npgs++; 1158530e725dSJohn Baldwin m->m_epg_last_len = todo; 1159530e725dSJohn Baldwin m->m_len += todo; 1160530e725dSJohn Baldwin m->m_ext.ext_size += PAGE_SIZE; 1161530e725dSJohn Baldwin MBUF_EXT_PGS_ASSERT_SANITY(m); 1162530e725dSJohn Baldwin 1163530e725dSJohn Baldwin if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) { 1164530e725dSJohn Baldwin if (m_tail != NULL) 1165530e725dSJohn Baldwin m_tail->m_next = m; 1166530e725dSJohn Baldwin else 1167530e725dSJohn Baldwin request->ip_data_mbuf = m; 1168530e725dSJohn Baldwin m_tail = m; 1169530e725dSJohn Baldwin request->ip_data_len += m->m_len; 1170530e725dSJohn Baldwin m = NULL; 1171530e725dSJohn Baldwin } 1172530e725dSJohn Baldwin 1173530e725dSJohn Baldwin page_offset = 0; 1174530e725dSJohn Baldwin len -= todo; 1175530e725dSJohn Baldwin i++; 1176530e725dSJohn Baldwin } 1177530e725dSJohn Baldwin 1178530e725dSJohn Baldwin if (m != NULL) { 1179530e725dSJohn Baldwin if (m_tail != NULL) 1180530e725dSJohn Baldwin m_tail->m_next = m; 1181530e725dSJohn Baldwin else 1182530e725dSJohn Baldwin request->ip_data_mbuf = m; 1183530e725dSJohn Baldwin request->ip_data_len += m->m_len; 1184530e725dSJohn Baldwin } 1185530e725dSJohn Baldwin return (0); 1186530e725dSJohn Baldwin } 1187530e725dSJohn Baldwin 1188530e725dSJohn Baldwin m = m_getm2(NULL, len, flags, MT_DATA, 0); 1189530e725dSJohn Baldwin if (__predict_false(m == NULL)) 1190530e725dSJohn Baldwin return (ENOMEM); 1191530e725dSJohn Baldwin 1192530e725dSJohn Baldwin if (request->ip_data_mbuf == NULL) { 1193530e725dSJohn Baldwin request->ip_data_mbuf = m; 1194530e725dSJohn Baldwin request->ip_data_len = len; 1195530e725dSJohn Baldwin } else { 1196530e725dSJohn Baldwin m_tail->m_next = m; 1197530e725dSJohn Baldwin request->ip_data_len += len; 1198530e725dSJohn Baldwin } 1199530e725dSJohn Baldwin 1200530e725dSJohn Baldwin while (len > 0) { 1201530e725dSJohn Baldwin todo = MIN(len, PAGE_SIZE - page_offset); 1202832acea9SJohn Baldwin vaddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(bp->bio_ma[i])); 1203530e725dSJohn Baldwin 1204530e725dSJohn Baldwin do { 1205530e725dSJohn Baldwin mtodo = min(todo, M_SIZE(m) - m->m_len); 1206530e725dSJohn Baldwin memcpy(mtod(m, char *) + m->m_len, (char *)vaddr + 1207530e725dSJohn Baldwin page_offset, mtodo); 1208530e725dSJohn Baldwin m->m_len += mtodo; 1209530e725dSJohn Baldwin if (m->m_len == M_SIZE(m)) 1210530e725dSJohn Baldwin m = m->m_next; 1211530e725dSJohn Baldwin page_offset += mtodo; 1212530e725dSJohn Baldwin todo -= mtodo; 1213530e725dSJohn Baldwin } while (todo > 0); 1214530e725dSJohn Baldwin 1215530e725dSJohn Baldwin page_offset = 0; 1216530e725dSJohn Baldwin len -= todo; 1217530e725dSJohn Baldwin i++; 1218530e725dSJohn Baldwin } 1219530e725dSJohn Baldwin 1220530e725dSJohn Baldwin return (0); 1221530e725dSJohn Baldwin } 1222530e725dSJohn Baldwin 1223321b17ecSEdward Tomasz Napierala static int 122482f7fa7aSAlexander Motin icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request, 122582f7fa7aSAlexander Motin const void *addr, size_t len, int flags) 1226321b17ecSEdward Tomasz Napierala { 12279a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)request; 1228321b17ecSEdward Tomasz Napierala struct mbuf *mb, *newmb; 1229321b17ecSEdward Tomasz Napierala size_t copylen, off = 0; 1230321b17ecSEdward Tomasz Napierala 1231321b17ecSEdward Tomasz Napierala KASSERT(len > 0, ("len == 0")); 1232321b17ecSEdward Tomasz Napierala 12339a4510acSAlexander Motin if (flags & ICL_NOCOPY) { 12349a4510acSAlexander Motin newmb = m_get(flags & ~ICL_NOCOPY, MT_DATA); 12359a4510acSAlexander Motin if (newmb == NULL) { 12369a4510acSAlexander Motin ICL_WARN("failed to allocate mbuf"); 12379a4510acSAlexander Motin return (ENOMEM); 12389a4510acSAlexander Motin } 12399a4510acSAlexander Motin 12409a4510acSAlexander Motin newmb->m_flags |= M_RDONLY; 12419a4510acSAlexander Motin m_extaddref(newmb, __DECONST(char *, addr), len, &isp->ref_cnt, 12429a4510acSAlexander Motin icl_soft_mbuf_done, isp, NULL); 12439a4510acSAlexander Motin newmb->m_len = len; 12449a4510acSAlexander Motin } else { 1245898fd11fSAlexander Motin newmb = m_getm2(NULL, len, flags, MT_DATA, 0); 1246321b17ecSEdward Tomasz Napierala if (newmb == NULL) { 1247321b17ecSEdward Tomasz Napierala ICL_WARN("failed to allocate mbuf for %zd bytes", len); 1248321b17ecSEdward Tomasz Napierala return (ENOMEM); 1249321b17ecSEdward Tomasz Napierala } 1250321b17ecSEdward Tomasz Napierala 1251321b17ecSEdward Tomasz Napierala for (mb = newmb; mb != NULL; mb = mb->m_next) { 1252321b17ecSEdward Tomasz Napierala copylen = min(M_TRAILINGSPACE(mb), len - off); 1253321b17ecSEdward Tomasz Napierala memcpy(mtod(mb, char *), (const char *)addr + off, copylen); 1254321b17ecSEdward Tomasz Napierala mb->m_len = copylen; 1255321b17ecSEdward Tomasz Napierala off += copylen; 1256321b17ecSEdward Tomasz Napierala } 1257321b17ecSEdward Tomasz Napierala KASSERT(off == len, ("%s: off != len", __func__)); 12589a4510acSAlexander Motin } 1259321b17ecSEdward Tomasz Napierala 1260321b17ecSEdward Tomasz Napierala if (request->ip_data_mbuf == NULL) { 1261321b17ecSEdward Tomasz Napierala request->ip_data_mbuf = newmb; 1262321b17ecSEdward Tomasz Napierala request->ip_data_len = len; 1263321b17ecSEdward Tomasz Napierala } else { 1264321b17ecSEdward Tomasz Napierala m_cat(request->ip_data_mbuf, newmb); 1265321b17ecSEdward Tomasz Napierala request->ip_data_len += len; 1266321b17ecSEdward Tomasz Napierala } 1267321b17ecSEdward Tomasz Napierala 1268321b17ecSEdward Tomasz Napierala return (0); 1269321b17ecSEdward Tomasz Napierala } 1270321b17ecSEdward Tomasz Napierala 1271321b17ecSEdward Tomasz Napierala void 1272530e725dSJohn Baldwin icl_soft_conn_pdu_get_bio(struct icl_conn *ic, struct icl_pdu *ip, 1273530e725dSJohn Baldwin size_t pdu_off, struct bio *bp, size_t bio_off, size_t len) 1274530e725dSJohn Baldwin { 1275530e725dSJohn Baldwin vm_offset_t vaddr; 1276530e725dSJohn Baldwin size_t page_offset, todo; 1277530e725dSJohn Baldwin int i; 1278530e725dSJohn Baldwin 1279530e725dSJohn Baldwin MPASS(bp->bio_flags & BIO_UNMAPPED); 1280530e725dSJohn Baldwin if (bio_off < PAGE_SIZE - bp->bio_ma_offset) { 1281530e725dSJohn Baldwin page_offset = bp->bio_ma_offset + bio_off; 1282530e725dSJohn Baldwin i = 0; 1283530e725dSJohn Baldwin } else { 1284530e725dSJohn Baldwin bio_off -= PAGE_SIZE - bp->bio_ma_offset; 1285530e725dSJohn Baldwin for (i = 1; bio_off >= PAGE_SIZE; i++) 1286530e725dSJohn Baldwin bio_off -= PAGE_SIZE; 1287530e725dSJohn Baldwin page_offset = bio_off; 1288530e725dSJohn Baldwin } 1289530e725dSJohn Baldwin 1290530e725dSJohn Baldwin while (len > 0) { 1291530e725dSJohn Baldwin todo = MIN(len, PAGE_SIZE - page_offset); 1292530e725dSJohn Baldwin 1293832acea9SJohn Baldwin vaddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(bp->bio_ma[i])); 1294530e725dSJohn Baldwin m_copydata(ip->ip_data_mbuf, pdu_off, todo, (char *)vaddr + 1295530e725dSJohn Baldwin page_offset); 1296530e725dSJohn Baldwin 1297530e725dSJohn Baldwin page_offset = 0; 1298530e725dSJohn Baldwin pdu_off += todo; 1299530e725dSJohn Baldwin len -= todo; 1300530e725dSJohn Baldwin i++; 1301530e725dSJohn Baldwin } 1302530e725dSJohn Baldwin } 1303530e725dSJohn Baldwin 1304530e725dSJohn Baldwin void 1305321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 1306321b17ecSEdward Tomasz Napierala size_t off, void *addr, size_t len) 1307321b17ecSEdward Tomasz Napierala { 1308321b17ecSEdward Tomasz Napierala 130982f7fa7aSAlexander Motin m_copydata(ip->ip_data_mbuf, off, len, addr); 1310321b17ecSEdward Tomasz Napierala } 1311321b17ecSEdward Tomasz Napierala 1312321b17ecSEdward Tomasz Napierala static void 13139a4510acSAlexander Motin icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 1314321b17ecSEdward Tomasz Napierala { 1315321b17ecSEdward Tomasz Napierala 13169a4510acSAlexander Motin icl_soft_conn_pdu_queue_cb(ic, ip, NULL); 13179a4510acSAlexander Motin } 13189a4510acSAlexander Motin 13199a4510acSAlexander Motin static void 13209a4510acSAlexander Motin icl_soft_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip, 13219a4510acSAlexander Motin icl_pdu_cb cb) 13229a4510acSAlexander Motin { 132387322a90SJohn Baldwin struct icl_soft_conn *isc = (struct icl_soft_conn *)ic; 13249a4510acSAlexander Motin struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip; 1325321b17ecSEdward Tomasz Napierala 1326321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT(ic); 13279a4510acSAlexander Motin isp->ref_cnt++; 13289a4510acSAlexander Motin isp->cb = cb; 1329321b17ecSEdward Tomasz Napierala 1330321b17ecSEdward Tomasz Napierala if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1331321b17ecSEdward Tomasz Napierala ICL_DEBUG("icl_pdu_queue on closed connection"); 13329a4510acSAlexander Motin icl_soft_pdu_done(ip, ENOTCONN); 1333321b17ecSEdward Tomasz Napierala return; 1334321b17ecSEdward Tomasz Napierala } 1335321b17ecSEdward Tomasz Napierala 133687322a90SJohn Baldwin if (!STAILQ_EMPTY(&isc->to_send)) { 133787322a90SJohn Baldwin STAILQ_INSERT_TAIL(&isc->to_send, ip, ip_next); 1338321b17ecSEdward Tomasz Napierala /* 1339321b17ecSEdward Tomasz Napierala * If the queue is not empty, someone else had already 1340321b17ecSEdward Tomasz Napierala * signaled the send thread; no need to do that again, 1341321b17ecSEdward Tomasz Napierala * just return. 1342321b17ecSEdward Tomasz Napierala */ 1343321b17ecSEdward Tomasz Napierala return; 1344321b17ecSEdward Tomasz Napierala } 1345321b17ecSEdward Tomasz Napierala 134687322a90SJohn Baldwin STAILQ_INSERT_TAIL(&isc->to_send, ip, ip_next); 134787322a90SJohn Baldwin cv_signal(&isc->send_cv); 1348321b17ecSEdward Tomasz Napierala } 1349321b17ecSEdward Tomasz Napierala 1350321b17ecSEdward Tomasz Napierala static struct icl_conn * 1351321b17ecSEdward Tomasz Napierala icl_soft_new_conn(const char *name, struct mtx *lock) 1352321b17ecSEdward Tomasz Napierala { 135387322a90SJohn Baldwin struct icl_soft_conn *isc; 1354321b17ecSEdward Tomasz Napierala struct icl_conn *ic; 1355321b17ecSEdward Tomasz Napierala 1356321b17ecSEdward Tomasz Napierala refcount_acquire(&icl_ncons); 1357321b17ecSEdward Tomasz Napierala 135887322a90SJohn Baldwin isc = (struct icl_soft_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, 135987322a90SJohn Baldwin M_WAITOK | M_ZERO); 1360321b17ecSEdward Tomasz Napierala 136187322a90SJohn Baldwin STAILQ_INIT(&isc->to_send); 136287322a90SJohn Baldwin cv_init(&isc->send_cv, "icl_tx"); 136387322a90SJohn Baldwin cv_init(&isc->receive_cv, "icl_rx"); 136487322a90SJohn Baldwin 136587322a90SJohn Baldwin ic = &isc->ic; 1366321b17ecSEdward Tomasz Napierala ic->ic_lock = lock; 1367321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC 1368321b17ecSEdward Tomasz Napierala refcount_init(&ic->ic_outstanding_pdus, 0); 1369321b17ecSEdward Tomasz Napierala #endif 1370321b17ecSEdward Tomasz Napierala ic->ic_name = name; 1371d4b195d3SEdward Tomasz Napierala ic->ic_offload = "None"; 1372530e725dSJohn Baldwin ic->ic_unmapped = PMAP_HAS_DMAP; 1373321b17ecSEdward Tomasz Napierala 1374321b17ecSEdward Tomasz Napierala return (ic); 1375321b17ecSEdward Tomasz Napierala } 1376321b17ecSEdward Tomasz Napierala 1377321b17ecSEdward Tomasz Napierala void 1378321b17ecSEdward Tomasz Napierala icl_soft_conn_free(struct icl_conn *ic) 1379321b17ecSEdward Tomasz Napierala { 138087322a90SJohn Baldwin struct icl_soft_conn *isc = (struct icl_soft_conn *)ic; 1381321b17ecSEdward Tomasz Napierala 138222d3bb26SEdward Tomasz Napierala #ifdef DIAGNOSTIC 138322d3bb26SEdward Tomasz Napierala KASSERT(ic->ic_outstanding_pdus == 0, 138422d3bb26SEdward Tomasz Napierala ("destroying session with %d outstanding PDUs", 138522d3bb26SEdward Tomasz Napierala ic->ic_outstanding_pdus)); 138622d3bb26SEdward Tomasz Napierala #endif 138787322a90SJohn Baldwin cv_destroy(&isc->send_cv); 138887322a90SJohn Baldwin cv_destroy(&isc->receive_cv); 138987322a90SJohn Baldwin kobj_delete((struct kobj *)isc, M_ICL_SOFT); 1390321b17ecSEdward Tomasz Napierala refcount_release(&icl_ncons); 1391321b17ecSEdward Tomasz Napierala } 1392321b17ecSEdward Tomasz Napierala 1393321b17ecSEdward Tomasz Napierala static int 1394321b17ecSEdward Tomasz Napierala icl_conn_start(struct icl_conn *ic) 1395321b17ecSEdward Tomasz Napierala { 139687322a90SJohn Baldwin struct icl_soft_conn *isc = (struct icl_soft_conn *)ic; 1397321b17ecSEdward Tomasz Napierala size_t minspace; 1398321b17ecSEdward Tomasz Napierala struct sockopt opt; 1399321b17ecSEdward Tomasz Napierala int error, one = 1; 1400321b17ecSEdward Tomasz Napierala 1401321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1402321b17ecSEdward Tomasz Napierala 1403321b17ecSEdward Tomasz Napierala /* 1404321b17ecSEdward Tomasz Napierala * XXX: Ugly hack. 1405321b17ecSEdward Tomasz Napierala */ 1406321b17ecSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1407321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1408321b17ecSEdward Tomasz Napierala return (EINVAL); 1409321b17ecSEdward Tomasz Napierala } 1410321b17ecSEdward Tomasz Napierala 141187322a90SJohn Baldwin isc->receive_state = ICL_CONN_STATE_BHS; 141287322a90SJohn Baldwin isc->receive_len = sizeof(struct iscsi_bhs); 1413321b17ecSEdward Tomasz Napierala ic->ic_disconnecting = false; 1414321b17ecSEdward Tomasz Napierala 1415321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1416321b17ecSEdward Tomasz Napierala 1417321b17ecSEdward Tomasz Napierala /* 1418321b17ecSEdward Tomasz Napierala * For sendspace, this is required because the current code cannot 1419321b17ecSEdward Tomasz Napierala * send a PDU in pieces; thus, the minimum buffer size is equal 1420321b17ecSEdward Tomasz Napierala * to the maximum PDU size. "+4" is to account for possible padding. 1421321b17ecSEdward Tomasz Napierala */ 14220cc7d64aSJohn Baldwin minspace = sizeof(struct iscsi_bhs) + 14230cc7d64aSJohn Baldwin ic->ic_max_send_data_segment_length + 1424321b17ecSEdward Tomasz Napierala ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1425321b17ecSEdward Tomasz Napierala if (sendspace < minspace) { 1426321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.sendspace too low; must be at least %zd", 1427321b17ecSEdward Tomasz Napierala minspace); 1428321b17ecSEdward Tomasz Napierala sendspace = minspace; 1429321b17ecSEdward Tomasz Napierala } 14300cc7d64aSJohn Baldwin minspace = sizeof(struct iscsi_bhs) + 14310cc7d64aSJohn Baldwin ic->ic_max_recv_data_segment_length + 14320cc7d64aSJohn Baldwin ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4; 1433321b17ecSEdward Tomasz Napierala if (recvspace < minspace) { 1434321b17ecSEdward Tomasz Napierala ICL_WARN("kern.icl.recvspace too low; must be at least %zd", 1435321b17ecSEdward Tomasz Napierala minspace); 1436321b17ecSEdward Tomasz Napierala recvspace = minspace; 1437321b17ecSEdward Tomasz Napierala } 1438321b17ecSEdward Tomasz Napierala 1439321b17ecSEdward Tomasz Napierala error = soreserve(ic->ic_socket, sendspace, recvspace); 1440321b17ecSEdward Tomasz Napierala if (error != 0) { 1441321b17ecSEdward Tomasz Napierala ICL_WARN("soreserve failed with error %d", error); 14425aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(ic); 1443321b17ecSEdward Tomasz Napierala return (error); 1444321b17ecSEdward Tomasz Napierala } 1445321b17ecSEdward Tomasz Napierala ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE; 1446321b17ecSEdward Tomasz Napierala ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE; 1447321b17ecSEdward Tomasz Napierala 1448321b17ecSEdward Tomasz Napierala /* 1449321b17ecSEdward Tomasz Napierala * Disable Nagle. 1450321b17ecSEdward Tomasz Napierala */ 1451321b17ecSEdward Tomasz Napierala bzero(&opt, sizeof(opt)); 1452321b17ecSEdward Tomasz Napierala opt.sopt_dir = SOPT_SET; 1453321b17ecSEdward Tomasz Napierala opt.sopt_level = IPPROTO_TCP; 1454321b17ecSEdward Tomasz Napierala opt.sopt_name = TCP_NODELAY; 1455321b17ecSEdward Tomasz Napierala opt.sopt_val = &one; 1456321b17ecSEdward Tomasz Napierala opt.sopt_valsize = sizeof(one); 1457321b17ecSEdward Tomasz Napierala error = sosetopt(ic->ic_socket, &opt); 1458321b17ecSEdward Tomasz Napierala if (error != 0) { 1459321b17ecSEdward Tomasz Napierala ICL_WARN("disabling TCP_NODELAY failed with error %d", error); 14605aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(ic); 1461321b17ecSEdward Tomasz Napierala return (error); 1462321b17ecSEdward Tomasz Napierala } 1463321b17ecSEdward Tomasz Napierala 1464321b17ecSEdward Tomasz Napierala /* 1465321b17ecSEdward Tomasz Napierala * Register socket upcall, to get notified about incoming PDUs 1466321b17ecSEdward Tomasz Napierala * and free space to send outgoing ones. 1467321b17ecSEdward Tomasz Napierala */ 1468321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_snd); 146987322a90SJohn Baldwin soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, isc); 1470321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_snd); 1471321b17ecSEdward Tomasz Napierala SOCKBUF_LOCK(&ic->ic_socket->so_rcv); 147287322a90SJohn Baldwin soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, isc); 1473321b17ecSEdward Tomasz Napierala SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv); 1474321b17ecSEdward Tomasz Napierala 14755b157f21SAlexander Motin /* 14765b157f21SAlexander Motin * Start threads. 14775b157f21SAlexander Motin */ 14785b157f21SAlexander Motin ICL_CONN_LOCK(ic); 147987322a90SJohn Baldwin isc->send_running = isc->receive_running = true; 14805b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 14815b157f21SAlexander Motin error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx", 14825b157f21SAlexander Motin ic->ic_name); 14835b157f21SAlexander Motin if (error != 0) { 14845b157f21SAlexander Motin ICL_WARN("kthread_add(9) failed with error %d", error); 14855b157f21SAlexander Motin ICL_CONN_LOCK(ic); 148687322a90SJohn Baldwin isc->send_running = isc->receive_running = false; 148787322a90SJohn Baldwin cv_signal(&isc->send_cv); 14885b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 14895b157f21SAlexander Motin icl_soft_conn_close(ic); 14905b157f21SAlexander Motin return (error); 14915b157f21SAlexander Motin } 14925b157f21SAlexander Motin error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx", 14935b157f21SAlexander Motin ic->ic_name); 14945b157f21SAlexander Motin if (error != 0) { 14955b157f21SAlexander Motin ICL_WARN("kthread_add(9) failed with error %d", error); 14965b157f21SAlexander Motin ICL_CONN_LOCK(ic); 149787322a90SJohn Baldwin isc->receive_running = false; 149887322a90SJohn Baldwin cv_signal(&isc->send_cv); 14995b157f21SAlexander Motin ICL_CONN_UNLOCK(ic); 15005b157f21SAlexander Motin icl_soft_conn_close(ic); 15015b157f21SAlexander Motin return (error); 15025b157f21SAlexander Motin } 15035b157f21SAlexander Motin 1504321b17ecSEdward Tomasz Napierala return (0); 1505321b17ecSEdward Tomasz Napierala } 1506321b17ecSEdward Tomasz Napierala 1507321b17ecSEdward Tomasz Napierala int 1508321b17ecSEdward Tomasz Napierala icl_soft_conn_handoff(struct icl_conn *ic, int fd) 1509321b17ecSEdward Tomasz Napierala { 1510321b17ecSEdward Tomasz Napierala struct file *fp; 1511321b17ecSEdward Tomasz Napierala struct socket *so; 1512321b17ecSEdward Tomasz Napierala cap_rights_t rights; 1513321b17ecSEdward Tomasz Napierala int error; 1514321b17ecSEdward Tomasz Napierala 1515321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1516321b17ecSEdward Tomasz Napierala 1517906a424bSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1518906a424bSEdward Tomasz Napierala /* 1519906a424bSEdward Tomasz Napierala * We're transitioning to Full Feature phase, and we don't 1520906a424bSEdward Tomasz Napierala * really care. 1521906a424bSEdward Tomasz Napierala */ 1522906a424bSEdward Tomasz Napierala if (fd == 0) { 1523906a424bSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1524906a424bSEdward Tomasz Napierala if (ic->ic_socket == NULL) { 1525906a424bSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1526906a424bSEdward Tomasz Napierala ICL_WARN("proxy handoff without connect"); 1527906a424bSEdward Tomasz Napierala return (EINVAL); 1528906a424bSEdward Tomasz Napierala } 1529906a424bSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1530906a424bSEdward Tomasz Napierala return (0); 1531906a424bSEdward Tomasz Napierala } 1532906a424bSEdward Tomasz Napierala #endif 1533906a424bSEdward Tomasz Napierala 1534321b17ecSEdward Tomasz Napierala /* 1535321b17ecSEdward Tomasz Napierala * Steal the socket from userland. 1536321b17ecSEdward Tomasz Napierala */ 1537321b17ecSEdward Tomasz Napierala error = fget(curthread, fd, 15386b3a9a0fSMateusz Guzik cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); 1539321b17ecSEdward Tomasz Napierala if (error != 0) 1540321b17ecSEdward Tomasz Napierala return (error); 1541321b17ecSEdward Tomasz Napierala if (fp->f_type != DTYPE_SOCKET) { 1542321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1543321b17ecSEdward Tomasz Napierala return (EINVAL); 1544321b17ecSEdward Tomasz Napierala } 1545321b17ecSEdward Tomasz Napierala so = fp->f_data; 1546321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) { 1547321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1548321b17ecSEdward Tomasz Napierala return (EINVAL); 1549321b17ecSEdward Tomasz Napierala } 1550321b17ecSEdward Tomasz Napierala 1551321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1552321b17ecSEdward Tomasz Napierala 1553321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1554321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1555321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1556321b17ecSEdward Tomasz Napierala return (EBUSY); 1557321b17ecSEdward Tomasz Napierala } 1558321b17ecSEdward Tomasz Napierala 1559321b17ecSEdward Tomasz Napierala ic->ic_socket = fp->f_data; 1560321b17ecSEdward Tomasz Napierala fp->f_ops = &badfileops; 1561321b17ecSEdward Tomasz Napierala fp->f_data = NULL; 1562321b17ecSEdward Tomasz Napierala fdrop(fp, curthread); 1563321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1564321b17ecSEdward Tomasz Napierala 1565321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1566321b17ecSEdward Tomasz Napierala 1567321b17ecSEdward Tomasz Napierala return (error); 1568321b17ecSEdward Tomasz Napierala } 1569321b17ecSEdward Tomasz Napierala 1570321b17ecSEdward Tomasz Napierala void 15715aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(struct icl_conn *ic) 1572321b17ecSEdward Tomasz Napierala { 157387322a90SJohn Baldwin struct icl_soft_conn *isc = (struct icl_soft_conn *)ic; 1574321b17ecSEdward Tomasz Napierala struct icl_pdu *pdu; 15755b157f21SAlexander Motin struct socket *so; 1576321b17ecSEdward Tomasz Napierala 15775b157f21SAlexander Motin /* 15785b157f21SAlexander Motin * Wake up the threads, so they can properly terminate. 15796895f89fSAlexander Motin * Receive thread sleeps on so->so_rcv lock, send on ic->ic_lock. 15805b157f21SAlexander Motin */ 15816895f89fSAlexander Motin ICL_CONN_LOCK(ic); 15826895f89fSAlexander Motin if (!ic->ic_disconnecting) { 15836895f89fSAlexander Motin so = ic->ic_socket; 158406e9c710SAlexander Motin if (so) 15856895f89fSAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 15865b157f21SAlexander Motin ic->ic_disconnecting = true; 158706e9c710SAlexander Motin if (so) 15886895f89fSAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 15896895f89fSAlexander Motin } 159087322a90SJohn Baldwin while (isc->receive_running || isc->send_running) { 159187322a90SJohn Baldwin cv_signal(&isc->receive_cv); 159287322a90SJohn Baldwin cv_signal(&isc->send_cv); 159387322a90SJohn Baldwin cv_wait(&isc->send_cv, ic->ic_lock); 15945b157f21SAlexander Motin } 15955b157f21SAlexander Motin 15965b157f21SAlexander Motin /* Some other thread could close the connection same time. */ 15975b157f21SAlexander Motin so = ic->ic_socket; 15985b157f21SAlexander Motin if (so == NULL) { 1599321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1600321b17ecSEdward Tomasz Napierala return; 1601321b17ecSEdward Tomasz Napierala } 16025b157f21SAlexander Motin ic->ic_socket = NULL; 1603321b17ecSEdward Tomasz Napierala 1604321b17ecSEdward Tomasz Napierala /* 1605321b17ecSEdward Tomasz Napierala * Deregister socket upcalls. 1606321b17ecSEdward Tomasz Napierala */ 1607321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 16085b157f21SAlexander Motin SOCKBUF_LOCK(&so->so_snd); 16095b157f21SAlexander Motin if (so->so_snd.sb_upcall != NULL) 16105b157f21SAlexander Motin soupcall_clear(so, SO_SND); 16115b157f21SAlexander Motin SOCKBUF_UNLOCK(&so->so_snd); 16125b157f21SAlexander Motin SOCKBUF_LOCK(&so->so_rcv); 16135b157f21SAlexander Motin if (so->so_rcv.sb_upcall != NULL) 16145b157f21SAlexander Motin soupcall_clear(so, SO_RCV); 16155b157f21SAlexander Motin SOCKBUF_UNLOCK(&so->so_rcv); 16165b157f21SAlexander Motin soclose(so); 1617321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1618321b17ecSEdward Tomasz Napierala 161987322a90SJohn Baldwin if (isc->receive_pdu != NULL) { 1620321b17ecSEdward Tomasz Napierala //ICL_DEBUG("freeing partially received PDU"); 162187322a90SJohn Baldwin icl_soft_conn_pdu_free(ic, isc->receive_pdu); 162287322a90SJohn Baldwin isc->receive_pdu = NULL; 1623321b17ecSEdward Tomasz Napierala } 1624321b17ecSEdward Tomasz Napierala 1625321b17ecSEdward Tomasz Napierala /* 1626321b17ecSEdward Tomasz Napierala * Remove any outstanding PDUs from the send queue. 1627321b17ecSEdward Tomasz Napierala */ 162887322a90SJohn Baldwin while (!STAILQ_EMPTY(&isc->to_send)) { 162987322a90SJohn Baldwin pdu = STAILQ_FIRST(&isc->to_send); 163087322a90SJohn Baldwin STAILQ_REMOVE_HEAD(&isc->to_send, ip_next); 16319a4510acSAlexander Motin icl_soft_pdu_done(pdu, ENOTCONN); 1632321b17ecSEdward Tomasz Napierala } 1633321b17ecSEdward Tomasz Napierala 163487322a90SJohn Baldwin KASSERT(STAILQ_EMPTY(&isc->to_send), 1635321b17ecSEdward Tomasz Napierala ("destroying session with non-empty send queue")); 1636321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1637321b17ecSEdward Tomasz Napierala } 1638321b17ecSEdward Tomasz Napierala 16397a03d007SEdward Tomasz Napierala int 1640604c023fSEdward Tomasz Napierala icl_soft_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 1641604c023fSEdward Tomasz Napierala struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp) 16427a03d007SEdward Tomasz Napierala { 16437a03d007SEdward Tomasz Napierala 16447a03d007SEdward Tomasz Napierala return (0); 16457a03d007SEdward Tomasz Napierala } 16467a03d007SEdward Tomasz Napierala 16477a03d007SEdward Tomasz Napierala void 16487a03d007SEdward Tomasz Napierala icl_soft_conn_task_done(struct icl_conn *ic, void *prv) 16497a03d007SEdward Tomasz Napierala { 16507a03d007SEdward Tomasz Napierala } 16517a03d007SEdward Tomasz Napierala 16527a03d007SEdward Tomasz Napierala int 16538903d8e3SJohn Baldwin icl_soft_conn_transfer_setup(struct icl_conn *ic, struct icl_pdu *ip, 16548903d8e3SJohn Baldwin union ctl_io *io, uint32_t *transfer_tag, void **prvp) 16557a03d007SEdward Tomasz Napierala { 16567a03d007SEdward Tomasz Napierala 16577a03d007SEdward Tomasz Napierala return (0); 16587a03d007SEdward Tomasz Napierala } 16597a03d007SEdward Tomasz Napierala 16607a03d007SEdward Tomasz Napierala void 16617a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv) 16627a03d007SEdward Tomasz Napierala { 16637a03d007SEdward Tomasz Napierala } 16647a03d007SEdward Tomasz Napierala 1665321b17ecSEdward Tomasz Napierala static int 16667b02c1e8SJohn Baldwin icl_soft_limits(struct icl_drv_limits *idl, int socket) 1667321b17ecSEdward Tomasz Napierala { 1668321b17ecSEdward Tomasz Napierala 1669b75168edSAlexander Motin idl->idl_max_recv_data_segment_length = max_data_segment_length; 1670b75168edSAlexander Motin idl->idl_max_send_data_segment_length = max_data_segment_length; 1671b75168edSAlexander Motin idl->idl_max_burst_length = max_burst_length; 1672b75168edSAlexander Motin idl->idl_first_burst_length = first_burst_length; 1673321b17ecSEdward Tomasz Napierala 1674321b17ecSEdward Tomasz Napierala return (0); 1675321b17ecSEdward Tomasz Napierala } 1676321b17ecSEdward Tomasz Napierala 1677321b17ecSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY 1678321b17ecSEdward Tomasz Napierala int 1679f41492b0SEdward Tomasz Napierala icl_soft_conn_connect(struct icl_conn *ic, int domain, int socktype, 1680f41492b0SEdward Tomasz Napierala int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa) 1681f41492b0SEdward Tomasz Napierala { 1682f41492b0SEdward Tomasz Napierala 1683f41492b0SEdward Tomasz Napierala return (icl_soft_proxy_connect(ic, domain, socktype, protocol, 1684f41492b0SEdward Tomasz Napierala from_sa, to_sa)); 1685f41492b0SEdward Tomasz Napierala } 1686f41492b0SEdward Tomasz Napierala 1687f41492b0SEdward Tomasz Napierala int 1688f41492b0SEdward Tomasz Napierala icl_soft_handoff_sock(struct icl_conn *ic, struct socket *so) 1689321b17ecSEdward Tomasz Napierala { 1690321b17ecSEdward Tomasz Napierala int error; 1691321b17ecSEdward Tomasz Napierala 1692321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK_ASSERT_NOT(ic); 1693321b17ecSEdward Tomasz Napierala 1694321b17ecSEdward Tomasz Napierala if (so->so_type != SOCK_STREAM) 1695321b17ecSEdward Tomasz Napierala return (EINVAL); 1696321b17ecSEdward Tomasz Napierala 1697321b17ecSEdward Tomasz Napierala ICL_CONN_LOCK(ic); 1698321b17ecSEdward Tomasz Napierala if (ic->ic_socket != NULL) { 1699321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1700321b17ecSEdward Tomasz Napierala return (EBUSY); 1701321b17ecSEdward Tomasz Napierala } 1702321b17ecSEdward Tomasz Napierala ic->ic_socket = so; 1703321b17ecSEdward Tomasz Napierala ICL_CONN_UNLOCK(ic); 1704321b17ecSEdward Tomasz Napierala 1705321b17ecSEdward Tomasz Napierala error = icl_conn_start(ic); 1706321b17ecSEdward Tomasz Napierala 1707321b17ecSEdward Tomasz Napierala return (error); 1708321b17ecSEdward Tomasz Napierala } 1709321b17ecSEdward Tomasz Napierala #endif /* ICL_KERNEL_PROXY */ 1710321b17ecSEdward Tomasz Napierala 1711321b17ecSEdward Tomasz Napierala static int 1712321b17ecSEdward Tomasz Napierala icl_soft_load(void) 1713321b17ecSEdward Tomasz Napierala { 1714321b17ecSEdward Tomasz Napierala int error; 1715321b17ecSEdward Tomasz Napierala 17169a4510acSAlexander Motin icl_soft_pdu_zone = uma_zcreate("icl_soft_pdu", 17179a4510acSAlexander Motin sizeof(struct icl_soft_pdu), NULL, NULL, NULL, NULL, 1718321b17ecSEdward Tomasz Napierala UMA_ALIGN_PTR, 0); 1719321b17ecSEdward Tomasz Napierala refcount_init(&icl_ncons, 0); 1720321b17ecSEdward Tomasz Napierala 1721321b17ecSEdward Tomasz Napierala /* 1722321b17ecSEdward Tomasz Napierala * The reason we call this "none" is that to the user, 1723321b17ecSEdward Tomasz Napierala * it's known as "offload driver"; "offload driver: soft" 1724321b17ecSEdward Tomasz Napierala * doesn't make much sense. 1725321b17ecSEdward Tomasz Napierala */ 1726b8911594SEdward Tomasz Napierala error = icl_register("none", false, 0, 1727b8911594SEdward Tomasz Napierala icl_soft_limits, icl_soft_new_conn); 1728321b17ecSEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1729321b17ecSEdward Tomasz Napierala 1730b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0 1731b8911594SEdward Tomasz Napierala /* 1732b8911594SEdward Tomasz Napierala * Debugging aid for kernel proxy functionality. 1733b8911594SEdward Tomasz Napierala */ 1734b8911594SEdward Tomasz Napierala error = icl_register("proxytest", true, 0, 1735b8911594SEdward Tomasz Napierala icl_soft_limits, icl_soft_new_conn); 1736b8911594SEdward Tomasz Napierala KASSERT(error == 0, ("failed to register")); 1737b8911594SEdward Tomasz Napierala #endif 1738b8911594SEdward Tomasz Napierala 1739321b17ecSEdward Tomasz Napierala return (error); 1740321b17ecSEdward Tomasz Napierala } 1741321b17ecSEdward Tomasz Napierala 1742321b17ecSEdward Tomasz Napierala static int 1743321b17ecSEdward Tomasz Napierala icl_soft_unload(void) 1744321b17ecSEdward Tomasz Napierala { 1745321b17ecSEdward Tomasz Napierala 1746321b17ecSEdward Tomasz Napierala if (icl_ncons != 0) 1747321b17ecSEdward Tomasz Napierala return (EBUSY); 1748321b17ecSEdward Tomasz Napierala 1749b8911594SEdward Tomasz Napierala icl_unregister("none", false); 1750b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0 1751b8911594SEdward Tomasz Napierala icl_unregister("proxytest", true); 1752b8911594SEdward Tomasz Napierala #endif 1753321b17ecSEdward Tomasz Napierala 17549a4510acSAlexander Motin uma_zdestroy(icl_soft_pdu_zone); 1755321b17ecSEdward Tomasz Napierala 1756321b17ecSEdward Tomasz Napierala return (0); 1757321b17ecSEdward Tomasz Napierala } 1758321b17ecSEdward Tomasz Napierala 1759321b17ecSEdward Tomasz Napierala static int 1760321b17ecSEdward Tomasz Napierala icl_soft_modevent(module_t mod, int what, void *arg) 1761321b17ecSEdward Tomasz Napierala { 1762321b17ecSEdward Tomasz Napierala 1763321b17ecSEdward Tomasz Napierala switch (what) { 1764321b17ecSEdward Tomasz Napierala case MOD_LOAD: 1765321b17ecSEdward Tomasz Napierala return (icl_soft_load()); 1766321b17ecSEdward Tomasz Napierala case MOD_UNLOAD: 1767321b17ecSEdward Tomasz Napierala return (icl_soft_unload()); 1768321b17ecSEdward Tomasz Napierala default: 1769321b17ecSEdward Tomasz Napierala return (EINVAL); 1770321b17ecSEdward Tomasz Napierala } 1771321b17ecSEdward Tomasz Napierala } 1772321b17ecSEdward Tomasz Napierala 1773321b17ecSEdward Tomasz Napierala moduledata_t icl_soft_data = { 1774321b17ecSEdward Tomasz Napierala "icl_soft", 1775321b17ecSEdward Tomasz Napierala icl_soft_modevent, 1776321b17ecSEdward Tomasz Napierala 0 1777321b17ecSEdward Tomasz Napierala }; 1778321b17ecSEdward Tomasz Napierala 1779321b17ecSEdward Tomasz Napierala DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE); 1780321b17ecSEdward Tomasz Napierala MODULE_DEPEND(icl_soft, icl, 1, 1, 1); 1781872d2d92SEdward Tomasz Napierala MODULE_VERSION(icl_soft, 1); 1782