xref: /freebsd/sys/dev/iscsi/icl_soft.c (revision d0d587c787344615f6b17017dd7a3641212f21c4)
1321b17ecSEdward Tomasz Napierala /*-
2321b17ecSEdward Tomasz Napierala  * Copyright (c) 2012 The FreeBSD Foundation
3321b17ecSEdward Tomasz Napierala  * All rights reserved.
4321b17ecSEdward Tomasz Napierala  *
5321b17ecSEdward Tomasz Napierala  * This software was developed by Edward Tomasz Napierala under sponsorship
6321b17ecSEdward Tomasz Napierala  * from the FreeBSD Foundation.
7321b17ecSEdward Tomasz Napierala  *
8321b17ecSEdward Tomasz Napierala  * Redistribution and use in source and binary forms, with or without
9321b17ecSEdward Tomasz Napierala  * modification, are permitted provided that the following conditions
10321b17ecSEdward Tomasz Napierala  * are met:
11321b17ecSEdward Tomasz Napierala  * 1. Redistributions of source code must retain the above copyright
12321b17ecSEdward Tomasz Napierala  *    notice, this list of conditions and the following disclaimer.
13321b17ecSEdward Tomasz Napierala  * 2. Redistributions in binary form must reproduce the above copyright
14321b17ecSEdward Tomasz Napierala  *    notice, this list of conditions and the following disclaimer in the
15321b17ecSEdward Tomasz Napierala  *    documentation and/or other materials provided with the distribution.
16321b17ecSEdward Tomasz Napierala  *
17321b17ecSEdward Tomasz Napierala  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18321b17ecSEdward Tomasz Napierala  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19321b17ecSEdward Tomasz Napierala  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20321b17ecSEdward Tomasz Napierala  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21321b17ecSEdward Tomasz Napierala  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22321b17ecSEdward Tomasz Napierala  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23321b17ecSEdward Tomasz Napierala  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24321b17ecSEdward Tomasz Napierala  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25321b17ecSEdward Tomasz Napierala  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26321b17ecSEdward Tomasz Napierala  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27321b17ecSEdward Tomasz Napierala  * SUCH DAMAGE.
28321b17ecSEdward Tomasz Napierala  *
29321b17ecSEdward Tomasz Napierala  */
30321b17ecSEdward Tomasz Napierala 
31321b17ecSEdward Tomasz Napierala /*
325aabcd7cSEdward Tomasz Napierala  * Software implementation of iSCSI Common Layer kobj(9) interface.
33321b17ecSEdward Tomasz Napierala  */
34321b17ecSEdward Tomasz Napierala 
35321b17ecSEdward Tomasz Napierala #include <sys/cdefs.h>
36321b17ecSEdward Tomasz Napierala __FBSDID("$FreeBSD$");
37321b17ecSEdward Tomasz Napierala 
38321b17ecSEdward Tomasz Napierala #include <sys/param.h>
39321b17ecSEdward Tomasz Napierala #include <sys/capsicum.h>
40321b17ecSEdward Tomasz Napierala #include <sys/condvar.h>
41321b17ecSEdward Tomasz Napierala #include <sys/conf.h>
42321b17ecSEdward Tomasz Napierala #include <sys/file.h>
43321b17ecSEdward Tomasz Napierala #include <sys/kernel.h>
44321b17ecSEdward Tomasz Napierala #include <sys/kthread.h>
45321b17ecSEdward Tomasz Napierala #include <sys/lock.h>
46321b17ecSEdward Tomasz Napierala #include <sys/mbuf.h>
47321b17ecSEdward Tomasz Napierala #include <sys/mutex.h>
48321b17ecSEdward Tomasz Napierala #include <sys/module.h>
49321b17ecSEdward Tomasz Napierala #include <sys/protosw.h>
50321b17ecSEdward Tomasz Napierala #include <sys/socket.h>
51321b17ecSEdward Tomasz Napierala #include <sys/socketvar.h>
52321b17ecSEdward Tomasz Napierala #include <sys/sysctl.h>
53321b17ecSEdward Tomasz Napierala #include <sys/systm.h>
54321b17ecSEdward Tomasz Napierala #include <sys/sx.h>
55321b17ecSEdward Tomasz Napierala #include <sys/uio.h>
56321b17ecSEdward Tomasz Napierala #include <vm/uma.h>
57321b17ecSEdward Tomasz Napierala #include <netinet/in.h>
58321b17ecSEdward Tomasz Napierala #include <netinet/tcp.h>
59321b17ecSEdward Tomasz Napierala 
60321b17ecSEdward Tomasz Napierala #include <dev/iscsi/icl.h>
61321b17ecSEdward Tomasz Napierala #include <dev/iscsi/iscsi_proto.h>
62321b17ecSEdward Tomasz Napierala #include <icl_conn_if.h>
63321b17ecSEdward Tomasz Napierala 
64321b17ecSEdward Tomasz Napierala static int coalesce = 1;
65321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN,
66321b17ecSEdward Tomasz Napierala     &coalesce, 0, "Try to coalesce PDUs before sending");
67321b17ecSEdward Tomasz Napierala static int partial_receive_len = 128 * 1024;
68321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN,
69321b17ecSEdward Tomasz Napierala     &partial_receive_len, 0, "Minimum read size for partially received "
70321b17ecSEdward Tomasz Napierala     "data segment");
71321b17ecSEdward Tomasz Napierala static int sendspace = 1048576;
72321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN,
73321b17ecSEdward Tomasz Napierala     &sendspace, 0, "Default send socket buffer size");
74321b17ecSEdward Tomasz Napierala static int recvspace = 1048576;
75321b17ecSEdward Tomasz Napierala SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN,
76321b17ecSEdward Tomasz Napierala     &recvspace, 0, "Default receive socket buffer size");
77321b17ecSEdward Tomasz Napierala 
78321b17ecSEdward Tomasz Napierala static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend");
79321b17ecSEdward Tomasz Napierala static uma_zone_t icl_pdu_zone;
80321b17ecSEdward Tomasz Napierala 
81321b17ecSEdward Tomasz Napierala static volatile u_int	icl_ncons;
82321b17ecSEdward Tomasz Napierala 
83321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK(X)		mtx_lock(X->ic_lock)
84321b17ecSEdward Tomasz Napierala #define ICL_CONN_UNLOCK(X)		mtx_unlock(X->ic_lock)
85321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT(X)		mtx_assert(X->ic_lock, MA_OWNED)
86321b17ecSEdward Tomasz Napierala #define ICL_CONN_LOCK_ASSERT_NOT(X)	mtx_assert(X->ic_lock, MA_NOTOWNED)
87321b17ecSEdward Tomasz Napierala 
88321b17ecSEdward Tomasz Napierala STAILQ_HEAD(icl_pdu_stailq, icl_pdu);
89321b17ecSEdward Tomasz Napierala 
90321b17ecSEdward Tomasz Napierala static icl_conn_new_pdu_t	icl_soft_conn_new_pdu;
91321b17ecSEdward Tomasz Napierala static icl_conn_pdu_free_t	icl_soft_conn_pdu_free;
92321b17ecSEdward Tomasz Napierala static icl_conn_pdu_data_segment_length_t
93321b17ecSEdward Tomasz Napierala 				    icl_soft_conn_pdu_data_segment_length;
94321b17ecSEdward Tomasz Napierala static icl_conn_pdu_append_data_t	icl_soft_conn_pdu_append_data;
95321b17ecSEdward Tomasz Napierala static icl_conn_pdu_get_data_t	icl_soft_conn_pdu_get_data;
96321b17ecSEdward Tomasz Napierala static icl_conn_pdu_queue_t	icl_soft_conn_pdu_queue;
97321b17ecSEdward Tomasz Napierala static icl_conn_handoff_t	icl_soft_conn_handoff;
98321b17ecSEdward Tomasz Napierala static icl_conn_free_t		icl_soft_conn_free;
99321b17ecSEdward Tomasz Napierala static icl_conn_close_t		icl_soft_conn_close;
1007a03d007SEdward Tomasz Napierala static icl_conn_task_setup_t	icl_soft_conn_task_setup;
1017a03d007SEdward Tomasz Napierala static icl_conn_task_done_t	icl_soft_conn_task_done;
1027a03d007SEdward Tomasz Napierala static icl_conn_transfer_setup_t	icl_soft_conn_transfer_setup;
1037a03d007SEdward Tomasz Napierala static icl_conn_transfer_done_t	icl_soft_conn_transfer_done;
104f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY
105f41492b0SEdward Tomasz Napierala static icl_conn_connect_t	icl_soft_conn_connect;
106f41492b0SEdward Tomasz Napierala #endif
107321b17ecSEdward Tomasz Napierala 
108321b17ecSEdward Tomasz Napierala static kobj_method_t icl_soft_methods[] = {
109321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu),
110321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free),
111321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_pdu_data_segment_length,
112321b17ecSEdward Tomasz Napierala 	    icl_soft_conn_pdu_data_segment_length),
113321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data),
114321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data),
115321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue),
116321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff),
117321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_free, icl_soft_conn_free),
118321b17ecSEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_close, icl_soft_conn_close),
1197a03d007SEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup),
1207a03d007SEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done),
1217a03d007SEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup),
1227a03d007SEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done),
123f41492b0SEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY
124f41492b0SEdward Tomasz Napierala 	KOBJMETHOD(icl_conn_connect, icl_soft_conn_connect),
125f41492b0SEdward Tomasz Napierala #endif
126321b17ecSEdward Tomasz Napierala 	{ 0, 0 }
127321b17ecSEdward Tomasz Napierala };
128321b17ecSEdward Tomasz Napierala 
129321b17ecSEdward Tomasz Napierala DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn));
130321b17ecSEdward Tomasz Napierala 
131321b17ecSEdward Tomasz Napierala static void
132321b17ecSEdward Tomasz Napierala icl_conn_fail(struct icl_conn *ic)
133321b17ecSEdward Tomasz Napierala {
134321b17ecSEdward Tomasz Napierala 	if (ic->ic_socket == NULL)
135321b17ecSEdward Tomasz Napierala 		return;
136321b17ecSEdward Tomasz Napierala 
137321b17ecSEdward Tomasz Napierala 	/*
138321b17ecSEdward Tomasz Napierala 	 * XXX
139321b17ecSEdward Tomasz Napierala 	 */
140321b17ecSEdward Tomasz Napierala 	ic->ic_socket->so_error = EDOOFUS;
141321b17ecSEdward Tomasz Napierala 	(ic->ic_error)(ic);
142321b17ecSEdward Tomasz Napierala }
143321b17ecSEdward Tomasz Napierala 
144321b17ecSEdward Tomasz Napierala static struct mbuf *
145321b17ecSEdward Tomasz Napierala icl_conn_receive(struct icl_conn *ic, size_t len)
146321b17ecSEdward Tomasz Napierala {
147321b17ecSEdward Tomasz Napierala 	struct uio uio;
148321b17ecSEdward Tomasz Napierala 	struct socket *so;
149321b17ecSEdward Tomasz Napierala 	struct mbuf *m;
150321b17ecSEdward Tomasz Napierala 	int error, flags;
151321b17ecSEdward Tomasz Napierala 
152321b17ecSEdward Tomasz Napierala 	so = ic->ic_socket;
153321b17ecSEdward Tomasz Napierala 
154321b17ecSEdward Tomasz Napierala 	memset(&uio, 0, sizeof(uio));
155321b17ecSEdward Tomasz Napierala 	uio.uio_resid = len;
156321b17ecSEdward Tomasz Napierala 
157321b17ecSEdward Tomasz Napierala 	flags = MSG_DONTWAIT;
158321b17ecSEdward Tomasz Napierala 	error = soreceive(so, NULL, &uio, &m, NULL, &flags);
159321b17ecSEdward Tomasz Napierala 	if (error != 0) {
160321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("soreceive error %d", error);
161321b17ecSEdward Tomasz Napierala 		return (NULL);
162321b17ecSEdward Tomasz Napierala 	}
163321b17ecSEdward Tomasz Napierala 	if (uio.uio_resid != 0) {
164321b17ecSEdward Tomasz Napierala 		m_freem(m);
165321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("short read");
166321b17ecSEdward Tomasz Napierala 		return (NULL);
167321b17ecSEdward Tomasz Napierala 	}
168321b17ecSEdward Tomasz Napierala 
169321b17ecSEdward Tomasz Napierala 	return (m);
170321b17ecSEdward Tomasz Napierala }
171321b17ecSEdward Tomasz Napierala 
172*d0d587c7SAlexander Motin static int
173*d0d587c7SAlexander Motin icl_conn_receive_buf(struct icl_conn *ic, void *buf, size_t len)
174*d0d587c7SAlexander Motin {
175*d0d587c7SAlexander Motin 	struct iovec iov[1];
176*d0d587c7SAlexander Motin 	struct uio uio;
177*d0d587c7SAlexander Motin 	struct socket *so;
178*d0d587c7SAlexander Motin 	int error, flags;
179*d0d587c7SAlexander Motin 
180*d0d587c7SAlexander Motin 	so = ic->ic_socket;
181*d0d587c7SAlexander Motin 
182*d0d587c7SAlexander Motin 	memset(&uio, 0, sizeof(uio));
183*d0d587c7SAlexander Motin 	iov[0].iov_base = buf;
184*d0d587c7SAlexander Motin 	iov[0].iov_len = len;
185*d0d587c7SAlexander Motin 	uio.uio_iov = iov;
186*d0d587c7SAlexander Motin 	uio.uio_iovcnt = 1;
187*d0d587c7SAlexander Motin 	uio.uio_offset = 0;
188*d0d587c7SAlexander Motin 	uio.uio_resid = len;
189*d0d587c7SAlexander Motin 	uio.uio_segflg = UIO_SYSSPACE;
190*d0d587c7SAlexander Motin 	uio.uio_rw = UIO_READ;
191*d0d587c7SAlexander Motin 
192*d0d587c7SAlexander Motin 	flags = MSG_DONTWAIT;
193*d0d587c7SAlexander Motin 	error = soreceive(so, NULL, &uio, NULL, NULL, &flags);
194*d0d587c7SAlexander Motin 	if (error != 0) {
195*d0d587c7SAlexander Motin 		ICL_DEBUG("soreceive error %d", error);
196*d0d587c7SAlexander Motin 		return (-1);
197*d0d587c7SAlexander Motin 	}
198*d0d587c7SAlexander Motin 	if (uio.uio_resid != 0) {
199*d0d587c7SAlexander Motin 		ICL_DEBUG("short read");
200*d0d587c7SAlexander Motin 		return (-1);
201*d0d587c7SAlexander Motin 	}
202*d0d587c7SAlexander Motin 
203*d0d587c7SAlexander Motin 	return (0);
204*d0d587c7SAlexander Motin }
205*d0d587c7SAlexander Motin 
206321b17ecSEdward Tomasz Napierala static struct icl_pdu *
207321b17ecSEdward Tomasz Napierala icl_pdu_new_empty(struct icl_conn *ic, int flags)
208321b17ecSEdward Tomasz Napierala {
209321b17ecSEdward Tomasz Napierala 	struct icl_pdu *ip;
210321b17ecSEdward Tomasz Napierala 
211321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC
212321b17ecSEdward Tomasz Napierala 	refcount_acquire(&ic->ic_outstanding_pdus);
213321b17ecSEdward Tomasz Napierala #endif
214321b17ecSEdward Tomasz Napierala 	ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO);
215321b17ecSEdward Tomasz Napierala 	if (ip == NULL) {
216321b17ecSEdward Tomasz Napierala 		ICL_WARN("failed to allocate %zd bytes", sizeof(*ip));
217321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC
218321b17ecSEdward Tomasz Napierala 		refcount_release(&ic->ic_outstanding_pdus);
219321b17ecSEdward Tomasz Napierala #endif
220321b17ecSEdward Tomasz Napierala 		return (NULL);
221321b17ecSEdward Tomasz Napierala 	}
222321b17ecSEdward Tomasz Napierala 
223321b17ecSEdward Tomasz Napierala 	ip->ip_conn = ic;
224321b17ecSEdward Tomasz Napierala 
225321b17ecSEdward Tomasz Napierala 	return (ip);
226321b17ecSEdward Tomasz Napierala }
227321b17ecSEdward Tomasz Napierala 
228321b17ecSEdward Tomasz Napierala static void
229321b17ecSEdward Tomasz Napierala icl_pdu_free(struct icl_pdu *ip)
230321b17ecSEdward Tomasz Napierala {
231321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
232321b17ecSEdward Tomasz Napierala 
233321b17ecSEdward Tomasz Napierala 	ic = ip->ip_conn;
234321b17ecSEdward Tomasz Napierala 
235321b17ecSEdward Tomasz Napierala 	m_freem(ip->ip_bhs_mbuf);
236321b17ecSEdward Tomasz Napierala 	m_freem(ip->ip_ahs_mbuf);
237321b17ecSEdward Tomasz Napierala 	m_freem(ip->ip_data_mbuf);
238321b17ecSEdward Tomasz Napierala 	uma_zfree(icl_pdu_zone, ip);
239321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC
240321b17ecSEdward Tomasz Napierala 	refcount_release(&ic->ic_outstanding_pdus);
241321b17ecSEdward Tomasz Napierala #endif
242321b17ecSEdward Tomasz Napierala }
243321b17ecSEdward Tomasz Napierala 
244321b17ecSEdward Tomasz Napierala void
245321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
246321b17ecSEdward Tomasz Napierala {
2475aabcd7cSEdward Tomasz Napierala 
248321b17ecSEdward Tomasz Napierala 	icl_pdu_free(ip);
249321b17ecSEdward Tomasz Napierala }
250321b17ecSEdward Tomasz Napierala 
251321b17ecSEdward Tomasz Napierala /*
252321b17ecSEdward Tomasz Napierala  * Allocate icl_pdu with empty BHS to fill up by the caller.
253321b17ecSEdward Tomasz Napierala  */
254321b17ecSEdward Tomasz Napierala struct icl_pdu *
255321b17ecSEdward Tomasz Napierala icl_soft_conn_new_pdu(struct icl_conn *ic, int flags)
256321b17ecSEdward Tomasz Napierala {
257321b17ecSEdward Tomasz Napierala 	struct icl_pdu *ip;
258321b17ecSEdward Tomasz Napierala 
259321b17ecSEdward Tomasz Napierala 	ip = icl_pdu_new_empty(ic, flags);
260321b17ecSEdward Tomasz Napierala 	if (ip == NULL)
261321b17ecSEdward Tomasz Napierala 		return (NULL);
262321b17ecSEdward Tomasz Napierala 
263321b17ecSEdward Tomasz Napierala 	ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs),
264321b17ecSEdward Tomasz Napierala 	    flags, MT_DATA, M_PKTHDR);
265321b17ecSEdward Tomasz Napierala 	if (ip->ip_bhs_mbuf == NULL) {
266*d0d587c7SAlexander Motin 		ICL_WARN("failed to allocate BHS mbuf");
267321b17ecSEdward Tomasz Napierala 		icl_pdu_free(ip);
268321b17ecSEdward Tomasz Napierala 		return (NULL);
269321b17ecSEdward Tomasz Napierala 	}
270321b17ecSEdward Tomasz Napierala 	ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *);
271321b17ecSEdward Tomasz Napierala 	memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs));
272321b17ecSEdward Tomasz Napierala 	ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs);
273321b17ecSEdward Tomasz Napierala 
274321b17ecSEdward Tomasz Napierala 	return (ip);
275321b17ecSEdward Tomasz Napierala }
276321b17ecSEdward Tomasz Napierala 
277321b17ecSEdward Tomasz Napierala static int
278321b17ecSEdward Tomasz Napierala icl_pdu_ahs_length(const struct icl_pdu *request)
279321b17ecSEdward Tomasz Napierala {
280321b17ecSEdward Tomasz Napierala 
281321b17ecSEdward Tomasz Napierala 	return (request->ip_bhs->bhs_total_ahs_len * 4);
282321b17ecSEdward Tomasz Napierala }
283321b17ecSEdward Tomasz Napierala 
284321b17ecSEdward Tomasz Napierala static size_t
285321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_length(const struct icl_pdu *request)
286321b17ecSEdward Tomasz Napierala {
287321b17ecSEdward Tomasz Napierala 	uint32_t len = 0;
288321b17ecSEdward Tomasz Napierala 
289321b17ecSEdward Tomasz Napierala 	len += request->ip_bhs->bhs_data_segment_len[0];
290321b17ecSEdward Tomasz Napierala 	len <<= 8;
291321b17ecSEdward Tomasz Napierala 	len += request->ip_bhs->bhs_data_segment_len[1];
292321b17ecSEdward Tomasz Napierala 	len <<= 8;
293321b17ecSEdward Tomasz Napierala 	len += request->ip_bhs->bhs_data_segment_len[2];
294321b17ecSEdward Tomasz Napierala 
295321b17ecSEdward Tomasz Napierala 	return (len);
296321b17ecSEdward Tomasz Napierala }
297321b17ecSEdward Tomasz Napierala 
298321b17ecSEdward Tomasz Napierala size_t
299321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic,
300321b17ecSEdward Tomasz Napierala     const struct icl_pdu *request)
301321b17ecSEdward Tomasz Napierala {
302321b17ecSEdward Tomasz Napierala 
303321b17ecSEdward Tomasz Napierala 	return (icl_pdu_data_segment_length(request));
304321b17ecSEdward Tomasz Napierala }
305321b17ecSEdward Tomasz Napierala 
306321b17ecSEdward Tomasz Napierala static void
307321b17ecSEdward Tomasz Napierala icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len)
308321b17ecSEdward Tomasz Napierala {
309321b17ecSEdward Tomasz Napierala 
310321b17ecSEdward Tomasz Napierala 	response->ip_bhs->bhs_data_segment_len[2] = len;
311321b17ecSEdward Tomasz Napierala 	response->ip_bhs->bhs_data_segment_len[1] = len >> 8;
312321b17ecSEdward Tomasz Napierala 	response->ip_bhs->bhs_data_segment_len[0] = len >> 16;
313321b17ecSEdward Tomasz Napierala }
314321b17ecSEdward Tomasz Napierala 
315321b17ecSEdward Tomasz Napierala static size_t
316321b17ecSEdward Tomasz Napierala icl_pdu_padding(const struct icl_pdu *ip)
317321b17ecSEdward Tomasz Napierala {
318321b17ecSEdward Tomasz Napierala 
319321b17ecSEdward Tomasz Napierala 	if ((ip->ip_data_len % 4) != 0)
320321b17ecSEdward Tomasz Napierala 		return (4 - (ip->ip_data_len % 4));
321321b17ecSEdward Tomasz Napierala 
322321b17ecSEdward Tomasz Napierala 	return (0);
323321b17ecSEdward Tomasz Napierala }
324321b17ecSEdward Tomasz Napierala 
325321b17ecSEdward Tomasz Napierala static size_t
326321b17ecSEdward Tomasz Napierala icl_pdu_size(const struct icl_pdu *response)
327321b17ecSEdward Tomasz Napierala {
328321b17ecSEdward Tomasz Napierala 	size_t len;
329321b17ecSEdward Tomasz Napierala 
330321b17ecSEdward Tomasz Napierala 	KASSERT(response->ip_ahs_len == 0, ("responding with AHS"));
331321b17ecSEdward Tomasz Napierala 
332321b17ecSEdward Tomasz Napierala 	len = sizeof(struct iscsi_bhs) + response->ip_data_len +
333321b17ecSEdward Tomasz Napierala 	    icl_pdu_padding(response);
334321b17ecSEdward Tomasz Napierala 	if (response->ip_conn->ic_header_crc32c)
335321b17ecSEdward Tomasz Napierala 		len += ISCSI_HEADER_DIGEST_SIZE;
336321b17ecSEdward Tomasz Napierala 	if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c)
337321b17ecSEdward Tomasz Napierala 		len += ISCSI_DATA_DIGEST_SIZE;
338321b17ecSEdward Tomasz Napierala 
339321b17ecSEdward Tomasz Napierala 	return (len);
340321b17ecSEdward Tomasz Napierala }
341321b17ecSEdward Tomasz Napierala 
342321b17ecSEdward Tomasz Napierala static int
343321b17ecSEdward Tomasz Napierala icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep)
344321b17ecSEdward Tomasz Napierala {
345321b17ecSEdward Tomasz Napierala 
346*d0d587c7SAlexander Motin 	if (icl_conn_receive_buf(request->ip_conn,
347*d0d587c7SAlexander Motin 	    request->ip_bhs, sizeof(struct iscsi_bhs))) {
348321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("failed to receive BHS");
349321b17ecSEdward Tomasz Napierala 		return (-1);
350321b17ecSEdward Tomasz Napierala 	}
351321b17ecSEdward Tomasz Napierala 
352321b17ecSEdward Tomasz Napierala 	*availablep -= sizeof(struct iscsi_bhs);
353321b17ecSEdward Tomasz Napierala 	return (0);
354321b17ecSEdward Tomasz Napierala }
355321b17ecSEdward Tomasz Napierala 
356321b17ecSEdward Tomasz Napierala static int
357321b17ecSEdward Tomasz Napierala icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep)
358321b17ecSEdward Tomasz Napierala {
359321b17ecSEdward Tomasz Napierala 
360321b17ecSEdward Tomasz Napierala 	request->ip_ahs_len = icl_pdu_ahs_length(request);
361321b17ecSEdward Tomasz Napierala 	if (request->ip_ahs_len == 0)
362321b17ecSEdward Tomasz Napierala 		return (0);
363321b17ecSEdward Tomasz Napierala 
364321b17ecSEdward Tomasz Napierala 	request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn,
365321b17ecSEdward Tomasz Napierala 	    request->ip_ahs_len);
366321b17ecSEdward Tomasz Napierala 	if (request->ip_ahs_mbuf == NULL) {
367321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("failed to receive AHS");
368321b17ecSEdward Tomasz Napierala 		return (-1);
369321b17ecSEdward Tomasz Napierala 	}
370321b17ecSEdward Tomasz Napierala 
371321b17ecSEdward Tomasz Napierala 	*availablep -= request->ip_ahs_len;
372321b17ecSEdward Tomasz Napierala 	return (0);
373321b17ecSEdward Tomasz Napierala }
374321b17ecSEdward Tomasz Napierala 
375321b17ecSEdward Tomasz Napierala static uint32_t
376321b17ecSEdward Tomasz Napierala icl_mbuf_to_crc32c(const struct mbuf *m0)
377321b17ecSEdward Tomasz Napierala {
378321b17ecSEdward Tomasz Napierala 	uint32_t digest = 0xffffffff;
379321b17ecSEdward Tomasz Napierala 	const struct mbuf *m;
380321b17ecSEdward Tomasz Napierala 
381321b17ecSEdward Tomasz Napierala 	for (m = m0; m != NULL; m = m->m_next)
382321b17ecSEdward Tomasz Napierala 		digest = calculate_crc32c(digest,
383321b17ecSEdward Tomasz Napierala 		    mtod(m, const void *), m->m_len);
384321b17ecSEdward Tomasz Napierala 
385321b17ecSEdward Tomasz Napierala 	digest = digest ^ 0xffffffff;
386321b17ecSEdward Tomasz Napierala 
387321b17ecSEdward Tomasz Napierala 	return (digest);
388321b17ecSEdward Tomasz Napierala }
389321b17ecSEdward Tomasz Napierala 
390321b17ecSEdward Tomasz Napierala static int
391321b17ecSEdward Tomasz Napierala icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep)
392321b17ecSEdward Tomasz Napierala {
393321b17ecSEdward Tomasz Napierala 	uint32_t received_digest, valid_digest;
394321b17ecSEdward Tomasz Napierala 
395321b17ecSEdward Tomasz Napierala 	if (request->ip_conn->ic_header_crc32c == false)
396321b17ecSEdward Tomasz Napierala 		return (0);
397321b17ecSEdward Tomasz Napierala 
398*d0d587c7SAlexander Motin 	CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE);
399*d0d587c7SAlexander Motin 	if (icl_conn_receive_buf(request->ip_conn,
400*d0d587c7SAlexander Motin 	    &received_digest, ISCSI_HEADER_DIGEST_SIZE)) {
401321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("failed to receive header digest");
402321b17ecSEdward Tomasz Napierala 		return (-1);
403321b17ecSEdward Tomasz Napierala 	}
404321b17ecSEdward Tomasz Napierala 	*availablep -= ISCSI_HEADER_DIGEST_SIZE;
405321b17ecSEdward Tomasz Napierala 
406321b17ecSEdward Tomasz Napierala 	/*
407321b17ecSEdward Tomasz Napierala 	 * XXX: Handle AHS.
408321b17ecSEdward Tomasz Napierala 	 */
409321b17ecSEdward Tomasz Napierala 	valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf);
410321b17ecSEdward Tomasz Napierala 	if (received_digest != valid_digest) {
411321b17ecSEdward Tomasz Napierala 		ICL_WARN("header digest check failed; got 0x%x, "
412321b17ecSEdward Tomasz Napierala 		    "should be 0x%x", received_digest, valid_digest);
413321b17ecSEdward Tomasz Napierala 		return (-1);
414321b17ecSEdward Tomasz Napierala 	}
415321b17ecSEdward Tomasz Napierala 
416321b17ecSEdward Tomasz Napierala 	return (0);
417321b17ecSEdward Tomasz Napierala }
418321b17ecSEdward Tomasz Napierala 
419321b17ecSEdward Tomasz Napierala /*
420321b17ecSEdward Tomasz Napierala  * Return the number of bytes that should be waiting in the receive socket
421321b17ecSEdward Tomasz Napierala  * before icl_pdu_receive_data_segment() gets called.
422321b17ecSEdward Tomasz Napierala  */
423321b17ecSEdward Tomasz Napierala static size_t
424321b17ecSEdward Tomasz Napierala icl_pdu_data_segment_receive_len(const struct icl_pdu *request)
425321b17ecSEdward Tomasz Napierala {
426321b17ecSEdward Tomasz Napierala 	size_t len;
427321b17ecSEdward Tomasz Napierala 
428321b17ecSEdward Tomasz Napierala 	len = icl_pdu_data_segment_length(request);
429321b17ecSEdward Tomasz Napierala 	if (len == 0)
430321b17ecSEdward Tomasz Napierala 		return (0);
431321b17ecSEdward Tomasz Napierala 
432321b17ecSEdward Tomasz Napierala 	/*
433321b17ecSEdward Tomasz Napierala 	 * Account for the parts of data segment already read from
434321b17ecSEdward Tomasz Napierala 	 * the socket buffer.
435321b17ecSEdward Tomasz Napierala 	 */
436321b17ecSEdward Tomasz Napierala 	KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len"));
437321b17ecSEdward Tomasz Napierala 	len -= request->ip_data_len;
438321b17ecSEdward Tomasz Napierala 
439321b17ecSEdward Tomasz Napierala 	/*
440321b17ecSEdward Tomasz Napierala 	 * Don't always wait for the full data segment to be delivered
441321b17ecSEdward Tomasz Napierala 	 * to the socket; this might badly affect performance due to
442321b17ecSEdward Tomasz Napierala 	 * TCP window scaling.
443321b17ecSEdward Tomasz Napierala 	 */
444321b17ecSEdward Tomasz Napierala 	if (len > partial_receive_len) {
445321b17ecSEdward Tomasz Napierala #if 0
446321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("need %zd bytes of data, limiting to %zd",
447321b17ecSEdward Tomasz Napierala 		    len, partial_receive_len));
448321b17ecSEdward Tomasz Napierala #endif
449321b17ecSEdward Tomasz Napierala 		len = partial_receive_len;
450321b17ecSEdward Tomasz Napierala 
451321b17ecSEdward Tomasz Napierala 		return (len);
452321b17ecSEdward Tomasz Napierala 	}
453321b17ecSEdward Tomasz Napierala 
454321b17ecSEdward Tomasz Napierala 	/*
455321b17ecSEdward Tomasz Napierala 	 * Account for padding.  Note that due to the way code is written,
456321b17ecSEdward Tomasz Napierala 	 * the icl_pdu_receive_data_segment() must always receive padding
457321b17ecSEdward Tomasz Napierala 	 * along with the last part of data segment, because it would be
458321b17ecSEdward Tomasz Napierala 	 * impossible to tell whether we've already received the full data
459321b17ecSEdward Tomasz Napierala 	 * segment including padding, or without it.
460321b17ecSEdward Tomasz Napierala 	 */
461321b17ecSEdward Tomasz Napierala 	if ((len % 4) != 0)
462321b17ecSEdward Tomasz Napierala 		len += 4 - (len % 4);
463321b17ecSEdward Tomasz Napierala 
464321b17ecSEdward Tomasz Napierala #if 0
465321b17ecSEdward Tomasz Napierala 	ICL_DEBUG("need %zd bytes of data", len));
466321b17ecSEdward Tomasz Napierala #endif
467321b17ecSEdward Tomasz Napierala 
468321b17ecSEdward Tomasz Napierala 	return (len);
469321b17ecSEdward Tomasz Napierala }
470321b17ecSEdward Tomasz Napierala 
471321b17ecSEdward Tomasz Napierala static int
472321b17ecSEdward Tomasz Napierala icl_pdu_receive_data_segment(struct icl_pdu *request,
473321b17ecSEdward Tomasz Napierala     size_t *availablep, bool *more_neededp)
474321b17ecSEdward Tomasz Napierala {
475321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
476321b17ecSEdward Tomasz Napierala 	size_t len, padding = 0;
477321b17ecSEdward Tomasz Napierala 	struct mbuf *m;
478321b17ecSEdward Tomasz Napierala 
479321b17ecSEdward Tomasz Napierala 	ic = request->ip_conn;
480321b17ecSEdward Tomasz Napierala 
481321b17ecSEdward Tomasz Napierala 	*more_neededp = false;
482321b17ecSEdward Tomasz Napierala 	ic->ic_receive_len = 0;
483321b17ecSEdward Tomasz Napierala 
484321b17ecSEdward Tomasz Napierala 	len = icl_pdu_data_segment_length(request);
485321b17ecSEdward Tomasz Napierala 	if (len == 0)
486321b17ecSEdward Tomasz Napierala 		return (0);
487321b17ecSEdward Tomasz Napierala 
488321b17ecSEdward Tomasz Napierala 	if ((len % 4) != 0)
489321b17ecSEdward Tomasz Napierala 		padding = 4 - (len % 4);
490321b17ecSEdward Tomasz Napierala 
491321b17ecSEdward Tomasz Napierala 	/*
492321b17ecSEdward Tomasz Napierala 	 * Account for already received parts of data segment.
493321b17ecSEdward Tomasz Napierala 	 */
494321b17ecSEdward Tomasz Napierala 	KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len"));
495321b17ecSEdward Tomasz Napierala 	len -= request->ip_data_len;
496321b17ecSEdward Tomasz Napierala 
497321b17ecSEdward Tomasz Napierala 	if (len + padding > *availablep) {
498321b17ecSEdward Tomasz Napierala 		/*
499321b17ecSEdward Tomasz Napierala 		 * Not enough data in the socket buffer.  Receive as much
500321b17ecSEdward Tomasz Napierala 		 * as we can.  Don't receive padding, since, obviously, it's
501321b17ecSEdward Tomasz Napierala 		 * not the end of data segment yet.
502321b17ecSEdward Tomasz Napierala 		 */
503321b17ecSEdward Tomasz Napierala #if 0
504321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("limited from %zd to %zd",
505321b17ecSEdward Tomasz Napierala 		    len + padding, *availablep - padding));
506321b17ecSEdward Tomasz Napierala #endif
507321b17ecSEdward Tomasz Napierala 		len = *availablep - padding;
508321b17ecSEdward Tomasz Napierala 		*more_neededp = true;
509321b17ecSEdward Tomasz Napierala 		padding = 0;
510321b17ecSEdward Tomasz Napierala 	}
511321b17ecSEdward Tomasz Napierala 
512321b17ecSEdward Tomasz Napierala 	/*
513321b17ecSEdward Tomasz Napierala 	 * Must not try to receive padding without at least one byte
514321b17ecSEdward Tomasz Napierala 	 * of actual data segment.
515321b17ecSEdward Tomasz Napierala 	 */
516321b17ecSEdward Tomasz Napierala 	if (len > 0) {
517321b17ecSEdward Tomasz Napierala 		m = icl_conn_receive(request->ip_conn, len + padding);
518321b17ecSEdward Tomasz Napierala 		if (m == NULL) {
519321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to receive data segment");
520321b17ecSEdward Tomasz Napierala 			return (-1);
521321b17ecSEdward Tomasz Napierala 		}
522321b17ecSEdward Tomasz Napierala 
523321b17ecSEdward Tomasz Napierala 		if (request->ip_data_mbuf == NULL)
524321b17ecSEdward Tomasz Napierala 			request->ip_data_mbuf = m;
525321b17ecSEdward Tomasz Napierala 		else
526321b17ecSEdward Tomasz Napierala 			m_cat(request->ip_data_mbuf, m);
527321b17ecSEdward Tomasz Napierala 
528321b17ecSEdward Tomasz Napierala 		request->ip_data_len += len;
529321b17ecSEdward Tomasz Napierala 		*availablep -= len + padding;
530321b17ecSEdward Tomasz Napierala 	} else
531321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("len 0");
532321b17ecSEdward Tomasz Napierala 
533321b17ecSEdward Tomasz Napierala 	if (*more_neededp)
534321b17ecSEdward Tomasz Napierala 		ic->ic_receive_len =
535321b17ecSEdward Tomasz Napierala 		    icl_pdu_data_segment_receive_len(request);
536321b17ecSEdward Tomasz Napierala 
537321b17ecSEdward Tomasz Napierala 	return (0);
538321b17ecSEdward Tomasz Napierala }
539321b17ecSEdward Tomasz Napierala 
540321b17ecSEdward Tomasz Napierala static int
541321b17ecSEdward Tomasz Napierala icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep)
542321b17ecSEdward Tomasz Napierala {
543321b17ecSEdward Tomasz Napierala 	uint32_t received_digest, valid_digest;
544321b17ecSEdward Tomasz Napierala 
545321b17ecSEdward Tomasz Napierala 	if (request->ip_conn->ic_data_crc32c == false)
546321b17ecSEdward Tomasz Napierala 		return (0);
547321b17ecSEdward Tomasz Napierala 
548321b17ecSEdward Tomasz Napierala 	if (request->ip_data_len == 0)
549321b17ecSEdward Tomasz Napierala 		return (0);
550321b17ecSEdward Tomasz Napierala 
551*d0d587c7SAlexander Motin 	CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE);
552*d0d587c7SAlexander Motin 	if (icl_conn_receive_buf(request->ip_conn,
553*d0d587c7SAlexander Motin 	    &received_digest, ISCSI_DATA_DIGEST_SIZE)) {
554321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("failed to receive data digest");
555321b17ecSEdward Tomasz Napierala 		return (-1);
556321b17ecSEdward Tomasz Napierala 	}
557321b17ecSEdward Tomasz Napierala 	*availablep -= ISCSI_DATA_DIGEST_SIZE;
558321b17ecSEdward Tomasz Napierala 
559321b17ecSEdward Tomasz Napierala 	/*
560321b17ecSEdward Tomasz Napierala 	 * Note that ip_data_mbuf also contains padding; since digest
561321b17ecSEdward Tomasz Napierala 	 * calculation is supposed to include that, we iterate over
562321b17ecSEdward Tomasz Napierala 	 * the entire ip_data_mbuf chain, not just ip_data_len bytes of it.
563321b17ecSEdward Tomasz Napierala 	 */
564321b17ecSEdward Tomasz Napierala 	valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf);
565321b17ecSEdward Tomasz Napierala 	if (received_digest != valid_digest) {
566321b17ecSEdward Tomasz Napierala 		ICL_WARN("data digest check failed; got 0x%x, "
567321b17ecSEdward Tomasz Napierala 		    "should be 0x%x", received_digest, valid_digest);
568321b17ecSEdward Tomasz Napierala 		return (-1);
569321b17ecSEdward Tomasz Napierala 	}
570321b17ecSEdward Tomasz Napierala 
571321b17ecSEdward Tomasz Napierala 	return (0);
572321b17ecSEdward Tomasz Napierala }
573321b17ecSEdward Tomasz Napierala 
574321b17ecSEdward Tomasz Napierala /*
575321b17ecSEdward Tomasz Napierala  * Somewhat contrary to the name, this attempts to receive only one
576321b17ecSEdward Tomasz Napierala  * "part" of PDU at a time; call it repeatedly until it returns non-NULL.
577321b17ecSEdward Tomasz Napierala  */
578321b17ecSEdward Tomasz Napierala static struct icl_pdu *
579321b17ecSEdward Tomasz Napierala icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep)
580321b17ecSEdward Tomasz Napierala {
581321b17ecSEdward Tomasz Napierala 	struct icl_pdu *request;
582321b17ecSEdward Tomasz Napierala 	struct socket *so;
583321b17ecSEdward Tomasz Napierala 	size_t len;
584321b17ecSEdward Tomasz Napierala 	int error;
585321b17ecSEdward Tomasz Napierala 	bool more_needed;
586321b17ecSEdward Tomasz Napierala 
587321b17ecSEdward Tomasz Napierala 	so = ic->ic_socket;
588321b17ecSEdward Tomasz Napierala 
589321b17ecSEdward Tomasz Napierala 	if (ic->ic_receive_state == ICL_CONN_STATE_BHS) {
590321b17ecSEdward Tomasz Napierala 		KASSERT(ic->ic_receive_pdu == NULL,
591321b17ecSEdward Tomasz Napierala 		    ("ic->ic_receive_pdu != NULL"));
592*d0d587c7SAlexander Motin 		request = icl_soft_conn_new_pdu(ic, M_NOWAIT);
593321b17ecSEdward Tomasz Napierala 		if (request == NULL) {
594321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to allocate PDU; "
595321b17ecSEdward Tomasz Napierala 			    "dropping connection");
596321b17ecSEdward Tomasz Napierala 			icl_conn_fail(ic);
597321b17ecSEdward Tomasz Napierala 			return (NULL);
598321b17ecSEdward Tomasz Napierala 		}
599321b17ecSEdward Tomasz Napierala 		ic->ic_receive_pdu = request;
600321b17ecSEdward Tomasz Napierala 	} else {
601321b17ecSEdward Tomasz Napierala 		KASSERT(ic->ic_receive_pdu != NULL,
602321b17ecSEdward Tomasz Napierala 		    ("ic->ic_receive_pdu == NULL"));
603321b17ecSEdward Tomasz Napierala 		request = ic->ic_receive_pdu;
604321b17ecSEdward Tomasz Napierala 	}
605321b17ecSEdward Tomasz Napierala 
606321b17ecSEdward Tomasz Napierala 	if (*availablep < ic->ic_receive_len) {
607321b17ecSEdward Tomasz Napierala #if 0
608321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("not enough data; need %zd, "
609321b17ecSEdward Tomasz Napierala 		    "have %zd", ic->ic_receive_len, *availablep);
610321b17ecSEdward Tomasz Napierala #endif
611321b17ecSEdward Tomasz Napierala 		return (NULL);
612321b17ecSEdward Tomasz Napierala 	}
613321b17ecSEdward Tomasz Napierala 
614321b17ecSEdward Tomasz Napierala 	switch (ic->ic_receive_state) {
615321b17ecSEdward Tomasz Napierala 	case ICL_CONN_STATE_BHS:
616321b17ecSEdward Tomasz Napierala 		//ICL_DEBUG("receiving BHS");
617321b17ecSEdward Tomasz Napierala 		error = icl_pdu_receive_bhs(request, availablep);
618321b17ecSEdward Tomasz Napierala 		if (error != 0) {
619321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to receive BHS; "
620321b17ecSEdward Tomasz Napierala 			    "dropping connection");
621321b17ecSEdward Tomasz Napierala 			break;
622321b17ecSEdward Tomasz Napierala 		}
623321b17ecSEdward Tomasz Napierala 
624321b17ecSEdward Tomasz Napierala 		/*
625321b17ecSEdward Tomasz Napierala 		 * We don't enforce any limit for AHS length;
626321b17ecSEdward Tomasz Napierala 		 * its length is stored in 8 bit field.
627321b17ecSEdward Tomasz Napierala 		 */
628321b17ecSEdward Tomasz Napierala 
629321b17ecSEdward Tomasz Napierala 		len = icl_pdu_data_segment_length(request);
630321b17ecSEdward Tomasz Napierala 		if (len > ic->ic_max_data_segment_length) {
631321b17ecSEdward Tomasz Napierala 			ICL_WARN("received data segment "
632321b17ecSEdward Tomasz Napierala 			    "length %zd is larger than negotiated "
633321b17ecSEdward Tomasz Napierala 			    "MaxDataSegmentLength %zd; "
634321b17ecSEdward Tomasz Napierala 			    "dropping connection",
635321b17ecSEdward Tomasz Napierala 			    len, ic->ic_max_data_segment_length);
636321b17ecSEdward Tomasz Napierala 			error = EINVAL;
637321b17ecSEdward Tomasz Napierala 			break;
638321b17ecSEdward Tomasz Napierala 		}
639321b17ecSEdward Tomasz Napierala 
640321b17ecSEdward Tomasz Napierala 		ic->ic_receive_state = ICL_CONN_STATE_AHS;
641321b17ecSEdward Tomasz Napierala 		ic->ic_receive_len = icl_pdu_ahs_length(request);
642321b17ecSEdward Tomasz Napierala 		break;
643321b17ecSEdward Tomasz Napierala 
644321b17ecSEdward Tomasz Napierala 	case ICL_CONN_STATE_AHS:
645321b17ecSEdward Tomasz Napierala 		//ICL_DEBUG("receiving AHS");
646321b17ecSEdward Tomasz Napierala 		error = icl_pdu_receive_ahs(request, availablep);
647321b17ecSEdward Tomasz Napierala 		if (error != 0) {
648321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to receive AHS; "
649321b17ecSEdward Tomasz Napierala 			    "dropping connection");
650321b17ecSEdward Tomasz Napierala 			break;
651321b17ecSEdward Tomasz Napierala 		}
652321b17ecSEdward Tomasz Napierala 		ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST;
653321b17ecSEdward Tomasz Napierala 		if (ic->ic_header_crc32c == false)
654321b17ecSEdward Tomasz Napierala 			ic->ic_receive_len = 0;
655321b17ecSEdward Tomasz Napierala 		else
656321b17ecSEdward Tomasz Napierala 			ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE;
657321b17ecSEdward Tomasz Napierala 		break;
658321b17ecSEdward Tomasz Napierala 
659321b17ecSEdward Tomasz Napierala 	case ICL_CONN_STATE_HEADER_DIGEST:
660321b17ecSEdward Tomasz Napierala 		//ICL_DEBUG("receiving header digest");
661321b17ecSEdward Tomasz Napierala 		error = icl_pdu_check_header_digest(request, availablep);
662321b17ecSEdward Tomasz Napierala 		if (error != 0) {
663321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("header digest failed; "
664321b17ecSEdward Tomasz Napierala 			    "dropping connection");
665321b17ecSEdward Tomasz Napierala 			break;
666321b17ecSEdward Tomasz Napierala 		}
667321b17ecSEdward Tomasz Napierala 
668321b17ecSEdward Tomasz Napierala 		ic->ic_receive_state = ICL_CONN_STATE_DATA;
669321b17ecSEdward Tomasz Napierala 		ic->ic_receive_len =
670321b17ecSEdward Tomasz Napierala 		    icl_pdu_data_segment_receive_len(request);
671321b17ecSEdward Tomasz Napierala 		break;
672321b17ecSEdward Tomasz Napierala 
673321b17ecSEdward Tomasz Napierala 	case ICL_CONN_STATE_DATA:
674321b17ecSEdward Tomasz Napierala 		//ICL_DEBUG("receiving data segment");
675321b17ecSEdward Tomasz Napierala 		error = icl_pdu_receive_data_segment(request, availablep,
676321b17ecSEdward Tomasz Napierala 		    &more_needed);
677321b17ecSEdward Tomasz Napierala 		if (error != 0) {
678321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to receive data segment;"
679321b17ecSEdward Tomasz Napierala 			    "dropping connection");
680321b17ecSEdward Tomasz Napierala 			break;
681321b17ecSEdward Tomasz Napierala 		}
682321b17ecSEdward Tomasz Napierala 
683321b17ecSEdward Tomasz Napierala 		if (more_needed)
684321b17ecSEdward Tomasz Napierala 			break;
685321b17ecSEdward Tomasz Napierala 
686321b17ecSEdward Tomasz Napierala 		ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST;
687321b17ecSEdward Tomasz Napierala 		if (request->ip_data_len == 0 || ic->ic_data_crc32c == false)
688321b17ecSEdward Tomasz Napierala 			ic->ic_receive_len = 0;
689321b17ecSEdward Tomasz Napierala 		else
690321b17ecSEdward Tomasz Napierala 			ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE;
691321b17ecSEdward Tomasz Napierala 		break;
692321b17ecSEdward Tomasz Napierala 
693321b17ecSEdward Tomasz Napierala 	case ICL_CONN_STATE_DATA_DIGEST:
694321b17ecSEdward Tomasz Napierala 		//ICL_DEBUG("receiving data digest");
695321b17ecSEdward Tomasz Napierala 		error = icl_pdu_check_data_digest(request, availablep);
696321b17ecSEdward Tomasz Napierala 		if (error != 0) {
697321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("data digest failed; "
698321b17ecSEdward Tomasz Napierala 			    "dropping connection");
699321b17ecSEdward Tomasz Napierala 			break;
700321b17ecSEdward Tomasz Napierala 		}
701321b17ecSEdward Tomasz Napierala 
702321b17ecSEdward Tomasz Napierala 		/*
703321b17ecSEdward Tomasz Napierala 		 * We've received complete PDU; reset the receive state machine
704321b17ecSEdward Tomasz Napierala 		 * and return the PDU.
705321b17ecSEdward Tomasz Napierala 		 */
706321b17ecSEdward Tomasz Napierala 		ic->ic_receive_state = ICL_CONN_STATE_BHS;
707321b17ecSEdward Tomasz Napierala 		ic->ic_receive_len = sizeof(struct iscsi_bhs);
708321b17ecSEdward Tomasz Napierala 		ic->ic_receive_pdu = NULL;
709321b17ecSEdward Tomasz Napierala 		return (request);
710321b17ecSEdward Tomasz Napierala 
711321b17ecSEdward Tomasz Napierala 	default:
712321b17ecSEdward Tomasz Napierala 		panic("invalid ic_receive_state %d\n", ic->ic_receive_state);
713321b17ecSEdward Tomasz Napierala 	}
714321b17ecSEdward Tomasz Napierala 
715321b17ecSEdward Tomasz Napierala 	if (error != 0) {
716321b17ecSEdward Tomasz Napierala 		/*
717321b17ecSEdward Tomasz Napierala 		 * Don't free the PDU; it's pointed to by ic->ic_receive_pdu
7185aabcd7cSEdward Tomasz Napierala 		 * and will get freed in icl_soft_conn_close().
719321b17ecSEdward Tomasz Napierala 		 */
720321b17ecSEdward Tomasz Napierala 		icl_conn_fail(ic);
721321b17ecSEdward Tomasz Napierala 	}
722321b17ecSEdward Tomasz Napierala 
723321b17ecSEdward Tomasz Napierala 	return (NULL);
724321b17ecSEdward Tomasz Napierala }
725321b17ecSEdward Tomasz Napierala 
726321b17ecSEdward Tomasz Napierala static void
727321b17ecSEdward Tomasz Napierala icl_conn_receive_pdus(struct icl_conn *ic, size_t available)
728321b17ecSEdward Tomasz Napierala {
729321b17ecSEdward Tomasz Napierala 	struct icl_pdu *response;
730321b17ecSEdward Tomasz Napierala 	struct socket *so;
731321b17ecSEdward Tomasz Napierala 
732321b17ecSEdward Tomasz Napierala 	so = ic->ic_socket;
733321b17ecSEdward Tomasz Napierala 
734321b17ecSEdward Tomasz Napierala 	/*
735321b17ecSEdward Tomasz Napierala 	 * This can never happen; we're careful to only mess with ic->ic_socket
736321b17ecSEdward Tomasz Napierala 	 * pointer when the send/receive threads are not running.
737321b17ecSEdward Tomasz Napierala 	 */
738321b17ecSEdward Tomasz Napierala 	KASSERT(so != NULL, ("NULL socket"));
739321b17ecSEdward Tomasz Napierala 
740321b17ecSEdward Tomasz Napierala 	for (;;) {
741321b17ecSEdward Tomasz Napierala 		if (ic->ic_disconnecting)
742321b17ecSEdward Tomasz Napierala 			return;
743321b17ecSEdward Tomasz Napierala 
744321b17ecSEdward Tomasz Napierala 		if (so->so_error != 0) {
745321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("connection error %d; "
746321b17ecSEdward Tomasz Napierala 			    "dropping connection", so->so_error);
747321b17ecSEdward Tomasz Napierala 			icl_conn_fail(ic);
748321b17ecSEdward Tomasz Napierala 			return;
749321b17ecSEdward Tomasz Napierala 		}
750321b17ecSEdward Tomasz Napierala 
751321b17ecSEdward Tomasz Napierala 		/*
752321b17ecSEdward Tomasz Napierala 		 * Loop until we have a complete PDU or there is not enough
753321b17ecSEdward Tomasz Napierala 		 * data in the socket buffer.
754321b17ecSEdward Tomasz Napierala 		 */
755321b17ecSEdward Tomasz Napierala 		if (available < ic->ic_receive_len) {
756321b17ecSEdward Tomasz Napierala #if 0
757321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("not enough data; have %zd, "
758321b17ecSEdward Tomasz Napierala 			    "need %zd", available,
759321b17ecSEdward Tomasz Napierala 			    ic->ic_receive_len);
760321b17ecSEdward Tomasz Napierala #endif
761321b17ecSEdward Tomasz Napierala 			return;
762321b17ecSEdward Tomasz Napierala 		}
763321b17ecSEdward Tomasz Napierala 
764321b17ecSEdward Tomasz Napierala 		response = icl_conn_receive_pdu(ic, &available);
765321b17ecSEdward Tomasz Napierala 		if (response == NULL)
766321b17ecSEdward Tomasz Napierala 			continue;
767321b17ecSEdward Tomasz Napierala 
768321b17ecSEdward Tomasz Napierala 		if (response->ip_ahs_len > 0) {
769321b17ecSEdward Tomasz Napierala 			ICL_WARN("received PDU with unsupported "
770321b17ecSEdward Tomasz Napierala 			    "AHS; opcode 0x%x; dropping connection",
771321b17ecSEdward Tomasz Napierala 			    response->ip_bhs->bhs_opcode);
772321b17ecSEdward Tomasz Napierala 			icl_pdu_free(response);
773321b17ecSEdward Tomasz Napierala 			icl_conn_fail(ic);
774321b17ecSEdward Tomasz Napierala 			return;
775321b17ecSEdward Tomasz Napierala 		}
776321b17ecSEdward Tomasz Napierala 
777321b17ecSEdward Tomasz Napierala 		(ic->ic_receive)(response);
778321b17ecSEdward Tomasz Napierala 	}
779321b17ecSEdward Tomasz Napierala }
780321b17ecSEdward Tomasz Napierala 
781321b17ecSEdward Tomasz Napierala static void
782321b17ecSEdward Tomasz Napierala icl_receive_thread(void *arg)
783321b17ecSEdward Tomasz Napierala {
784321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
785321b17ecSEdward Tomasz Napierala 	size_t available;
786321b17ecSEdward Tomasz Napierala 	struct socket *so;
787321b17ecSEdward Tomasz Napierala 
788321b17ecSEdward Tomasz Napierala 	ic = arg;
789321b17ecSEdward Tomasz Napierala 	so = ic->ic_socket;
790321b17ecSEdward Tomasz Napierala 
791321b17ecSEdward Tomasz Napierala 	for (;;) {
792321b17ecSEdward Tomasz Napierala 		if (ic->ic_disconnecting) {
793321b17ecSEdward Tomasz Napierala 			//ICL_DEBUG("terminating");
794321b17ecSEdward Tomasz Napierala 			break;
795321b17ecSEdward Tomasz Napierala 		}
796321b17ecSEdward Tomasz Napierala 
797321b17ecSEdward Tomasz Napierala 		/*
798321b17ecSEdward Tomasz Napierala 		 * Set the low watermark, to be checked by
799321b17ecSEdward Tomasz Napierala 		 * soreadable() in icl_soupcall_receive()
800266078c6SPedro F. Giffuni 		 * to avoid unnecessary wakeups until there
801321b17ecSEdward Tomasz Napierala 		 * is enough data received to read the PDU.
802321b17ecSEdward Tomasz Napierala 		 */
803321b17ecSEdward Tomasz Napierala 		SOCKBUF_LOCK(&so->so_rcv);
804321b17ecSEdward Tomasz Napierala 		available = sbavail(&so->so_rcv);
805321b17ecSEdward Tomasz Napierala 		if (available < ic->ic_receive_len) {
806321b17ecSEdward Tomasz Napierala 			so->so_rcv.sb_lowat = ic->ic_receive_len;
807321b17ecSEdward Tomasz Napierala 			cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx);
808321b17ecSEdward Tomasz Napierala 		} else
809321b17ecSEdward Tomasz Napierala 			so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1;
810321b17ecSEdward Tomasz Napierala 		SOCKBUF_UNLOCK(&so->so_rcv);
811321b17ecSEdward Tomasz Napierala 
812321b17ecSEdward Tomasz Napierala 		icl_conn_receive_pdus(ic, available);
813321b17ecSEdward Tomasz Napierala 	}
814321b17ecSEdward Tomasz Napierala 
815321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
816321b17ecSEdward Tomasz Napierala 	ic->ic_receive_running = false;
817321b17ecSEdward Tomasz Napierala 	cv_signal(&ic->ic_send_cv);
818321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
819321b17ecSEdward Tomasz Napierala 	kthread_exit();
820321b17ecSEdward Tomasz Napierala }
821321b17ecSEdward Tomasz Napierala 
822321b17ecSEdward Tomasz Napierala static int
823321b17ecSEdward Tomasz Napierala icl_soupcall_receive(struct socket *so, void *arg, int waitflag)
824321b17ecSEdward Tomasz Napierala {
825321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
826321b17ecSEdward Tomasz Napierala 
827321b17ecSEdward Tomasz Napierala 	if (!soreadable(so))
828321b17ecSEdward Tomasz Napierala 		return (SU_OK);
829321b17ecSEdward Tomasz Napierala 
830321b17ecSEdward Tomasz Napierala 	ic = arg;
831321b17ecSEdward Tomasz Napierala 	cv_signal(&ic->ic_receive_cv);
832321b17ecSEdward Tomasz Napierala 	return (SU_OK);
833321b17ecSEdward Tomasz Napierala }
834321b17ecSEdward Tomasz Napierala 
835321b17ecSEdward Tomasz Napierala static int
836321b17ecSEdward Tomasz Napierala icl_pdu_finalize(struct icl_pdu *request)
837321b17ecSEdward Tomasz Napierala {
838321b17ecSEdward Tomasz Napierala 	size_t padding, pdu_len;
839321b17ecSEdward Tomasz Napierala 	uint32_t digest, zero = 0;
840321b17ecSEdward Tomasz Napierala 	int ok;
841321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
842321b17ecSEdward Tomasz Napierala 
843321b17ecSEdward Tomasz Napierala 	ic = request->ip_conn;
844321b17ecSEdward Tomasz Napierala 
845321b17ecSEdward Tomasz Napierala 	icl_pdu_set_data_segment_length(request, request->ip_data_len);
846321b17ecSEdward Tomasz Napierala 
847321b17ecSEdward Tomasz Napierala 	pdu_len = icl_pdu_size(request);
848321b17ecSEdward Tomasz Napierala 
849321b17ecSEdward Tomasz Napierala 	if (ic->ic_header_crc32c) {
850321b17ecSEdward Tomasz Napierala 		digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf);
851321b17ecSEdward Tomasz Napierala 		ok = m_append(request->ip_bhs_mbuf, sizeof(digest),
852321b17ecSEdward Tomasz Napierala 		    (void *)&digest);
853321b17ecSEdward Tomasz Napierala 		if (ok != 1) {
854321b17ecSEdward Tomasz Napierala 			ICL_WARN("failed to append header digest");
855321b17ecSEdward Tomasz Napierala 			return (1);
856321b17ecSEdward Tomasz Napierala 		}
857321b17ecSEdward Tomasz Napierala 	}
858321b17ecSEdward Tomasz Napierala 
859321b17ecSEdward Tomasz Napierala 	if (request->ip_data_len != 0) {
860321b17ecSEdward Tomasz Napierala 		padding = icl_pdu_padding(request);
861321b17ecSEdward Tomasz Napierala 		if (padding > 0) {
862321b17ecSEdward Tomasz Napierala 			ok = m_append(request->ip_data_mbuf, padding,
863321b17ecSEdward Tomasz Napierala 			    (void *)&zero);
864321b17ecSEdward Tomasz Napierala 			if (ok != 1) {
865321b17ecSEdward Tomasz Napierala 				ICL_WARN("failed to append padding");
866321b17ecSEdward Tomasz Napierala 				return (1);
867321b17ecSEdward Tomasz Napierala 			}
868321b17ecSEdward Tomasz Napierala 		}
869321b17ecSEdward Tomasz Napierala 
870321b17ecSEdward Tomasz Napierala 		if (ic->ic_data_crc32c) {
871321b17ecSEdward Tomasz Napierala 			digest = icl_mbuf_to_crc32c(request->ip_data_mbuf);
872321b17ecSEdward Tomasz Napierala 
873321b17ecSEdward Tomasz Napierala 			ok = m_append(request->ip_data_mbuf, sizeof(digest),
874321b17ecSEdward Tomasz Napierala 			    (void *)&digest);
875321b17ecSEdward Tomasz Napierala 			if (ok != 1) {
876321b17ecSEdward Tomasz Napierala 				ICL_WARN("failed to append data digest");
877321b17ecSEdward Tomasz Napierala 				return (1);
878321b17ecSEdward Tomasz Napierala 			}
879321b17ecSEdward Tomasz Napierala 		}
880321b17ecSEdward Tomasz Napierala 
881321b17ecSEdward Tomasz Napierala 		m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf);
882321b17ecSEdward Tomasz Napierala 		request->ip_data_mbuf = NULL;
883321b17ecSEdward Tomasz Napierala 	}
884321b17ecSEdward Tomasz Napierala 
885321b17ecSEdward Tomasz Napierala 	request->ip_bhs_mbuf->m_pkthdr.len = pdu_len;
886321b17ecSEdward Tomasz Napierala 
887321b17ecSEdward Tomasz Napierala 	return (0);
888321b17ecSEdward Tomasz Napierala }
889321b17ecSEdward Tomasz Napierala 
890321b17ecSEdward Tomasz Napierala static void
891321b17ecSEdward Tomasz Napierala icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue)
892321b17ecSEdward Tomasz Napierala {
893321b17ecSEdward Tomasz Napierala 	struct icl_pdu *request, *request2;
894321b17ecSEdward Tomasz Napierala 	struct socket *so;
895321b17ecSEdward Tomasz Napierala 	size_t available, size, size2;
896321b17ecSEdward Tomasz Napierala 	int coalesced, error;
897321b17ecSEdward Tomasz Napierala 
898321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK_ASSERT_NOT(ic);
899321b17ecSEdward Tomasz Napierala 
900321b17ecSEdward Tomasz Napierala 	so = ic->ic_socket;
901321b17ecSEdward Tomasz Napierala 
902321b17ecSEdward Tomasz Napierala 	SOCKBUF_LOCK(&so->so_snd);
903321b17ecSEdward Tomasz Napierala 	/*
904321b17ecSEdward Tomasz Napierala 	 * Check how much space do we have for transmit.  We can't just
905321b17ecSEdward Tomasz Napierala 	 * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE,
906321b17ecSEdward Tomasz Napierala 	 * as it always frees the mbuf chain passed to it, even in case
907321b17ecSEdward Tomasz Napierala 	 * of error.
908321b17ecSEdward Tomasz Napierala 	 */
909321b17ecSEdward Tomasz Napierala 	available = sbspace(&so->so_snd);
910321b17ecSEdward Tomasz Napierala 
911321b17ecSEdward Tomasz Napierala 	/*
912321b17ecSEdward Tomasz Napierala 	 * Notify the socket upcall that we don't need wakeups
913321b17ecSEdward Tomasz Napierala 	 * for the time being.
914321b17ecSEdward Tomasz Napierala 	 */
915321b17ecSEdward Tomasz Napierala 	so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1;
916321b17ecSEdward Tomasz Napierala 	SOCKBUF_UNLOCK(&so->so_snd);
917321b17ecSEdward Tomasz Napierala 
918321b17ecSEdward Tomasz Napierala 	while (!STAILQ_EMPTY(queue)) {
919321b17ecSEdward Tomasz Napierala 		request = STAILQ_FIRST(queue);
920321b17ecSEdward Tomasz Napierala 		size = icl_pdu_size(request);
921321b17ecSEdward Tomasz Napierala 		if (available < size) {
922321b17ecSEdward Tomasz Napierala 
923321b17ecSEdward Tomasz Napierala 			/*
924321b17ecSEdward Tomasz Napierala 			 * Set the low watermark, to be checked by
925321b17ecSEdward Tomasz Napierala 			 * sowriteable() in icl_soupcall_send()
926266078c6SPedro F. Giffuni 			 * to avoid unnecessary wakeups until there
927321b17ecSEdward Tomasz Napierala 			 * is enough space for the PDU to fit.
928321b17ecSEdward Tomasz Napierala 			 */
929321b17ecSEdward Tomasz Napierala 			SOCKBUF_LOCK(&so->so_snd);
930321b17ecSEdward Tomasz Napierala 			available = sbspace(&so->so_snd);
931321b17ecSEdward Tomasz Napierala 			if (available < size) {
932321b17ecSEdward Tomasz Napierala #if 1
933321b17ecSEdward Tomasz Napierala 				ICL_DEBUG("no space to send; "
934321b17ecSEdward Tomasz Napierala 				    "have %zd, need %zd",
935321b17ecSEdward Tomasz Napierala 				    available, size);
936321b17ecSEdward Tomasz Napierala #endif
937321b17ecSEdward Tomasz Napierala 				so->so_snd.sb_lowat = size;
938321b17ecSEdward Tomasz Napierala 				SOCKBUF_UNLOCK(&so->so_snd);
939321b17ecSEdward Tomasz Napierala 				return;
940321b17ecSEdward Tomasz Napierala 			}
941321b17ecSEdward Tomasz Napierala 			SOCKBUF_UNLOCK(&so->so_snd);
942321b17ecSEdward Tomasz Napierala 		}
943321b17ecSEdward Tomasz Napierala 		STAILQ_REMOVE_HEAD(queue, ip_next);
944321b17ecSEdward Tomasz Napierala 		error = icl_pdu_finalize(request);
945321b17ecSEdward Tomasz Napierala 		if (error != 0) {
946321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to finalize PDU; "
947321b17ecSEdward Tomasz Napierala 			    "dropping connection");
948321b17ecSEdward Tomasz Napierala 			icl_conn_fail(ic);
949321b17ecSEdward Tomasz Napierala 			icl_pdu_free(request);
950321b17ecSEdward Tomasz Napierala 			return;
951321b17ecSEdward Tomasz Napierala 		}
952321b17ecSEdward Tomasz Napierala 		if (coalesce) {
953321b17ecSEdward Tomasz Napierala 			coalesced = 1;
954321b17ecSEdward Tomasz Napierala 			for (;;) {
955321b17ecSEdward Tomasz Napierala 				request2 = STAILQ_FIRST(queue);
956321b17ecSEdward Tomasz Napierala 				if (request2 == NULL)
957321b17ecSEdward Tomasz Napierala 					break;
958321b17ecSEdward Tomasz Napierala 				size2 = icl_pdu_size(request2);
959321b17ecSEdward Tomasz Napierala 				if (available < size + size2)
960321b17ecSEdward Tomasz Napierala 					break;
961321b17ecSEdward Tomasz Napierala 				STAILQ_REMOVE_HEAD(queue, ip_next);
962321b17ecSEdward Tomasz Napierala 				error = icl_pdu_finalize(request2);
963321b17ecSEdward Tomasz Napierala 				if (error != 0) {
964321b17ecSEdward Tomasz Napierala 					ICL_DEBUG("failed to finalize PDU; "
965321b17ecSEdward Tomasz Napierala 					    "dropping connection");
966321b17ecSEdward Tomasz Napierala 					icl_conn_fail(ic);
967321b17ecSEdward Tomasz Napierala 					icl_pdu_free(request);
968321b17ecSEdward Tomasz Napierala 					icl_pdu_free(request2);
969321b17ecSEdward Tomasz Napierala 					return;
970321b17ecSEdward Tomasz Napierala 				}
971321b17ecSEdward Tomasz Napierala 				m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf);
972321b17ecSEdward Tomasz Napierala 				request2->ip_bhs_mbuf = NULL;
973321b17ecSEdward Tomasz Napierala 				request->ip_bhs_mbuf->m_pkthdr.len += size2;
974321b17ecSEdward Tomasz Napierala 				size += size2;
975321b17ecSEdward Tomasz Napierala 				STAILQ_REMOVE_AFTER(queue, request, ip_next);
976321b17ecSEdward Tomasz Napierala 				icl_pdu_free(request2);
977321b17ecSEdward Tomasz Napierala 				coalesced++;
978321b17ecSEdward Tomasz Napierala 			}
979321b17ecSEdward Tomasz Napierala #if 0
980321b17ecSEdward Tomasz Napierala 			if (coalesced > 1) {
981321b17ecSEdward Tomasz Napierala 				ICL_DEBUG("coalesced %d PDUs into %zd bytes",
982321b17ecSEdward Tomasz Napierala 				    coalesced, size);
983321b17ecSEdward Tomasz Napierala 			}
984321b17ecSEdward Tomasz Napierala #endif
985321b17ecSEdward Tomasz Napierala 		}
986321b17ecSEdward Tomasz Napierala 		available -= size;
987321b17ecSEdward Tomasz Napierala 		error = sosend(so, NULL, NULL, request->ip_bhs_mbuf,
988321b17ecSEdward Tomasz Napierala 		    NULL, MSG_DONTWAIT, curthread);
989321b17ecSEdward Tomasz Napierala 		request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */
990321b17ecSEdward Tomasz Napierala 		if (error != 0) {
991321b17ecSEdward Tomasz Napierala 			ICL_DEBUG("failed to send PDU, error %d; "
992321b17ecSEdward Tomasz Napierala 			    "dropping connection", error);
993321b17ecSEdward Tomasz Napierala 			icl_conn_fail(ic);
994321b17ecSEdward Tomasz Napierala 			icl_pdu_free(request);
995321b17ecSEdward Tomasz Napierala 			return;
996321b17ecSEdward Tomasz Napierala 		}
997321b17ecSEdward Tomasz Napierala 		icl_pdu_free(request);
998321b17ecSEdward Tomasz Napierala 	}
999321b17ecSEdward Tomasz Napierala }
1000321b17ecSEdward Tomasz Napierala 
1001321b17ecSEdward Tomasz Napierala static void
1002321b17ecSEdward Tomasz Napierala icl_send_thread(void *arg)
1003321b17ecSEdward Tomasz Napierala {
1004321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
1005321b17ecSEdward Tomasz Napierala 	struct icl_pdu_stailq queue;
1006321b17ecSEdward Tomasz Napierala 
1007321b17ecSEdward Tomasz Napierala 	ic = arg;
1008321b17ecSEdward Tomasz Napierala 
1009321b17ecSEdward Tomasz Napierala 	STAILQ_INIT(&queue);
1010321b17ecSEdward Tomasz Napierala 
1011321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
1012321b17ecSEdward Tomasz Napierala 	for (;;) {
1013321b17ecSEdward Tomasz Napierala 		for (;;) {
1014321b17ecSEdward Tomasz Napierala 			/*
1015321b17ecSEdward Tomasz Napierala 			 * If the local queue is empty, populate it from
1016321b17ecSEdward Tomasz Napierala 			 * the main one.  This way the icl_conn_send_pdus()
1017321b17ecSEdward Tomasz Napierala 			 * can go through all the queued PDUs without holding
1018321b17ecSEdward Tomasz Napierala 			 * any locks.
1019321b17ecSEdward Tomasz Napierala 			 */
1020321b17ecSEdward Tomasz Napierala 			if (STAILQ_EMPTY(&queue))
1021321b17ecSEdward Tomasz Napierala 				STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu);
1022321b17ecSEdward Tomasz Napierala 
1023321b17ecSEdward Tomasz Napierala 			ic->ic_check_send_space = false;
1024321b17ecSEdward Tomasz Napierala 			ICL_CONN_UNLOCK(ic);
1025321b17ecSEdward Tomasz Napierala 			icl_conn_send_pdus(ic, &queue);
1026321b17ecSEdward Tomasz Napierala 			ICL_CONN_LOCK(ic);
1027321b17ecSEdward Tomasz Napierala 
1028321b17ecSEdward Tomasz Napierala 			/*
1029321b17ecSEdward Tomasz Napierala 			 * The icl_soupcall_send() was called since the last
1030321b17ecSEdward Tomasz Napierala 			 * call to sbspace(); go around;
1031321b17ecSEdward Tomasz Napierala 			 */
1032321b17ecSEdward Tomasz Napierala 			if (ic->ic_check_send_space)
1033321b17ecSEdward Tomasz Napierala 				continue;
1034321b17ecSEdward Tomasz Napierala 
1035321b17ecSEdward Tomasz Napierala 			/*
1036321b17ecSEdward Tomasz Napierala 			 * Local queue is empty, but we still have PDUs
1037321b17ecSEdward Tomasz Napierala 			 * in the main one; go around.
1038321b17ecSEdward Tomasz Napierala 			 */
1039321b17ecSEdward Tomasz Napierala 			if (STAILQ_EMPTY(&queue) &&
1040321b17ecSEdward Tomasz Napierala 			    !STAILQ_EMPTY(&ic->ic_to_send))
1041321b17ecSEdward Tomasz Napierala 				continue;
1042321b17ecSEdward Tomasz Napierala 
1043321b17ecSEdward Tomasz Napierala 			/*
1044321b17ecSEdward Tomasz Napierala 			 * There might be some stuff in the local queue,
1045321b17ecSEdward Tomasz Napierala 			 * which didn't get sent due to not having enough send
1046321b17ecSEdward Tomasz Napierala 			 * space.  Wait for socket upcall.
1047321b17ecSEdward Tomasz Napierala 			 */
1048321b17ecSEdward Tomasz Napierala 			break;
1049321b17ecSEdward Tomasz Napierala 		}
1050321b17ecSEdward Tomasz Napierala 
1051321b17ecSEdward Tomasz Napierala 		if (ic->ic_disconnecting) {
1052321b17ecSEdward Tomasz Napierala 			//ICL_DEBUG("terminating");
1053321b17ecSEdward Tomasz Napierala 			break;
1054321b17ecSEdward Tomasz Napierala 		}
1055321b17ecSEdward Tomasz Napierala 
1056321b17ecSEdward Tomasz Napierala 		cv_wait(&ic->ic_send_cv, ic->ic_lock);
1057321b17ecSEdward Tomasz Napierala 	}
1058321b17ecSEdward Tomasz Napierala 
1059321b17ecSEdward Tomasz Napierala 	/*
1060321b17ecSEdward Tomasz Napierala 	 * We're exiting; move PDUs back to the main queue, so they can
1061321b17ecSEdward Tomasz Napierala 	 * get freed properly.  At this point ordering doesn't matter.
1062321b17ecSEdward Tomasz Napierala 	 */
1063321b17ecSEdward Tomasz Napierala 	STAILQ_CONCAT(&ic->ic_to_send, &queue);
1064321b17ecSEdward Tomasz Napierala 
1065321b17ecSEdward Tomasz Napierala 	ic->ic_send_running = false;
1066321b17ecSEdward Tomasz Napierala 	cv_signal(&ic->ic_send_cv);
1067321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
1068321b17ecSEdward Tomasz Napierala 	kthread_exit();
1069321b17ecSEdward Tomasz Napierala }
1070321b17ecSEdward Tomasz Napierala 
1071321b17ecSEdward Tomasz Napierala static int
1072321b17ecSEdward Tomasz Napierala icl_soupcall_send(struct socket *so, void *arg, int waitflag)
1073321b17ecSEdward Tomasz Napierala {
1074321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
1075321b17ecSEdward Tomasz Napierala 
1076321b17ecSEdward Tomasz Napierala 	if (!sowriteable(so))
1077321b17ecSEdward Tomasz Napierala 		return (SU_OK);
1078321b17ecSEdward Tomasz Napierala 
1079321b17ecSEdward Tomasz Napierala 	ic = arg;
1080321b17ecSEdward Tomasz Napierala 
1081321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
1082321b17ecSEdward Tomasz Napierala 	ic->ic_check_send_space = true;
1083321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
1084321b17ecSEdward Tomasz Napierala 
1085321b17ecSEdward Tomasz Napierala 	cv_signal(&ic->ic_send_cv);
1086321b17ecSEdward Tomasz Napierala 
1087321b17ecSEdward Tomasz Napierala 	return (SU_OK);
1088321b17ecSEdward Tomasz Napierala }
1089321b17ecSEdward Tomasz Napierala 
1090321b17ecSEdward Tomasz Napierala static int
1091321b17ecSEdward Tomasz Napierala icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len,
1092321b17ecSEdward Tomasz Napierala     int flags)
1093321b17ecSEdward Tomasz Napierala {
1094321b17ecSEdward Tomasz Napierala 	struct mbuf *mb, *newmb;
1095321b17ecSEdward Tomasz Napierala 	size_t copylen, off = 0;
1096321b17ecSEdward Tomasz Napierala 
1097321b17ecSEdward Tomasz Napierala 	KASSERT(len > 0, ("len == 0"));
1098321b17ecSEdward Tomasz Napierala 
1099898fd11fSAlexander Motin 	newmb = m_getm2(NULL, len, flags, MT_DATA, 0);
1100321b17ecSEdward Tomasz Napierala 	if (newmb == NULL) {
1101321b17ecSEdward Tomasz Napierala 		ICL_WARN("failed to allocate mbuf for %zd bytes", len);
1102321b17ecSEdward Tomasz Napierala 		return (ENOMEM);
1103321b17ecSEdward Tomasz Napierala 	}
1104321b17ecSEdward Tomasz Napierala 
1105321b17ecSEdward Tomasz Napierala 	for (mb = newmb; mb != NULL; mb = mb->m_next) {
1106321b17ecSEdward Tomasz Napierala 		copylen = min(M_TRAILINGSPACE(mb), len - off);
1107321b17ecSEdward Tomasz Napierala 		memcpy(mtod(mb, char *), (const char *)addr + off, copylen);
1108321b17ecSEdward Tomasz Napierala 		mb->m_len = copylen;
1109321b17ecSEdward Tomasz Napierala 		off += copylen;
1110321b17ecSEdward Tomasz Napierala 	}
1111321b17ecSEdward Tomasz Napierala 	KASSERT(off == len, ("%s: off != len", __func__));
1112321b17ecSEdward Tomasz Napierala 
1113321b17ecSEdward Tomasz Napierala 	if (request->ip_data_mbuf == NULL) {
1114321b17ecSEdward Tomasz Napierala 		request->ip_data_mbuf = newmb;
1115321b17ecSEdward Tomasz Napierala 		request->ip_data_len = len;
1116321b17ecSEdward Tomasz Napierala 	} else {
1117321b17ecSEdward Tomasz Napierala 		m_cat(request->ip_data_mbuf, newmb);
1118321b17ecSEdward Tomasz Napierala 		request->ip_data_len += len;
1119321b17ecSEdward Tomasz Napierala 	}
1120321b17ecSEdward Tomasz Napierala 
1121321b17ecSEdward Tomasz Napierala 	return (0);
1122321b17ecSEdward Tomasz Napierala }
1123321b17ecSEdward Tomasz Napierala 
1124321b17ecSEdward Tomasz Napierala int
1125321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request,
1126321b17ecSEdward Tomasz Napierala     const void *addr, size_t len, int flags)
1127321b17ecSEdward Tomasz Napierala {
1128321b17ecSEdward Tomasz Napierala 
1129321b17ecSEdward Tomasz Napierala 	return (icl_pdu_append_data(request, addr, len, flags));
1130321b17ecSEdward Tomasz Napierala }
1131321b17ecSEdward Tomasz Napierala 
1132321b17ecSEdward Tomasz Napierala static void
1133321b17ecSEdward Tomasz Napierala icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len)
1134321b17ecSEdward Tomasz Napierala {
1135321b17ecSEdward Tomasz Napierala 
1136321b17ecSEdward Tomasz Napierala 	m_copydata(ip->ip_data_mbuf, off, len, addr);
1137321b17ecSEdward Tomasz Napierala }
1138321b17ecSEdward Tomasz Napierala 
1139321b17ecSEdward Tomasz Napierala void
1140321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
1141321b17ecSEdward Tomasz Napierala     size_t off, void *addr, size_t len)
1142321b17ecSEdward Tomasz Napierala {
1143321b17ecSEdward Tomasz Napierala 
1144321b17ecSEdward Tomasz Napierala 	return (icl_pdu_get_data(ip, off, addr, len));
1145321b17ecSEdward Tomasz Napierala }
1146321b17ecSEdward Tomasz Napierala 
1147321b17ecSEdward Tomasz Napierala static void
1148321b17ecSEdward Tomasz Napierala icl_pdu_queue(struct icl_pdu *ip)
1149321b17ecSEdward Tomasz Napierala {
1150321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
1151321b17ecSEdward Tomasz Napierala 
1152321b17ecSEdward Tomasz Napierala 	ic = ip->ip_conn;
1153321b17ecSEdward Tomasz Napierala 
1154321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK_ASSERT(ic);
1155321b17ecSEdward Tomasz Napierala 
1156321b17ecSEdward Tomasz Napierala 	if (ic->ic_disconnecting || ic->ic_socket == NULL) {
1157321b17ecSEdward Tomasz Napierala 		ICL_DEBUG("icl_pdu_queue on closed connection");
1158321b17ecSEdward Tomasz Napierala 		icl_pdu_free(ip);
1159321b17ecSEdward Tomasz Napierala 		return;
1160321b17ecSEdward Tomasz Napierala 	}
1161321b17ecSEdward Tomasz Napierala 
1162321b17ecSEdward Tomasz Napierala 	if (!STAILQ_EMPTY(&ic->ic_to_send)) {
1163321b17ecSEdward Tomasz Napierala 		STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next);
1164321b17ecSEdward Tomasz Napierala 		/*
1165321b17ecSEdward Tomasz Napierala 		 * If the queue is not empty, someone else had already
1166321b17ecSEdward Tomasz Napierala 		 * signaled the send thread; no need to do that again,
1167321b17ecSEdward Tomasz Napierala 		 * just return.
1168321b17ecSEdward Tomasz Napierala 		 */
1169321b17ecSEdward Tomasz Napierala 		return;
1170321b17ecSEdward Tomasz Napierala 	}
1171321b17ecSEdward Tomasz Napierala 
1172321b17ecSEdward Tomasz Napierala 	STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next);
1173321b17ecSEdward Tomasz Napierala 	cv_signal(&ic->ic_send_cv);
1174321b17ecSEdward Tomasz Napierala }
1175321b17ecSEdward Tomasz Napierala 
1176321b17ecSEdward Tomasz Napierala void
1177321b17ecSEdward Tomasz Napierala icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
1178321b17ecSEdward Tomasz Napierala {
1179321b17ecSEdward Tomasz Napierala 
1180321b17ecSEdward Tomasz Napierala 	icl_pdu_queue(ip);
1181321b17ecSEdward Tomasz Napierala }
1182321b17ecSEdward Tomasz Napierala 
1183321b17ecSEdward Tomasz Napierala static struct icl_conn *
1184321b17ecSEdward Tomasz Napierala icl_soft_new_conn(const char *name, struct mtx *lock)
1185321b17ecSEdward Tomasz Napierala {
1186321b17ecSEdward Tomasz Napierala 	struct icl_conn *ic;
1187321b17ecSEdward Tomasz Napierala 
1188321b17ecSEdward Tomasz Napierala 	refcount_acquire(&icl_ncons);
1189321b17ecSEdward Tomasz Napierala 
1190321b17ecSEdward Tomasz Napierala 	ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO);
1191321b17ecSEdward Tomasz Napierala 
1192321b17ecSEdward Tomasz Napierala 	STAILQ_INIT(&ic->ic_to_send);
1193321b17ecSEdward Tomasz Napierala 	ic->ic_lock = lock;
1194321b17ecSEdward Tomasz Napierala 	cv_init(&ic->ic_send_cv, "icl_tx");
1195321b17ecSEdward Tomasz Napierala 	cv_init(&ic->ic_receive_cv, "icl_rx");
1196321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC
1197321b17ecSEdward Tomasz Napierala 	refcount_init(&ic->ic_outstanding_pdus, 0);
1198321b17ecSEdward Tomasz Napierala #endif
1199321b17ecSEdward Tomasz Napierala 	ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH;
1200321b17ecSEdward Tomasz Napierala 	ic->ic_name = name;
1201d4b195d3SEdward Tomasz Napierala 	ic->ic_offload = "None";
12027deb68abSEdward Tomasz Napierala 	ic->ic_unmapped = false;
1203321b17ecSEdward Tomasz Napierala 
1204321b17ecSEdward Tomasz Napierala 	return (ic);
1205321b17ecSEdward Tomasz Napierala }
1206321b17ecSEdward Tomasz Napierala 
1207321b17ecSEdward Tomasz Napierala void
1208321b17ecSEdward Tomasz Napierala icl_soft_conn_free(struct icl_conn *ic)
1209321b17ecSEdward Tomasz Napierala {
1210321b17ecSEdward Tomasz Napierala 
1211321b17ecSEdward Tomasz Napierala 	cv_destroy(&ic->ic_send_cv);
1212321b17ecSEdward Tomasz Napierala 	cv_destroy(&ic->ic_receive_cv);
1213321b17ecSEdward Tomasz Napierala 	kobj_delete((struct kobj *)ic, M_ICL_SOFT);
1214321b17ecSEdward Tomasz Napierala 	refcount_release(&icl_ncons);
1215321b17ecSEdward Tomasz Napierala }
1216321b17ecSEdward Tomasz Napierala 
1217321b17ecSEdward Tomasz Napierala static int
1218321b17ecSEdward Tomasz Napierala icl_conn_start(struct icl_conn *ic)
1219321b17ecSEdward Tomasz Napierala {
1220321b17ecSEdward Tomasz Napierala 	size_t minspace;
1221321b17ecSEdward Tomasz Napierala 	struct sockopt opt;
1222321b17ecSEdward Tomasz Napierala 	int error, one = 1;
1223321b17ecSEdward Tomasz Napierala 
1224321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
1225321b17ecSEdward Tomasz Napierala 
1226321b17ecSEdward Tomasz Napierala 	/*
1227321b17ecSEdward Tomasz Napierala 	 * XXX: Ugly hack.
1228321b17ecSEdward Tomasz Napierala 	 */
1229321b17ecSEdward Tomasz Napierala 	if (ic->ic_socket == NULL) {
1230321b17ecSEdward Tomasz Napierala 		ICL_CONN_UNLOCK(ic);
1231321b17ecSEdward Tomasz Napierala 		return (EINVAL);
1232321b17ecSEdward Tomasz Napierala 	}
1233321b17ecSEdward Tomasz Napierala 
1234321b17ecSEdward Tomasz Napierala 	ic->ic_receive_state = ICL_CONN_STATE_BHS;
1235321b17ecSEdward Tomasz Napierala 	ic->ic_receive_len = sizeof(struct iscsi_bhs);
1236321b17ecSEdward Tomasz Napierala 	ic->ic_disconnecting = false;
1237321b17ecSEdward Tomasz Napierala 
1238321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
1239321b17ecSEdward Tomasz Napierala 
1240321b17ecSEdward Tomasz Napierala 	/*
1241321b17ecSEdward Tomasz Napierala 	 * For sendspace, this is required because the current code cannot
1242321b17ecSEdward Tomasz Napierala 	 * send a PDU in pieces; thus, the minimum buffer size is equal
1243321b17ecSEdward Tomasz Napierala 	 * to the maximum PDU size.  "+4" is to account for possible padding.
1244321b17ecSEdward Tomasz Napierala 	 *
1245321b17ecSEdward Tomasz Napierala 	 * What we should actually do here is to use autoscaling, but set
1246321b17ecSEdward Tomasz Napierala 	 * some minimal buffer size to "minspace".  I don't know a way to do
1247321b17ecSEdward Tomasz Napierala 	 * that, though.
1248321b17ecSEdward Tomasz Napierala 	 */
1249321b17ecSEdward Tomasz Napierala 	minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length +
1250321b17ecSEdward Tomasz Napierala 	    ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4;
1251321b17ecSEdward Tomasz Napierala 	if (sendspace < minspace) {
1252321b17ecSEdward Tomasz Napierala 		ICL_WARN("kern.icl.sendspace too low; must be at least %zd",
1253321b17ecSEdward Tomasz Napierala 		    minspace);
1254321b17ecSEdward Tomasz Napierala 		sendspace = minspace;
1255321b17ecSEdward Tomasz Napierala 	}
1256321b17ecSEdward Tomasz Napierala 	if (recvspace < minspace) {
1257321b17ecSEdward Tomasz Napierala 		ICL_WARN("kern.icl.recvspace too low; must be at least %zd",
1258321b17ecSEdward Tomasz Napierala 		    minspace);
1259321b17ecSEdward Tomasz Napierala 		recvspace = minspace;
1260321b17ecSEdward Tomasz Napierala 	}
1261321b17ecSEdward Tomasz Napierala 
1262321b17ecSEdward Tomasz Napierala 	error = soreserve(ic->ic_socket, sendspace, recvspace);
1263321b17ecSEdward Tomasz Napierala 	if (error != 0) {
1264321b17ecSEdward Tomasz Napierala 		ICL_WARN("soreserve failed with error %d", error);
12655aabcd7cSEdward Tomasz Napierala 		icl_soft_conn_close(ic);
1266321b17ecSEdward Tomasz Napierala 		return (error);
1267321b17ecSEdward Tomasz Napierala 	}
1268321b17ecSEdward Tomasz Napierala 	ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE;
1269321b17ecSEdward Tomasz Napierala 	ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE;
1270321b17ecSEdward Tomasz Napierala 
1271321b17ecSEdward Tomasz Napierala 	/*
1272321b17ecSEdward Tomasz Napierala 	 * Disable Nagle.
1273321b17ecSEdward Tomasz Napierala 	 */
1274321b17ecSEdward Tomasz Napierala 	bzero(&opt, sizeof(opt));
1275321b17ecSEdward Tomasz Napierala 	opt.sopt_dir = SOPT_SET;
1276321b17ecSEdward Tomasz Napierala 	opt.sopt_level = IPPROTO_TCP;
1277321b17ecSEdward Tomasz Napierala 	opt.sopt_name = TCP_NODELAY;
1278321b17ecSEdward Tomasz Napierala 	opt.sopt_val = &one;
1279321b17ecSEdward Tomasz Napierala 	opt.sopt_valsize = sizeof(one);
1280321b17ecSEdward Tomasz Napierala 	error = sosetopt(ic->ic_socket, &opt);
1281321b17ecSEdward Tomasz Napierala 	if (error != 0) {
1282321b17ecSEdward Tomasz Napierala 		ICL_WARN("disabling TCP_NODELAY failed with error %d", error);
12835aabcd7cSEdward Tomasz Napierala 		icl_soft_conn_close(ic);
1284321b17ecSEdward Tomasz Napierala 		return (error);
1285321b17ecSEdward Tomasz Napierala 	}
1286321b17ecSEdward Tomasz Napierala 
1287321b17ecSEdward Tomasz Napierala 	/*
1288321b17ecSEdward Tomasz Napierala 	 * Register socket upcall, to get notified about incoming PDUs
1289321b17ecSEdward Tomasz Napierala 	 * and free space to send outgoing ones.
1290321b17ecSEdward Tomasz Napierala 	 */
1291321b17ecSEdward Tomasz Napierala 	SOCKBUF_LOCK(&ic->ic_socket->so_snd);
1292321b17ecSEdward Tomasz Napierala 	soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic);
1293321b17ecSEdward Tomasz Napierala 	SOCKBUF_UNLOCK(&ic->ic_socket->so_snd);
1294321b17ecSEdward Tomasz Napierala 	SOCKBUF_LOCK(&ic->ic_socket->so_rcv);
1295321b17ecSEdward Tomasz Napierala 	soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic);
1296321b17ecSEdward Tomasz Napierala 	SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv);
1297321b17ecSEdward Tomasz Napierala 
12985b157f21SAlexander Motin 	/*
12995b157f21SAlexander Motin 	 * Start threads.
13005b157f21SAlexander Motin 	 */
13015b157f21SAlexander Motin 	ICL_CONN_LOCK(ic);
13025b157f21SAlexander Motin 	ic->ic_send_running = ic->ic_receive_running = true;
13035b157f21SAlexander Motin 	ICL_CONN_UNLOCK(ic);
13045b157f21SAlexander Motin 	error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx",
13055b157f21SAlexander Motin 	    ic->ic_name);
13065b157f21SAlexander Motin 	if (error != 0) {
13075b157f21SAlexander Motin 		ICL_WARN("kthread_add(9) failed with error %d", error);
13085b157f21SAlexander Motin 		ICL_CONN_LOCK(ic);
13095b157f21SAlexander Motin 		ic->ic_send_running = ic->ic_receive_running = false;
13105b157f21SAlexander Motin 		cv_signal(&ic->ic_send_cv);
13115b157f21SAlexander Motin 		ICL_CONN_UNLOCK(ic);
13125b157f21SAlexander Motin 		icl_soft_conn_close(ic);
13135b157f21SAlexander Motin 		return (error);
13145b157f21SAlexander Motin 	}
13155b157f21SAlexander Motin 	error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx",
13165b157f21SAlexander Motin 	    ic->ic_name);
13175b157f21SAlexander Motin 	if (error != 0) {
13185b157f21SAlexander Motin 		ICL_WARN("kthread_add(9) failed with error %d", error);
13195b157f21SAlexander Motin 		ICL_CONN_LOCK(ic);
13205b157f21SAlexander Motin 		ic->ic_receive_running = false;
13215b157f21SAlexander Motin 		cv_signal(&ic->ic_send_cv);
13225b157f21SAlexander Motin 		ICL_CONN_UNLOCK(ic);
13235b157f21SAlexander Motin 		icl_soft_conn_close(ic);
13245b157f21SAlexander Motin 		return (error);
13255b157f21SAlexander Motin 	}
13265b157f21SAlexander Motin 
1327321b17ecSEdward Tomasz Napierala 	return (0);
1328321b17ecSEdward Tomasz Napierala }
1329321b17ecSEdward Tomasz Napierala 
1330321b17ecSEdward Tomasz Napierala int
1331321b17ecSEdward Tomasz Napierala icl_soft_conn_handoff(struct icl_conn *ic, int fd)
1332321b17ecSEdward Tomasz Napierala {
1333321b17ecSEdward Tomasz Napierala 	struct file *fp;
1334321b17ecSEdward Tomasz Napierala 	struct socket *so;
1335321b17ecSEdward Tomasz Napierala 	cap_rights_t rights;
1336321b17ecSEdward Tomasz Napierala 	int error;
1337321b17ecSEdward Tomasz Napierala 
1338321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1339321b17ecSEdward Tomasz Napierala 
1340906a424bSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY
1341906a424bSEdward Tomasz Napierala 	/*
1342906a424bSEdward Tomasz Napierala 	 * We're transitioning to Full Feature phase, and we don't
1343906a424bSEdward Tomasz Napierala 	 * really care.
1344906a424bSEdward Tomasz Napierala 	 */
1345906a424bSEdward Tomasz Napierala 	if (fd == 0) {
1346906a424bSEdward Tomasz Napierala 		ICL_CONN_LOCK(ic);
1347906a424bSEdward Tomasz Napierala 		if (ic->ic_socket == NULL) {
1348906a424bSEdward Tomasz Napierala 			ICL_CONN_UNLOCK(ic);
1349906a424bSEdward Tomasz Napierala 			ICL_WARN("proxy handoff without connect");
1350906a424bSEdward Tomasz Napierala 			return (EINVAL);
1351906a424bSEdward Tomasz Napierala 		}
1352906a424bSEdward Tomasz Napierala 		ICL_CONN_UNLOCK(ic);
1353906a424bSEdward Tomasz Napierala 		return (0);
1354906a424bSEdward Tomasz Napierala 	}
1355906a424bSEdward Tomasz Napierala #endif
1356906a424bSEdward Tomasz Napierala 
1357321b17ecSEdward Tomasz Napierala 	/*
1358321b17ecSEdward Tomasz Napierala 	 * Steal the socket from userland.
1359321b17ecSEdward Tomasz Napierala 	 */
1360321b17ecSEdward Tomasz Napierala 	error = fget(curthread, fd,
1361321b17ecSEdward Tomasz Napierala 	    cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp);
1362321b17ecSEdward Tomasz Napierala 	if (error != 0)
1363321b17ecSEdward Tomasz Napierala 		return (error);
1364321b17ecSEdward Tomasz Napierala 	if (fp->f_type != DTYPE_SOCKET) {
1365321b17ecSEdward Tomasz Napierala 		fdrop(fp, curthread);
1366321b17ecSEdward Tomasz Napierala 		return (EINVAL);
1367321b17ecSEdward Tomasz Napierala 	}
1368321b17ecSEdward Tomasz Napierala 	so = fp->f_data;
1369321b17ecSEdward Tomasz Napierala 	if (so->so_type != SOCK_STREAM) {
1370321b17ecSEdward Tomasz Napierala 		fdrop(fp, curthread);
1371321b17ecSEdward Tomasz Napierala 		return (EINVAL);
1372321b17ecSEdward Tomasz Napierala 	}
1373321b17ecSEdward Tomasz Napierala 
1374321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
1375321b17ecSEdward Tomasz Napierala 
1376321b17ecSEdward Tomasz Napierala 	if (ic->ic_socket != NULL) {
1377321b17ecSEdward Tomasz Napierala 		ICL_CONN_UNLOCK(ic);
1378321b17ecSEdward Tomasz Napierala 		fdrop(fp, curthread);
1379321b17ecSEdward Tomasz Napierala 		return (EBUSY);
1380321b17ecSEdward Tomasz Napierala 	}
1381321b17ecSEdward Tomasz Napierala 
1382321b17ecSEdward Tomasz Napierala 	ic->ic_socket = fp->f_data;
1383321b17ecSEdward Tomasz Napierala 	fp->f_ops = &badfileops;
1384321b17ecSEdward Tomasz Napierala 	fp->f_data = NULL;
1385321b17ecSEdward Tomasz Napierala 	fdrop(fp, curthread);
1386321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
1387321b17ecSEdward Tomasz Napierala 
1388321b17ecSEdward Tomasz Napierala 	error = icl_conn_start(ic);
1389321b17ecSEdward Tomasz Napierala 
1390321b17ecSEdward Tomasz Napierala 	return (error);
1391321b17ecSEdward Tomasz Napierala }
1392321b17ecSEdward Tomasz Napierala 
1393321b17ecSEdward Tomasz Napierala void
13945aabcd7cSEdward Tomasz Napierala icl_soft_conn_close(struct icl_conn *ic)
1395321b17ecSEdward Tomasz Napierala {
1396321b17ecSEdward Tomasz Napierala 	struct icl_pdu *pdu;
13975b157f21SAlexander Motin 	struct socket *so;
1398321b17ecSEdward Tomasz Napierala 
1399321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
14005b157f21SAlexander Motin 
14015b157f21SAlexander Motin 	/*
14025b157f21SAlexander Motin 	 * Wake up the threads, so they can properly terminate.
14035b157f21SAlexander Motin 	 */
14045b157f21SAlexander Motin 	ic->ic_disconnecting = true;
14055b157f21SAlexander Motin 	while (ic->ic_receive_running || ic->ic_send_running) {
14065b157f21SAlexander Motin 		cv_signal(&ic->ic_receive_cv);
14075b157f21SAlexander Motin 		cv_signal(&ic->ic_send_cv);
14085b157f21SAlexander Motin 		cv_wait(&ic->ic_send_cv, ic->ic_lock);
14095b157f21SAlexander Motin 	}
14105b157f21SAlexander Motin 
14115b157f21SAlexander Motin 	/* Some other thread could close the connection same time. */
14125b157f21SAlexander Motin 	so = ic->ic_socket;
14135b157f21SAlexander Motin 	if (so == NULL) {
1414321b17ecSEdward Tomasz Napierala 		ICL_CONN_UNLOCK(ic);
1415321b17ecSEdward Tomasz Napierala 		return;
1416321b17ecSEdward Tomasz Napierala 	}
14175b157f21SAlexander Motin 	ic->ic_socket = NULL;
1418321b17ecSEdward Tomasz Napierala 
1419321b17ecSEdward Tomasz Napierala 	/*
1420321b17ecSEdward Tomasz Napierala 	 * Deregister socket upcalls.
1421321b17ecSEdward Tomasz Napierala 	 */
1422321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
14235b157f21SAlexander Motin 	SOCKBUF_LOCK(&so->so_snd);
14245b157f21SAlexander Motin 	if (so->so_snd.sb_upcall != NULL)
14255b157f21SAlexander Motin 		soupcall_clear(so, SO_SND);
14265b157f21SAlexander Motin 	SOCKBUF_UNLOCK(&so->so_snd);
14275b157f21SAlexander Motin 	SOCKBUF_LOCK(&so->so_rcv);
14285b157f21SAlexander Motin 	if (so->so_rcv.sb_upcall != NULL)
14295b157f21SAlexander Motin 		soupcall_clear(so, SO_RCV);
14305b157f21SAlexander Motin 	SOCKBUF_UNLOCK(&so->so_rcv);
14315b157f21SAlexander Motin 	soclose(so);
1432321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
1433321b17ecSEdward Tomasz Napierala 
1434321b17ecSEdward Tomasz Napierala 	if (ic->ic_receive_pdu != NULL) {
1435321b17ecSEdward Tomasz Napierala 		//ICL_DEBUG("freeing partially received PDU");
1436321b17ecSEdward Tomasz Napierala 		icl_pdu_free(ic->ic_receive_pdu);
1437321b17ecSEdward Tomasz Napierala 		ic->ic_receive_pdu = NULL;
1438321b17ecSEdward Tomasz Napierala 	}
1439321b17ecSEdward Tomasz Napierala 
1440321b17ecSEdward Tomasz Napierala 	/*
1441321b17ecSEdward Tomasz Napierala 	 * Remove any outstanding PDUs from the send queue.
1442321b17ecSEdward Tomasz Napierala 	 */
1443321b17ecSEdward Tomasz Napierala 	while (!STAILQ_EMPTY(&ic->ic_to_send)) {
1444321b17ecSEdward Tomasz Napierala 		pdu = STAILQ_FIRST(&ic->ic_to_send);
1445321b17ecSEdward Tomasz Napierala 		STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next);
1446321b17ecSEdward Tomasz Napierala 		icl_pdu_free(pdu);
1447321b17ecSEdward Tomasz Napierala 	}
1448321b17ecSEdward Tomasz Napierala 
1449321b17ecSEdward Tomasz Napierala 	KASSERT(STAILQ_EMPTY(&ic->ic_to_send),
1450321b17ecSEdward Tomasz Napierala 	    ("destroying session with non-empty send queue"));
1451321b17ecSEdward Tomasz Napierala #ifdef DIAGNOSTIC
1452321b17ecSEdward Tomasz Napierala 	KASSERT(ic->ic_outstanding_pdus == 0,
1453321b17ecSEdward Tomasz Napierala 	    ("destroying session with %d outstanding PDUs",
1454321b17ecSEdward Tomasz Napierala 	     ic->ic_outstanding_pdus));
1455321b17ecSEdward Tomasz Napierala #endif
1456321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
1457321b17ecSEdward Tomasz Napierala }
1458321b17ecSEdward Tomasz Napierala 
14597a03d007SEdward Tomasz Napierala int
1460604c023fSEdward Tomasz Napierala icl_soft_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
1461604c023fSEdward Tomasz Napierala     struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp)
14627a03d007SEdward Tomasz Napierala {
14637a03d007SEdward Tomasz Napierala 
14647a03d007SEdward Tomasz Napierala 	return (0);
14657a03d007SEdward Tomasz Napierala }
14667a03d007SEdward Tomasz Napierala 
14677a03d007SEdward Tomasz Napierala void
14687a03d007SEdward Tomasz Napierala icl_soft_conn_task_done(struct icl_conn *ic, void *prv)
14697a03d007SEdward Tomasz Napierala {
14707a03d007SEdward Tomasz Napierala }
14717a03d007SEdward Tomasz Napierala 
14727a03d007SEdward Tomasz Napierala int
14737a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
14747a03d007SEdward Tomasz Napierala     uint32_t *transfer_tag, void **prvp)
14757a03d007SEdward Tomasz Napierala {
14767a03d007SEdward Tomasz Napierala 
14777a03d007SEdward Tomasz Napierala 	return (0);
14787a03d007SEdward Tomasz Napierala }
14797a03d007SEdward Tomasz Napierala 
14807a03d007SEdward Tomasz Napierala void
14817a03d007SEdward Tomasz Napierala icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv)
14827a03d007SEdward Tomasz Napierala {
14837a03d007SEdward Tomasz Napierala }
14847a03d007SEdward Tomasz Napierala 
1485321b17ecSEdward Tomasz Napierala static int
148697b84d34SNavdeep Parhar icl_soft_limits(struct icl_drv_limits *idl)
1487321b17ecSEdward Tomasz Napierala {
1488321b17ecSEdward Tomasz Napierala 
148997b84d34SNavdeep Parhar 	idl->idl_max_recv_data_segment_length = 128 * 1024;
1490321b17ecSEdward Tomasz Napierala 
1491321b17ecSEdward Tomasz Napierala 	return (0);
1492321b17ecSEdward Tomasz Napierala }
1493321b17ecSEdward Tomasz Napierala 
1494321b17ecSEdward Tomasz Napierala #ifdef ICL_KERNEL_PROXY
1495321b17ecSEdward Tomasz Napierala int
1496f41492b0SEdward Tomasz Napierala icl_soft_conn_connect(struct icl_conn *ic, int domain, int socktype,
1497f41492b0SEdward Tomasz Napierala     int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa)
1498f41492b0SEdward Tomasz Napierala {
1499f41492b0SEdward Tomasz Napierala 
1500f41492b0SEdward Tomasz Napierala 	return (icl_soft_proxy_connect(ic, domain, socktype, protocol,
1501f41492b0SEdward Tomasz Napierala 	    from_sa, to_sa));
1502f41492b0SEdward Tomasz Napierala }
1503f41492b0SEdward Tomasz Napierala 
1504f41492b0SEdward Tomasz Napierala int
1505f41492b0SEdward Tomasz Napierala icl_soft_handoff_sock(struct icl_conn *ic, struct socket *so)
1506321b17ecSEdward Tomasz Napierala {
1507321b17ecSEdward Tomasz Napierala 	int error;
1508321b17ecSEdward Tomasz Napierala 
1509321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1510321b17ecSEdward Tomasz Napierala 
1511321b17ecSEdward Tomasz Napierala 	if (so->so_type != SOCK_STREAM)
1512321b17ecSEdward Tomasz Napierala 		return (EINVAL);
1513321b17ecSEdward Tomasz Napierala 
1514321b17ecSEdward Tomasz Napierala 	ICL_CONN_LOCK(ic);
1515321b17ecSEdward Tomasz Napierala 	if (ic->ic_socket != NULL) {
1516321b17ecSEdward Tomasz Napierala 		ICL_CONN_UNLOCK(ic);
1517321b17ecSEdward Tomasz Napierala 		return (EBUSY);
1518321b17ecSEdward Tomasz Napierala 	}
1519321b17ecSEdward Tomasz Napierala 	ic->ic_socket = so;
1520321b17ecSEdward Tomasz Napierala 	ICL_CONN_UNLOCK(ic);
1521321b17ecSEdward Tomasz Napierala 
1522321b17ecSEdward Tomasz Napierala 	error = icl_conn_start(ic);
1523321b17ecSEdward Tomasz Napierala 
1524321b17ecSEdward Tomasz Napierala 	return (error);
1525321b17ecSEdward Tomasz Napierala }
1526321b17ecSEdward Tomasz Napierala #endif /* ICL_KERNEL_PROXY */
1527321b17ecSEdward Tomasz Napierala 
1528321b17ecSEdward Tomasz Napierala static int
1529321b17ecSEdward Tomasz Napierala icl_soft_load(void)
1530321b17ecSEdward Tomasz Napierala {
1531321b17ecSEdward Tomasz Napierala 	int error;
1532321b17ecSEdward Tomasz Napierala 
1533321b17ecSEdward Tomasz Napierala 	icl_pdu_zone = uma_zcreate("icl_pdu",
1534321b17ecSEdward Tomasz Napierala 	    sizeof(struct icl_pdu), NULL, NULL, NULL, NULL,
1535321b17ecSEdward Tomasz Napierala 	    UMA_ALIGN_PTR, 0);
1536321b17ecSEdward Tomasz Napierala 	refcount_init(&icl_ncons, 0);
1537321b17ecSEdward Tomasz Napierala 
1538321b17ecSEdward Tomasz Napierala 	/*
1539321b17ecSEdward Tomasz Napierala 	 * The reason we call this "none" is that to the user,
1540321b17ecSEdward Tomasz Napierala 	 * it's known as "offload driver"; "offload driver: soft"
1541321b17ecSEdward Tomasz Napierala 	 * doesn't make much sense.
1542321b17ecSEdward Tomasz Napierala 	 */
1543b8911594SEdward Tomasz Napierala 	error = icl_register("none", false, 0,
1544b8911594SEdward Tomasz Napierala 	    icl_soft_limits, icl_soft_new_conn);
1545321b17ecSEdward Tomasz Napierala 	KASSERT(error == 0, ("failed to register"));
1546321b17ecSEdward Tomasz Napierala 
1547b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0
1548b8911594SEdward Tomasz Napierala 	/*
1549b8911594SEdward Tomasz Napierala 	 * Debugging aid for kernel proxy functionality.
1550b8911594SEdward Tomasz Napierala 	 */
1551b8911594SEdward Tomasz Napierala 	error = icl_register("proxytest", true, 0,
1552b8911594SEdward Tomasz Napierala 	    icl_soft_limits, icl_soft_new_conn);
1553b8911594SEdward Tomasz Napierala 	KASSERT(error == 0, ("failed to register"));
1554b8911594SEdward Tomasz Napierala #endif
1555b8911594SEdward Tomasz Napierala 
1556321b17ecSEdward Tomasz Napierala 	return (error);
1557321b17ecSEdward Tomasz Napierala }
1558321b17ecSEdward Tomasz Napierala 
1559321b17ecSEdward Tomasz Napierala static int
1560321b17ecSEdward Tomasz Napierala icl_soft_unload(void)
1561321b17ecSEdward Tomasz Napierala {
1562321b17ecSEdward Tomasz Napierala 
1563321b17ecSEdward Tomasz Napierala 	if (icl_ncons != 0)
1564321b17ecSEdward Tomasz Napierala 		return (EBUSY);
1565321b17ecSEdward Tomasz Napierala 
1566b8911594SEdward Tomasz Napierala 	icl_unregister("none", false);
1567b8911594SEdward Tomasz Napierala #if defined(ICL_KERNEL_PROXY) && 0
1568b8911594SEdward Tomasz Napierala 	icl_unregister("proxytest", true);
1569b8911594SEdward Tomasz Napierala #endif
1570321b17ecSEdward Tomasz Napierala 
1571321b17ecSEdward Tomasz Napierala 	uma_zdestroy(icl_pdu_zone);
1572321b17ecSEdward Tomasz Napierala 
1573321b17ecSEdward Tomasz Napierala 	return (0);
1574321b17ecSEdward Tomasz Napierala }
1575321b17ecSEdward Tomasz Napierala 
1576321b17ecSEdward Tomasz Napierala static int
1577321b17ecSEdward Tomasz Napierala icl_soft_modevent(module_t mod, int what, void *arg)
1578321b17ecSEdward Tomasz Napierala {
1579321b17ecSEdward Tomasz Napierala 
1580321b17ecSEdward Tomasz Napierala 	switch (what) {
1581321b17ecSEdward Tomasz Napierala 	case MOD_LOAD:
1582321b17ecSEdward Tomasz Napierala 		return (icl_soft_load());
1583321b17ecSEdward Tomasz Napierala 	case MOD_UNLOAD:
1584321b17ecSEdward Tomasz Napierala 		return (icl_soft_unload());
1585321b17ecSEdward Tomasz Napierala 	default:
1586321b17ecSEdward Tomasz Napierala 		return (EINVAL);
1587321b17ecSEdward Tomasz Napierala 	}
1588321b17ecSEdward Tomasz Napierala }
1589321b17ecSEdward Tomasz Napierala 
1590321b17ecSEdward Tomasz Napierala moduledata_t icl_soft_data = {
1591321b17ecSEdward Tomasz Napierala 	"icl_soft",
1592321b17ecSEdward Tomasz Napierala 	icl_soft_modevent,
1593321b17ecSEdward Tomasz Napierala 	0
1594321b17ecSEdward Tomasz Napierala };
1595321b17ecSEdward Tomasz Napierala 
1596321b17ecSEdward Tomasz Napierala DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
1597321b17ecSEdward Tomasz Napierala MODULE_DEPEND(icl_soft, icl, 1, 1, 1);
1598872d2d92SEdward Tomasz Napierala MODULE_VERSION(icl_soft, 1);
1599