xref: /freebsd/sys/dev/iscsi/icl.c (revision cc3f4b99653c34ae64f8a1fddea370abefef680e)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Edward Tomasz Napierala under sponsorship
6  * from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 /*
33  * iSCSI Common Layer.  It's used by both the initiator and target to send
34  * and receive iSCSI PDUs.
35  */
36 
37 #include <sys/param.h>
38 #include <sys/capsicum.h>
39 #include <sys/condvar.h>
40 #include <sys/conf.h>
41 #include <sys/file.h>
42 #include <sys/kernel.h>
43 #include <sys/kthread.h>
44 #include <sys/lock.h>
45 #include <sys/mbuf.h>
46 #include <sys/mutex.h>
47 #include <sys/module.h>
48 #include <sys/protosw.h>
49 #include <sys/socket.h>
50 #include <sys/socketvar.h>
51 #include <sys/sysctl.h>
52 #include <sys/systm.h>
53 #include <sys/sx.h>
54 #include <sys/uio.h>
55 #include <vm/uma.h>
56 #include <netinet/in.h>
57 #include <netinet/tcp.h>
58 
59 #include "icl.h"
60 #include "iscsi_proto.h"
61 
62 SYSCTL_NODE(_kern, OID_AUTO, icl, CTLFLAG_RD, 0, "iSCSI Common Layer");
63 static int debug = 1;
64 TUNABLE_INT("kern.icl.debug", &debug);
65 SYSCTL_INT(_kern_icl, OID_AUTO, debug, CTLFLAG_RWTUN,
66     &debug, 0, "Enable debug messages");
67 static int coalesce = 1;
68 TUNABLE_INT("kern.icl.coalesce", &coalesce);
69 SYSCTL_INT(_kern_icl, OID_AUTO, coalesce, CTLFLAG_RWTUN,
70     &coalesce, 0, "Try to coalesce PDUs before sending");
71 static int partial_receive_len = 128 * 1024;
72 TUNABLE_INT("kern.icl.partial_receive_len", &partial_receive_len);
73 SYSCTL_INT(_kern_icl, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN,
74     &partial_receive_len, 0, "Minimum read size for partially received "
75     "data segment");
76 static int sendspace = 1048576;
77 TUNABLE_INT("kern.icl.sendspace", &sendspace);
78 SYSCTL_INT(_kern_icl, OID_AUTO, sendspace, CTLFLAG_RWTUN,
79     &sendspace, 0, "Default send socket buffer size");
80 static int recvspace = 1048576;
81 TUNABLE_INT("kern.icl.recvspace", &recvspace);
82 SYSCTL_INT(_kern_icl, OID_AUTO, recvspace, CTLFLAG_RWTUN,
83     &recvspace, 0, "Default receive socket buffer size");
84 
85 static uma_zone_t icl_conn_zone;
86 static uma_zone_t icl_pdu_zone;
87 
88 static volatile u_int	icl_ncons;
89 
90 #define	ICL_DEBUG(X, ...)						\
91 	do {								\
92 		if (debug > 1)						\
93 			printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
94 	} while (0)
95 
96 #define	ICL_WARN(X, ...)						\
97 	do {								\
98 		if (debug > 0) {					\
99 			printf("WARNING: %s: " X "\n",			\
100 			    __func__, ## __VA_ARGS__);			\
101 		}							\
102 	} while (0)
103 
104 #define ICL_CONN_LOCK(X)		mtx_lock(X->ic_lock)
105 #define ICL_CONN_UNLOCK(X)		mtx_unlock(X->ic_lock)
106 #define ICL_CONN_LOCK_ASSERT(X)		mtx_assert(X->ic_lock, MA_OWNED)
107 #define ICL_CONN_LOCK_ASSERT_NOT(X)	mtx_assert(X->ic_lock, MA_NOTOWNED)
108 
109 STAILQ_HEAD(icl_pdu_stailq, icl_pdu);
110 
111 static void
112 icl_conn_fail(struct icl_conn *ic)
113 {
114 	if (ic->ic_socket == NULL)
115 		return;
116 
117 	/*
118 	 * XXX
119 	 */
120 	ic->ic_socket->so_error = EDOOFUS;
121 	(ic->ic_error)(ic);
122 }
123 
124 static struct mbuf *
125 icl_conn_receive(struct icl_conn *ic, size_t len)
126 {
127 	struct uio uio;
128 	struct socket *so;
129 	struct mbuf *m;
130 	int error, flags;
131 
132 	so = ic->ic_socket;
133 
134 	memset(&uio, 0, sizeof(uio));
135 	uio.uio_resid = len;
136 
137 	flags = MSG_DONTWAIT;
138 	error = soreceive(so, NULL, &uio, &m, NULL, &flags);
139 	if (error != 0) {
140 		ICL_DEBUG("soreceive error %d", error);
141 		return (NULL);
142 	}
143 	if (uio.uio_resid != 0) {
144 		m_freem(m);
145 		ICL_DEBUG("short read");
146 		return (NULL);
147 	}
148 
149 	return (m);
150 }
151 
152 static struct icl_pdu *
153 icl_pdu_new(struct icl_conn *ic, int flags)
154 {
155 	struct icl_pdu *ip;
156 
157 #ifdef DIAGNOSTIC
158 	refcount_acquire(&ic->ic_outstanding_pdus);
159 #endif
160 	ip = uma_zalloc(icl_pdu_zone, flags | M_ZERO);
161 	if (ip == NULL) {
162 		ICL_WARN("failed to allocate %zd bytes", sizeof(*ip));
163 #ifdef DIAGNOSTIC
164 		refcount_release(&ic->ic_outstanding_pdus);
165 #endif
166 		return (NULL);
167 	}
168 
169 	ip->ip_conn = ic;
170 
171 	return (ip);
172 }
173 
174 void
175 icl_pdu_free(struct icl_pdu *ip)
176 {
177 	struct icl_conn *ic;
178 
179 	ic = ip->ip_conn;
180 
181 	m_freem(ip->ip_bhs_mbuf);
182 	m_freem(ip->ip_ahs_mbuf);
183 	m_freem(ip->ip_data_mbuf);
184 	uma_zfree(icl_pdu_zone, ip);
185 #ifdef DIAGNOSTIC
186 	refcount_release(&ic->ic_outstanding_pdus);
187 #endif
188 }
189 
190 /*
191  * Allocate icl_pdu with empty BHS to fill up by the caller.
192  */
193 struct icl_pdu *
194 icl_pdu_new_bhs(struct icl_conn *ic, int flags)
195 {
196 	struct icl_pdu *ip;
197 
198 	ip = icl_pdu_new(ic, flags);
199 	if (ip == NULL)
200 		return (NULL);
201 
202 	ip->ip_bhs_mbuf = m_getm2(NULL, sizeof(struct iscsi_bhs),
203 	    flags, MT_DATA, M_PKTHDR);
204 	if (ip->ip_bhs_mbuf == NULL) {
205 		ICL_WARN("failed to allocate %zd bytes", sizeof(*ip));
206 		icl_pdu_free(ip);
207 		return (NULL);
208 	}
209 	ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *);
210 	memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs));
211 	ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs);
212 
213 	return (ip);
214 }
215 
216 static int
217 icl_pdu_ahs_length(const struct icl_pdu *request)
218 {
219 
220 	return (request->ip_bhs->bhs_total_ahs_len * 4);
221 }
222 
223 size_t
224 icl_pdu_data_segment_length(const struct icl_pdu *request)
225 {
226 	uint32_t len = 0;
227 
228 	len += request->ip_bhs->bhs_data_segment_len[0];
229 	len <<= 8;
230 	len += request->ip_bhs->bhs_data_segment_len[1];
231 	len <<= 8;
232 	len += request->ip_bhs->bhs_data_segment_len[2];
233 
234 	return (len);
235 }
236 
237 static void
238 icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len)
239 {
240 
241 	response->ip_bhs->bhs_data_segment_len[2] = len;
242 	response->ip_bhs->bhs_data_segment_len[1] = len >> 8;
243 	response->ip_bhs->bhs_data_segment_len[0] = len >> 16;
244 }
245 
246 static size_t
247 icl_pdu_padding(const struct icl_pdu *ip)
248 {
249 
250 	if ((ip->ip_data_len % 4) != 0)
251 		return (4 - (ip->ip_data_len % 4));
252 
253 	return (0);
254 }
255 
256 static size_t
257 icl_pdu_size(const struct icl_pdu *response)
258 {
259 	size_t len;
260 
261 	KASSERT(response->ip_ahs_len == 0, ("responding with AHS"));
262 
263 	len = sizeof(struct iscsi_bhs) + response->ip_data_len +
264 	    icl_pdu_padding(response);
265 	if (response->ip_conn->ic_header_crc32c)
266 		len += ISCSI_HEADER_DIGEST_SIZE;
267 	if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c)
268 		len += ISCSI_DATA_DIGEST_SIZE;
269 
270 	return (len);
271 }
272 
273 static int
274 icl_pdu_receive_bhs(struct icl_pdu *request, size_t *availablep)
275 {
276 	struct mbuf *m;
277 
278 	m = icl_conn_receive(request->ip_conn, sizeof(struct iscsi_bhs));
279 	if (m == NULL) {
280 		ICL_DEBUG("failed to receive BHS");
281 		return (-1);
282 	}
283 
284 	request->ip_bhs_mbuf = m_pullup(m, sizeof(struct iscsi_bhs));
285 	if (request->ip_bhs_mbuf == NULL) {
286 		ICL_WARN("m_pullup failed");
287 		return (-1);
288 	}
289 	request->ip_bhs = mtod(request->ip_bhs_mbuf, struct iscsi_bhs *);
290 
291 	/*
292 	 * XXX: For architectures with strict alignment requirements
293 	 * 	we may need to allocate ip_bhs and copy the data into it.
294 	 * 	For some reason, though, not doing this doesn't seem
295 	 * 	to cause problems; tested on sparc64.
296 	 */
297 
298 	*availablep -= sizeof(struct iscsi_bhs);
299 	return (0);
300 }
301 
302 static int
303 icl_pdu_receive_ahs(struct icl_pdu *request, size_t *availablep)
304 {
305 
306 	request->ip_ahs_len = icl_pdu_ahs_length(request);
307 	if (request->ip_ahs_len == 0)
308 		return (0);
309 
310 	request->ip_ahs_mbuf = icl_conn_receive(request->ip_conn,
311 	    request->ip_ahs_len);
312 	if (request->ip_ahs_mbuf == NULL) {
313 		ICL_DEBUG("failed to receive AHS");
314 		return (-1);
315 	}
316 
317 	*availablep -= request->ip_ahs_len;
318 	return (0);
319 }
320 
321 static uint32_t
322 icl_mbuf_to_crc32c(const struct mbuf *m0)
323 {
324 	uint32_t digest = 0xffffffff;
325 	const struct mbuf *m;
326 
327 	for (m = m0; m != NULL; m = m->m_next)
328 		digest = calculate_crc32c(digest,
329 		    mtod(m, const void *), m->m_len);
330 
331 	digest = digest ^ 0xffffffff;
332 
333 	return (digest);
334 }
335 
336 static int
337 icl_pdu_check_header_digest(struct icl_pdu *request, size_t *availablep)
338 {
339 	struct mbuf *m;
340 	uint32_t received_digest, valid_digest;
341 
342 	if (request->ip_conn->ic_header_crc32c == false)
343 		return (0);
344 
345 	m = icl_conn_receive(request->ip_conn, ISCSI_HEADER_DIGEST_SIZE);
346 	if (m == NULL) {
347 		ICL_DEBUG("failed to receive header digest");
348 		return (-1);
349 	}
350 
351 	CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE);
352 	m_copydata(m, 0, ISCSI_HEADER_DIGEST_SIZE, (void *)&received_digest);
353 	m_freem(m);
354 
355 	*availablep -= ISCSI_HEADER_DIGEST_SIZE;
356 
357 	/*
358 	 * XXX: Handle AHS.
359 	 */
360 	valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf);
361 	if (received_digest != valid_digest) {
362 		ICL_WARN("header digest check failed; got 0x%x, "
363 		    "should be 0x%x", received_digest, valid_digest);
364 		return (-1);
365 	}
366 
367 	return (0);
368 }
369 
370 /*
371  * Return the number of bytes that should be waiting in the receive socket
372  * before icl_pdu_receive_data_segment() gets called.
373  */
374 static size_t
375 icl_pdu_data_segment_receive_len(const struct icl_pdu *request)
376 {
377 	size_t len;
378 
379 	len = icl_pdu_data_segment_length(request);
380 	if (len == 0)
381 		return (0);
382 
383 	/*
384 	 * Account for the parts of data segment already read from
385 	 * the socket buffer.
386 	 */
387 	KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len"));
388 	len -= request->ip_data_len;
389 
390 	/*
391 	 * Don't always wait for the full data segment to be delivered
392 	 * to the socket; this might badly affect performance due to
393 	 * TCP window scaling.
394 	 */
395 	if (len > partial_receive_len) {
396 #if 0
397 		ICL_DEBUG("need %zd bytes of data, limiting to %zd",
398 		    len, partial_receive_len));
399 #endif
400 		len = partial_receive_len;
401 
402 		return (len);
403 	}
404 
405 	/*
406 	 * Account for padding.  Note that due to the way code is written,
407 	 * the icl_pdu_receive_data_segment() must always receive padding
408 	 * along with the last part of data segment, because it would be
409 	 * impossible to tell whether we've already received the full data
410 	 * segment including padding, or without it.
411 	 */
412 	if ((len % 4) != 0)
413 		len += 4 - (len % 4);
414 
415 #if 0
416 	ICL_DEBUG("need %zd bytes of data", len));
417 #endif
418 
419 	return (len);
420 }
421 
422 static int
423 icl_pdu_receive_data_segment(struct icl_pdu *request,
424     size_t *availablep, bool *more_neededp)
425 {
426 	struct icl_conn *ic;
427 	size_t len, padding = 0;
428 	struct mbuf *m;
429 
430 	ic = request->ip_conn;
431 
432 	*more_neededp = false;
433 	ic->ic_receive_len = 0;
434 
435 	len = icl_pdu_data_segment_length(request);
436 	if (len == 0)
437 		return (0);
438 
439 	if ((len % 4) != 0)
440 		padding = 4 - (len % 4);
441 
442 	/*
443 	 * Account for already received parts of data segment.
444 	 */
445 	KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len"));
446 	len -= request->ip_data_len;
447 
448 	if (len + padding > *availablep) {
449 		/*
450 		 * Not enough data in the socket buffer.  Receive as much
451 		 * as we can.  Don't receive padding, since, obviously, it's
452 		 * not the end of data segment yet.
453 		 */
454 #if 0
455 		ICL_DEBUG("limited from %zd to %zd",
456 		    len + padding, *availablep - padding));
457 #endif
458 		len = *availablep - padding;
459 		*more_neededp = true;
460 		padding = 0;
461 	}
462 
463 	/*
464 	 * Must not try to receive padding without at least one byte
465 	 * of actual data segment.
466 	 */
467 	if (len > 0) {
468 		m = icl_conn_receive(request->ip_conn, len + padding);
469 		if (m == NULL) {
470 			ICL_DEBUG("failed to receive data segment");
471 			return (-1);
472 		}
473 
474 		if (request->ip_data_mbuf == NULL)
475 			request->ip_data_mbuf = m;
476 		else
477 			m_cat(request->ip_data_mbuf, m);
478 
479 		request->ip_data_len += len;
480 		*availablep -= len + padding;
481 	} else
482 		ICL_DEBUG("len 0");
483 
484 	if (*more_neededp)
485 		ic->ic_receive_len =
486 		    icl_pdu_data_segment_receive_len(request);
487 
488 	return (0);
489 }
490 
491 static int
492 icl_pdu_check_data_digest(struct icl_pdu *request, size_t *availablep)
493 {
494 	struct mbuf *m;
495 	uint32_t received_digest, valid_digest;
496 
497 	if (request->ip_conn->ic_data_crc32c == false)
498 		return (0);
499 
500 	if (request->ip_data_len == 0)
501 		return (0);
502 
503 	m = icl_conn_receive(request->ip_conn, ISCSI_DATA_DIGEST_SIZE);
504 	if (m == NULL) {
505 		ICL_DEBUG("failed to receive data digest");
506 		return (-1);
507 	}
508 
509 	CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE);
510 	m_copydata(m, 0, ISCSI_DATA_DIGEST_SIZE, (void *)&received_digest);
511 	m_freem(m);
512 
513 	*availablep -= ISCSI_DATA_DIGEST_SIZE;
514 
515 	/*
516 	 * Note that ip_data_mbuf also contains padding; since digest
517 	 * calculation is supposed to include that, we iterate over
518 	 * the entire ip_data_mbuf chain, not just ip_data_len bytes of it.
519 	 */
520 	valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf);
521 	if (received_digest != valid_digest) {
522 		ICL_WARN("data digest check failed; got 0x%x, "
523 		    "should be 0x%x", received_digest, valid_digest);
524 		return (-1);
525 	}
526 
527 	return (0);
528 }
529 
530 /*
531  * Somewhat contrary to the name, this attempts to receive only one
532  * "part" of PDU at a time; call it repeatedly until it returns non-NULL.
533  */
534 static struct icl_pdu *
535 icl_conn_receive_pdu(struct icl_conn *ic, size_t *availablep)
536 {
537 	struct icl_pdu *request;
538 	struct socket *so;
539 	size_t len;
540 	int error;
541 	bool more_needed;
542 
543 	so = ic->ic_socket;
544 
545 	if (ic->ic_receive_state == ICL_CONN_STATE_BHS) {
546 		KASSERT(ic->ic_receive_pdu == NULL,
547 		    ("ic->ic_receive_pdu != NULL"));
548 		request = icl_pdu_new(ic, M_NOWAIT);
549 		if (request == NULL) {
550 			ICL_DEBUG("failed to allocate PDU; "
551 			    "dropping connection");
552 			icl_conn_fail(ic);
553 			return (NULL);
554 		}
555 		ic->ic_receive_pdu = request;
556 	} else {
557 		KASSERT(ic->ic_receive_pdu != NULL,
558 		    ("ic->ic_receive_pdu == NULL"));
559 		request = ic->ic_receive_pdu;
560 	}
561 
562 	if (*availablep < ic->ic_receive_len) {
563 #if 0
564 		ICL_DEBUG("not enough data; need %zd, "
565 		    "have %zd", ic->ic_receive_len, *availablep);
566 #endif
567 		return (NULL);
568 	}
569 
570 	switch (ic->ic_receive_state) {
571 	case ICL_CONN_STATE_BHS:
572 		//ICL_DEBUG("receiving BHS");
573 		error = icl_pdu_receive_bhs(request, availablep);
574 		if (error != 0) {
575 			ICL_DEBUG("failed to receive BHS; "
576 			    "dropping connection");
577 			break;
578 		}
579 
580 		/*
581 		 * We don't enforce any limit for AHS length;
582 		 * its length is stored in 8 bit field.
583 		 */
584 
585 		len = icl_pdu_data_segment_length(request);
586 		if (len > ic->ic_max_data_segment_length) {
587 			ICL_WARN("received data segment "
588 			    "length %zd is larger than negotiated "
589 			    "MaxDataSegmentLength %zd; "
590 			    "dropping connection",
591 			    len, ic->ic_max_data_segment_length);
592 			error = EINVAL;
593 			break;
594 		}
595 
596 		ic->ic_receive_state = ICL_CONN_STATE_AHS;
597 		ic->ic_receive_len = icl_pdu_ahs_length(request);
598 		break;
599 
600 	case ICL_CONN_STATE_AHS:
601 		//ICL_DEBUG("receiving AHS");
602 		error = icl_pdu_receive_ahs(request, availablep);
603 		if (error != 0) {
604 			ICL_DEBUG("failed to receive AHS; "
605 			    "dropping connection");
606 			break;
607 		}
608 		ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST;
609 		if (ic->ic_header_crc32c == false)
610 			ic->ic_receive_len = 0;
611 		else
612 			ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE;
613 		break;
614 
615 	case ICL_CONN_STATE_HEADER_DIGEST:
616 		//ICL_DEBUG("receiving header digest");
617 		error = icl_pdu_check_header_digest(request, availablep);
618 		if (error != 0) {
619 			ICL_DEBUG("header digest failed; "
620 			    "dropping connection");
621 			break;
622 		}
623 
624 		ic->ic_receive_state = ICL_CONN_STATE_DATA;
625 		ic->ic_receive_len =
626 		    icl_pdu_data_segment_receive_len(request);
627 		break;
628 
629 	case ICL_CONN_STATE_DATA:
630 		//ICL_DEBUG("receiving data segment");
631 		error = icl_pdu_receive_data_segment(request, availablep,
632 		    &more_needed);
633 		if (error != 0) {
634 			ICL_DEBUG("failed to receive data segment;"
635 			    "dropping connection");
636 			break;
637 		}
638 
639 		if (more_needed)
640 			break;
641 
642 		ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST;
643 		if (request->ip_data_len == 0 || ic->ic_data_crc32c == false)
644 			ic->ic_receive_len = 0;
645 		else
646 			ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE;
647 		break;
648 
649 	case ICL_CONN_STATE_DATA_DIGEST:
650 		//ICL_DEBUG("receiving data digest");
651 		error = icl_pdu_check_data_digest(request, availablep);
652 		if (error != 0) {
653 			ICL_DEBUG("data digest failed; "
654 			    "dropping connection");
655 			break;
656 		}
657 
658 		/*
659 		 * We've received complete PDU; reset the receive state machine
660 		 * and return the PDU.
661 		 */
662 		ic->ic_receive_state = ICL_CONN_STATE_BHS;
663 		ic->ic_receive_len = sizeof(struct iscsi_bhs);
664 		ic->ic_receive_pdu = NULL;
665 		return (request);
666 
667 	default:
668 		panic("invalid ic_receive_state %d\n", ic->ic_receive_state);
669 	}
670 
671 	if (error != 0) {
672 		icl_pdu_free(request);
673 		icl_conn_fail(ic);
674 	}
675 
676 	return (NULL);
677 }
678 
679 static void
680 icl_conn_receive_pdus(struct icl_conn *ic, size_t available)
681 {
682 	struct icl_pdu *response;
683 	struct socket *so;
684 
685 	so = ic->ic_socket;
686 
687 	/*
688 	 * This can never happen; we're careful to only mess with ic->ic_socket
689 	 * pointer when the send/receive threads are not running.
690 	 */
691 	KASSERT(so != NULL, ("NULL socket"));
692 
693 	for (;;) {
694 		if (ic->ic_disconnecting)
695 			return;
696 
697 		if (so->so_error != 0) {
698 			ICL_DEBUG("connection error %d; "
699 			    "dropping connection", so->so_error);
700 			icl_conn_fail(ic);
701 			return;
702 		}
703 
704 		/*
705 		 * Loop until we have a complete PDU or there is not enough
706 		 * data in the socket buffer.
707 		 */
708 		if (available < ic->ic_receive_len) {
709 #if 0
710 			ICL_DEBUG("not enough data; have %zd, "
711 			    "need %zd", available,
712 			    ic->ic_receive_len);
713 #endif
714 			return;
715 		}
716 
717 		response = icl_conn_receive_pdu(ic, &available);
718 		if (response == NULL)
719 			continue;
720 
721 		if (response->ip_ahs_len > 0) {
722 			ICL_WARN("received PDU with unsupported "
723 			    "AHS; opcode 0x%x; dropping connection",
724 			    response->ip_bhs->bhs_opcode);
725 			icl_pdu_free(response);
726 			icl_conn_fail(ic);
727 			return;
728 		}
729 
730 		(ic->ic_receive)(response);
731 	}
732 }
733 
734 static void
735 icl_receive_thread(void *arg)
736 {
737 	struct icl_conn *ic;
738 	size_t available;
739 	struct socket *so;
740 
741 	ic = arg;
742 	so = ic->ic_socket;
743 
744 	ICL_CONN_LOCK(ic);
745 	ic->ic_receive_running = true;
746 	ICL_CONN_UNLOCK(ic);
747 
748 	for (;;) {
749 		if (ic->ic_disconnecting) {
750 			//ICL_DEBUG("terminating");
751 			break;
752 		}
753 
754 		/*
755 		 * Set the low watermark, to be checked by
756 		 * soreadable() in icl_soupcall_receive()
757 		 * to avoid unneccessary wakeups until there
758 		 * is enough data received to read the PDU.
759 		 */
760 		SOCKBUF_LOCK(&so->so_rcv);
761 		available = so->so_rcv.sb_cc;
762 		if (available < ic->ic_receive_len) {
763 			so->so_rcv.sb_lowat = ic->ic_receive_len;
764 			cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx);
765 		} else
766 			so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1;
767 		SOCKBUF_UNLOCK(&so->so_rcv);
768 
769 		icl_conn_receive_pdus(ic, available);
770 	}
771 
772 	ICL_CONN_LOCK(ic);
773 	ic->ic_receive_running = false;
774 	ICL_CONN_UNLOCK(ic);
775 	kthread_exit();
776 }
777 
778 static int
779 icl_soupcall_receive(struct socket *so, void *arg, int waitflag)
780 {
781 	struct icl_conn *ic;
782 
783 	if (!soreadable(so))
784 		return (SU_OK);
785 
786 	ic = arg;
787 	cv_signal(&ic->ic_receive_cv);
788 	return (SU_OK);
789 }
790 
791 static int
792 icl_pdu_finalize(struct icl_pdu *request)
793 {
794 	size_t padding, pdu_len;
795 	uint32_t digest, zero = 0;
796 	int ok;
797 	struct icl_conn *ic;
798 
799 	ic = request->ip_conn;
800 
801 	icl_pdu_set_data_segment_length(request, request->ip_data_len);
802 
803 	pdu_len = icl_pdu_size(request);
804 
805 	if (ic->ic_header_crc32c) {
806 		digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf);
807 		ok = m_append(request->ip_bhs_mbuf, sizeof(digest),
808 		    (void *)&digest);
809 		if (ok != 1) {
810 			ICL_WARN("failed to append header digest");
811 			return (1);
812 		}
813 	}
814 
815 	if (request->ip_data_len != 0) {
816 		padding = icl_pdu_padding(request);
817 		if (padding > 0) {
818 			ok = m_append(request->ip_data_mbuf, padding,
819 			    (void *)&zero);
820 			if (ok != 1) {
821 				ICL_WARN("failed to append padding");
822 				return (1);
823 			}
824 		}
825 
826 		if (ic->ic_data_crc32c) {
827 			digest = icl_mbuf_to_crc32c(request->ip_data_mbuf);
828 
829 			ok = m_append(request->ip_data_mbuf, sizeof(digest),
830 			    (void *)&digest);
831 			if (ok != 1) {
832 				ICL_WARN("failed to append data digest");
833 				return (1);
834 			}
835 		}
836 
837 		m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf);
838 		request->ip_data_mbuf = NULL;
839 	}
840 
841 	request->ip_bhs_mbuf->m_pkthdr.len = pdu_len;
842 
843 	return (0);
844 }
845 
846 static void
847 icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue)
848 {
849 	struct icl_pdu *request, *request2;
850 	struct socket *so;
851 	size_t available, size, size2;
852 	int coalesced, error;
853 
854 	ICL_CONN_LOCK_ASSERT_NOT(ic);
855 
856 	so = ic->ic_socket;
857 
858 	SOCKBUF_LOCK(&so->so_snd);
859 	/*
860 	 * Check how much space do we have for transmit.  We can't just
861 	 * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE,
862 	 * as it always frees the mbuf chain passed to it, even in case
863 	 * of error.
864 	 */
865 	available = sbspace(&so->so_snd);
866 
867 	/*
868 	 * Notify the socket upcall that we don't need wakeups
869 	 * for the time being.
870 	 */
871 	so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1;
872 	SOCKBUF_UNLOCK(&so->so_snd);
873 
874 	while (!STAILQ_EMPTY(queue)) {
875 		if (ic->ic_disconnecting)
876 			return;
877 		request = STAILQ_FIRST(queue);
878 		size = icl_pdu_size(request);
879 		if (available < size) {
880 
881 			/*
882 			 * Set the low watermark, to be checked by
883 			 * sowriteable() in icl_soupcall_send()
884 			 * to avoid unneccessary wakeups until there
885 			 * is enough space for the PDU to fit.
886 			 */
887 			SOCKBUF_LOCK(&so->so_snd);
888 			available = sbspace(&so->so_snd);
889 			if (available < size) {
890 #if 1
891 				ICL_DEBUG("no space to send; "
892 				    "have %zd, need %zd",
893 				    available, size);
894 #endif
895 				so->so_snd.sb_lowat = size;
896 				SOCKBUF_UNLOCK(&so->so_snd);
897 				return;
898 			}
899 			SOCKBUF_UNLOCK(&so->so_snd);
900 		}
901 		STAILQ_REMOVE_HEAD(queue, ip_next);
902 		error = icl_pdu_finalize(request);
903 		if (error != 0) {
904 			ICL_DEBUG("failed to finalize PDU; "
905 			    "dropping connection");
906 			icl_conn_fail(ic);
907 			icl_pdu_free(request);
908 			return;
909 		}
910 		if (coalesce) {
911 			coalesced = 1;
912 			for (;;) {
913 				request2 = STAILQ_FIRST(queue);
914 				if (request2 == NULL)
915 					break;
916 				size2 = icl_pdu_size(request2);
917 				if (available < size + size2)
918 					break;
919 				STAILQ_REMOVE_HEAD(queue, ip_next);
920 				error = icl_pdu_finalize(request2);
921 				if (error != 0) {
922 					ICL_DEBUG("failed to finalize PDU; "
923 					    "dropping connection");
924 					icl_conn_fail(ic);
925 					icl_pdu_free(request);
926 					icl_pdu_free(request2);
927 					return;
928 				}
929 				m_cat(request->ip_bhs_mbuf, request2->ip_bhs_mbuf);
930 				request2->ip_bhs_mbuf = NULL;
931 				request->ip_bhs_mbuf->m_pkthdr.len += size2;
932 				size += size2;
933 				STAILQ_REMOVE_AFTER(queue, request, ip_next);
934 				icl_pdu_free(request2);
935 				coalesced++;
936 			}
937 #if 0
938 			if (coalesced > 1) {
939 				ICL_DEBUG("coalesced %d PDUs into %zd bytes",
940 				    coalesced, size);
941 			}
942 #endif
943 		}
944 		available -= size;
945 		error = sosend(so, NULL, NULL, request->ip_bhs_mbuf,
946 		    NULL, MSG_DONTWAIT, curthread);
947 		request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */
948 		if (error != 0) {
949 			ICL_DEBUG("failed to send PDU, error %d; "
950 			    "dropping connection", error);
951 			icl_conn_fail(ic);
952 			icl_pdu_free(request);
953 			return;
954 		}
955 		icl_pdu_free(request);
956 	}
957 }
958 
959 static void
960 icl_send_thread(void *arg)
961 {
962 	struct icl_conn *ic;
963 	struct icl_pdu_stailq queue;
964 
965 	ic = arg;
966 
967 	STAILQ_INIT(&queue);
968 
969 	ICL_CONN_LOCK(ic);
970 	ic->ic_send_running = true;
971 
972 	for (;;) {
973 		if (ic->ic_disconnecting) {
974 			//ICL_DEBUG("terminating");
975 			break;
976 		}
977 
978 		for (;;) {
979 			/*
980 			 * If the local queue is empty, populate it from
981 			 * the main one.  This way the icl_conn_send_pdus()
982 			 * can go through all the queued PDUs without holding
983 			 * any locks.
984 			 */
985 			if (STAILQ_EMPTY(&queue))
986 				STAILQ_SWAP(&ic->ic_to_send, &queue, icl_pdu);
987 
988 			ic->ic_check_send_space = false;
989 			ICL_CONN_UNLOCK(ic);
990 			icl_conn_send_pdus(ic, &queue);
991 			ICL_CONN_LOCK(ic);
992 
993 			/*
994 			 * The icl_soupcall_send() was called since the last
995 			 * call to sbspace(); go around;
996 			 */
997 			if (ic->ic_check_send_space)
998 				continue;
999 
1000 			/*
1001 			 * Local queue is empty, but we still have PDUs
1002 			 * in the main one; go around.
1003 			 */
1004 			if (STAILQ_EMPTY(&queue) &&
1005 			    !STAILQ_EMPTY(&ic->ic_to_send))
1006 				continue;
1007 
1008 			/*
1009 			 * There might be some stuff in the local queue,
1010 			 * which didn't get sent due to not having enough send
1011 			 * space.  Wait for socket upcall.
1012 			 */
1013 			break;
1014 		}
1015 
1016 		cv_wait(&ic->ic_send_cv, ic->ic_lock);
1017 	}
1018 
1019 	/*
1020 	 * We're exiting; move PDUs back to the main queue, so they can
1021 	 * get freed properly.  At this point ordering doesn't matter.
1022 	 */
1023 	STAILQ_CONCAT(&ic->ic_to_send, &queue);
1024 
1025 	ic->ic_send_running = false;
1026 	ICL_CONN_UNLOCK(ic);
1027 	kthread_exit();
1028 }
1029 
1030 static int
1031 icl_soupcall_send(struct socket *so, void *arg, int waitflag)
1032 {
1033 	struct icl_conn *ic;
1034 
1035 	if (!sowriteable(so))
1036 		return (SU_OK);
1037 
1038 	ic = arg;
1039 
1040 	ICL_CONN_LOCK(ic);
1041 	ic->ic_check_send_space = true;
1042 	ICL_CONN_UNLOCK(ic);
1043 
1044 	cv_signal(&ic->ic_send_cv);
1045 
1046 	return (SU_OK);
1047 }
1048 
1049 int
1050 icl_pdu_append_data(struct icl_pdu *request, const void *addr, size_t len,
1051     int flags)
1052 {
1053 	struct mbuf *mb, *newmb;
1054 	size_t copylen, off = 0;
1055 
1056 	KASSERT(len > 0, ("len == 0"));
1057 
1058 	newmb = m_getm2(NULL, len, flags, MT_DATA, M_PKTHDR);
1059 	if (newmb == NULL) {
1060 		ICL_WARN("failed to allocate mbuf for %zd bytes", len);
1061 		return (ENOMEM);
1062 	}
1063 
1064 	for (mb = newmb; mb != NULL; mb = mb->m_next) {
1065 		copylen = min(M_TRAILINGSPACE(mb), len - off);
1066 		memcpy(mtod(mb, char *), (const char *)addr + off, copylen);
1067 		mb->m_len = copylen;
1068 		off += copylen;
1069 	}
1070 	KASSERT(off == len, ("%s: off != len", __func__));
1071 
1072 	if (request->ip_data_mbuf == NULL) {
1073 		request->ip_data_mbuf = newmb;
1074 		request->ip_data_len = len;
1075 	} else {
1076 		m_cat(request->ip_data_mbuf, newmb);
1077 		request->ip_data_len += len;
1078 	}
1079 
1080 	return (0);
1081 }
1082 
1083 void
1084 icl_pdu_get_data(struct icl_pdu *ip, size_t off, void *addr, size_t len)
1085 {
1086 
1087 	m_copydata(ip->ip_data_mbuf, off, len, addr);
1088 }
1089 
1090 void
1091 icl_pdu_queue(struct icl_pdu *ip)
1092 {
1093 	struct icl_conn *ic;
1094 
1095 	ic = ip->ip_conn;
1096 
1097 	ICL_CONN_LOCK_ASSERT(ic);
1098 
1099 	if (ic->ic_disconnecting || ic->ic_socket == NULL) {
1100 		ICL_DEBUG("icl_pdu_queue on closed connection");
1101 		icl_pdu_free(ip);
1102 		return;
1103 	}
1104 
1105 	if (!STAILQ_EMPTY(&ic->ic_to_send)) {
1106 		STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next);
1107 		/*
1108 		 * If the queue is not empty, someone else had already
1109 		 * signaled the send thread; no need to do that again,
1110 		 * just return.
1111 		 */
1112 		return;
1113 	}
1114 
1115 	STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next);
1116 	cv_signal(&ic->ic_send_cv);
1117 }
1118 
1119 struct icl_conn *
1120 icl_conn_new(const char *name, struct mtx *lock)
1121 {
1122 	struct icl_conn *ic;
1123 
1124 	refcount_acquire(&icl_ncons);
1125 
1126 	ic = uma_zalloc(icl_conn_zone, M_WAITOK | M_ZERO);
1127 
1128 	STAILQ_INIT(&ic->ic_to_send);
1129 	ic->ic_lock = lock;
1130 	cv_init(&ic->ic_send_cv, "icl_tx");
1131 	cv_init(&ic->ic_receive_cv, "icl_rx");
1132 #ifdef DIAGNOSTIC
1133 	refcount_init(&ic->ic_outstanding_pdus, 0);
1134 #endif
1135 	ic->ic_max_data_segment_length = ICL_MAX_DATA_SEGMENT_LENGTH;
1136 	ic->ic_name = name;
1137 
1138 	return (ic);
1139 }
1140 
1141 void
1142 icl_conn_free(struct icl_conn *ic)
1143 {
1144 
1145 	cv_destroy(&ic->ic_send_cv);
1146 	cv_destroy(&ic->ic_receive_cv);
1147 	uma_zfree(icl_conn_zone, ic);
1148 	refcount_release(&icl_ncons);
1149 }
1150 
1151 static int
1152 icl_conn_start(struct icl_conn *ic)
1153 {
1154 	size_t minspace;
1155 	struct sockopt opt;
1156 	int error, one = 1;
1157 
1158 	ICL_CONN_LOCK(ic);
1159 
1160 	/*
1161 	 * XXX: Ugly hack.
1162 	 */
1163 	if (ic->ic_socket == NULL) {
1164 		ICL_CONN_UNLOCK(ic);
1165 		return (EINVAL);
1166 	}
1167 
1168 	ic->ic_receive_state = ICL_CONN_STATE_BHS;
1169 	ic->ic_receive_len = sizeof(struct iscsi_bhs);
1170 	ic->ic_disconnecting = false;
1171 
1172 	ICL_CONN_UNLOCK(ic);
1173 
1174 	/*
1175 	 * For sendspace, this is required because the current code cannot
1176 	 * send a PDU in pieces; thus, the minimum buffer size is equal
1177 	 * to the maximum PDU size.  "+4" is to account for possible padding.
1178 	 *
1179 	 * What we should actually do here is to use autoscaling, but set
1180 	 * some minimal buffer size to "minspace".  I don't know a way to do
1181 	 * that, though.
1182 	 */
1183 	minspace = sizeof(struct iscsi_bhs) + ic->ic_max_data_segment_length +
1184 	    ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4;
1185 	if (sendspace < minspace) {
1186 		ICL_WARN("kern.icl.sendspace too low; must be at least %zd",
1187 		    minspace);
1188 		sendspace = minspace;
1189 	}
1190 	if (recvspace < minspace) {
1191 		ICL_WARN("kern.icl.recvspace too low; must be at least %zd",
1192 		    minspace);
1193 		recvspace = minspace;
1194 	}
1195 
1196 	error = soreserve(ic->ic_socket, sendspace, recvspace);
1197 	if (error != 0) {
1198 		ICL_WARN("soreserve failed with error %d", error);
1199 		icl_conn_close(ic);
1200 		return (error);
1201 	}
1202 
1203 	/*
1204 	 * Disable Nagle.
1205 	 */
1206 	bzero(&opt, sizeof(opt));
1207 	opt.sopt_dir = SOPT_SET;
1208 	opt.sopt_level = IPPROTO_TCP;
1209 	opt.sopt_name = TCP_NODELAY;
1210 	opt.sopt_val = &one;
1211 	opt.sopt_valsize = sizeof(one);
1212 	error = sosetopt(ic->ic_socket, &opt);
1213 	if (error != 0) {
1214 		ICL_WARN("disabling TCP_NODELAY failed with error %d", error);
1215 		icl_conn_close(ic);
1216 		return (error);
1217 	}
1218 
1219 	/*
1220 	 * Start threads.
1221 	 */
1222 	error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx",
1223 	    ic->ic_name);
1224 	if (error != 0) {
1225 		ICL_WARN("kthread_add(9) failed with error %d", error);
1226 		icl_conn_close(ic);
1227 		return (error);
1228 	}
1229 
1230 	error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx",
1231 	    ic->ic_name);
1232 	if (error != 0) {
1233 		ICL_WARN("kthread_add(9) failed with error %d", error);
1234 		icl_conn_close(ic);
1235 		return (error);
1236 	}
1237 
1238 	/*
1239 	 * Register socket upcall, to get notified about incoming PDUs
1240 	 * and free space to send outgoing ones.
1241 	 */
1242 	SOCKBUF_LOCK(&ic->ic_socket->so_snd);
1243 	soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic);
1244 	SOCKBUF_UNLOCK(&ic->ic_socket->so_snd);
1245 	SOCKBUF_LOCK(&ic->ic_socket->so_rcv);
1246 	soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic);
1247 	SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv);
1248 
1249 	return (0);
1250 }
1251 
1252 int
1253 icl_conn_handoff(struct icl_conn *ic, int fd)
1254 {
1255 	struct file *fp;
1256 	struct socket *so;
1257 	cap_rights_t rights;
1258 	int error;
1259 
1260 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1261 
1262 	/*
1263 	 * Steal the socket from userland.
1264 	 */
1265 	error = fget(curthread, fd,
1266 	    cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp);
1267 	if (error != 0)
1268 		return (error);
1269 	if (fp->f_type != DTYPE_SOCKET) {
1270 		fdrop(fp, curthread);
1271 		return (EINVAL);
1272 	}
1273 	so = fp->f_data;
1274 	if (so->so_type != SOCK_STREAM) {
1275 		fdrop(fp, curthread);
1276 		return (EINVAL);
1277 	}
1278 
1279 	ICL_CONN_LOCK(ic);
1280 
1281 	if (ic->ic_socket != NULL) {
1282 		ICL_CONN_UNLOCK(ic);
1283 		fdrop(fp, curthread);
1284 		return (EBUSY);
1285 	}
1286 
1287 	ic->ic_socket = fp->f_data;
1288 	fp->f_ops = &badfileops;
1289 	fp->f_data = NULL;
1290 	fdrop(fp, curthread);
1291 	ICL_CONN_UNLOCK(ic);
1292 
1293 	error = icl_conn_start(ic);
1294 
1295 	return (error);
1296 }
1297 
1298 void
1299 icl_conn_shutdown(struct icl_conn *ic)
1300 {
1301 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1302 
1303 	ICL_CONN_LOCK(ic);
1304 	if (ic->ic_socket == NULL) {
1305 		ICL_CONN_UNLOCK(ic);
1306 		return;
1307 	}
1308 	ICL_CONN_UNLOCK(ic);
1309 
1310 	soshutdown(ic->ic_socket, SHUT_RDWR);
1311 }
1312 
1313 void
1314 icl_conn_close(struct icl_conn *ic)
1315 {
1316 	struct icl_pdu *pdu;
1317 
1318 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1319 
1320 	ICL_CONN_LOCK(ic);
1321 	if (ic->ic_socket == NULL) {
1322 		ICL_CONN_UNLOCK(ic);
1323 		return;
1324 	}
1325 
1326 	/*
1327 	 * Deregister socket upcalls.
1328 	 */
1329 	ICL_CONN_UNLOCK(ic);
1330 	SOCKBUF_LOCK(&ic->ic_socket->so_snd);
1331 	if (ic->ic_socket->so_snd.sb_upcall != NULL)
1332 		soupcall_clear(ic->ic_socket, SO_SND);
1333 	SOCKBUF_UNLOCK(&ic->ic_socket->so_snd);
1334 	SOCKBUF_LOCK(&ic->ic_socket->so_rcv);
1335 	if (ic->ic_socket->so_rcv.sb_upcall != NULL)
1336 		soupcall_clear(ic->ic_socket, SO_RCV);
1337 	SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv);
1338 	ICL_CONN_LOCK(ic);
1339 
1340 	ic->ic_disconnecting = true;
1341 
1342 	/*
1343 	 * Wake up the threads, so they can properly terminate.
1344 	 */
1345 	cv_signal(&ic->ic_receive_cv);
1346 	cv_signal(&ic->ic_send_cv);
1347 	while (ic->ic_receive_running || ic->ic_send_running) {
1348 		//ICL_DEBUG("waiting for send/receive threads to terminate");
1349 		ICL_CONN_UNLOCK(ic);
1350 		cv_signal(&ic->ic_receive_cv);
1351 		cv_signal(&ic->ic_send_cv);
1352 		pause("icl_close", 1 * hz);
1353 		ICL_CONN_LOCK(ic);
1354 	}
1355 	//ICL_DEBUG("send/receive threads terminated");
1356 
1357 	ICL_CONN_UNLOCK(ic);
1358 	soclose(ic->ic_socket);
1359 	ICL_CONN_LOCK(ic);
1360 	ic->ic_socket = NULL;
1361 
1362 	if (ic->ic_receive_pdu != NULL) {
1363 		//ICL_DEBUG("freeing partially received PDU");
1364 		icl_pdu_free(ic->ic_receive_pdu);
1365 		ic->ic_receive_pdu = NULL;
1366 	}
1367 
1368 	/*
1369 	 * Remove any outstanding PDUs from the send queue.
1370 	 */
1371 	while (!STAILQ_EMPTY(&ic->ic_to_send)) {
1372 		pdu = STAILQ_FIRST(&ic->ic_to_send);
1373 		STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next);
1374 		icl_pdu_free(pdu);
1375 	}
1376 
1377 	KASSERT(STAILQ_EMPTY(&ic->ic_to_send),
1378 	    ("destroying session with non-empty send queue"));
1379 #ifdef DIAGNOSTIC
1380 	KASSERT(ic->ic_outstanding_pdus == 0,
1381 	    ("destroying session with %d outstanding PDUs",
1382 	     ic->ic_outstanding_pdus));
1383 #endif
1384 	ICL_CONN_UNLOCK(ic);
1385 }
1386 
1387 bool
1388 icl_conn_connected(struct icl_conn *ic)
1389 {
1390 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1391 
1392 	ICL_CONN_LOCK(ic);
1393 	if (ic->ic_socket == NULL) {
1394 		ICL_CONN_UNLOCK(ic);
1395 		return (false);
1396 	}
1397 	if (ic->ic_socket->so_error != 0) {
1398 		ICL_CONN_UNLOCK(ic);
1399 		return (false);
1400 	}
1401 	ICL_CONN_UNLOCK(ic);
1402 	return (true);
1403 }
1404 
1405 #ifdef ICL_KERNEL_PROXY
1406 int
1407 icl_conn_handoff_sock(struct icl_conn *ic, struct socket *so)
1408 {
1409 	int error;
1410 
1411 	ICL_CONN_LOCK_ASSERT_NOT(ic);
1412 
1413 	if (so->so_type != SOCK_STREAM)
1414 		return (EINVAL);
1415 
1416 	ICL_CONN_LOCK(ic);
1417 	if (ic->ic_socket != NULL) {
1418 		ICL_CONN_UNLOCK(ic);
1419 		return (EBUSY);
1420 	}
1421 	ic->ic_socket = so;
1422 	ICL_CONN_UNLOCK(ic);
1423 
1424 	error = icl_conn_start(ic);
1425 
1426 	return (error);
1427 }
1428 #endif /* ICL_KERNEL_PROXY */
1429 
1430 static int
1431 icl_unload(void)
1432 {
1433 
1434 	if (icl_ncons != 0)
1435 		return (EBUSY);
1436 
1437 	uma_zdestroy(icl_conn_zone);
1438 	uma_zdestroy(icl_pdu_zone);
1439 
1440 	return (0);
1441 }
1442 
1443 static void
1444 icl_load(void)
1445 {
1446 
1447 	icl_conn_zone = uma_zcreate("icl_conn",
1448 	    sizeof(struct icl_conn), NULL, NULL, NULL, NULL,
1449 	    UMA_ALIGN_PTR, 0);
1450 	icl_pdu_zone = uma_zcreate("icl_pdu",
1451 	    sizeof(struct icl_pdu), NULL, NULL, NULL, NULL,
1452 	    UMA_ALIGN_PTR, 0);
1453 
1454 	refcount_init(&icl_ncons, 0);
1455 }
1456 
1457 static int
1458 icl_modevent(module_t mod, int what, void *arg)
1459 {
1460 
1461 	switch (what) {
1462 	case MOD_LOAD:
1463 		icl_load();
1464 		return (0);
1465 	case MOD_UNLOAD:
1466 		return (icl_unload());
1467 	default:
1468 		return (EINVAL);
1469 	}
1470 }
1471 
1472 moduledata_t icl_data = {
1473 	"icl",
1474 	icl_modevent,
1475 	0
1476 };
1477 
1478 DECLARE_MODULE(icl, icl_data, SI_SUB_DRIVERS, SI_ORDER_FIRST);
1479 MODULE_VERSION(icl, 1);
1480