xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision ec273ebf3b6aed5fba8c56b6ece5ad8693a48ea7)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Edward Tomasz Napierala under sponsorship
6  * from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 /*
33  * CTL frontend for the iSCSI protocol.
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include <sys/param.h>
40 #include <sys/capsicum.h>
41 #include <sys/condvar.h>
42 #include <sys/file.h>
43 #include <sys/kernel.h>
44 #include <sys/kthread.h>
45 #include <sys/lock.h>
46 #include <sys/malloc.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
49 #include <sys/queue.h>
50 #include <sys/sbuf.h>
51 #include <sys/sysctl.h>
52 #include <sys/systm.h>
53 #include <sys/uio.h>
54 #include <sys/unistd.h>
55 #include <vm/uma.h>
56 
57 #include <cam/scsi/scsi_all.h>
58 #include <cam/scsi/scsi_da.h>
59 #include <cam/ctl/ctl_io.h>
60 #include <cam/ctl/ctl.h>
61 #include <cam/ctl/ctl_backend.h>
62 #include <cam/ctl/ctl_error.h>
63 #include <cam/ctl/ctl_frontend.h>
64 #include <cam/ctl/ctl_frontend_internal.h>
65 #include <cam/ctl/ctl_debug.h>
66 #include <cam/ctl/ctl_ha.h>
67 #include <cam/ctl/ctl_ioctl.h>
68 #include <cam/ctl/ctl_private.h>
69 
70 #include <dev/iscsi/icl.h>
71 #include <dev/iscsi/icl_wrappers.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <cam/ctl/ctl_frontend_iscsi.h>
74 
75 #ifdef ICL_KERNEL_PROXY
76 #include <sys/socketvar.h>
77 #endif
78 
79 #ifdef ICL_KERNEL_PROXY
80 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
81 #endif
82 
83 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
84 static uma_zone_t cfiscsi_data_wait_zone;
85 
86 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
87     "CAM Target Layer iSCSI Frontend");
88 static int debug = 1;
89 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
90     &debug, 1, "Enable debug messages");
91 static int ping_timeout = 5;
92 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
93     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
94 static int login_timeout = 60;
95 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
96     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
97 static int maxcmdsn_delta = 256;
98 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN,
99     &maxcmdsn_delta, 256, "Number of commands the initiator can send "
100     "without confirmation");
101 
102 #define	CFISCSI_DEBUG(X, ...)						\
103 	do {								\
104 		if (debug > 1) {					\
105 			printf("%s: " X "\n",				\
106 			    __func__, ## __VA_ARGS__);			\
107 		}							\
108 	} while (0)
109 
110 #define	CFISCSI_WARN(X, ...)						\
111 	do {								\
112 		if (debug > 0) {					\
113 			printf("WARNING: %s: " X "\n",			\
114 			    __func__, ## __VA_ARGS__);			\
115 		}							\
116 	} while (0)
117 
118 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
119 	do {								\
120 		if (debug > 1) {					\
121 			printf("%s: %s (%s): " X "\n",			\
122 			    __func__, S->cs_initiator_addr,		\
123 			    S->cs_initiator_name, ## __VA_ARGS__);	\
124 		}							\
125 	} while (0)
126 
127 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
128 	do  {								\
129 		if (debug > 0) {					\
130 			printf("WARNING: %s (%s): " X "\n",		\
131 			    S->cs_initiator_addr,			\
132 			    S->cs_initiator_name, ## __VA_ARGS__);	\
133 		}							\
134 	} while (0)
135 
136 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
137 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
138 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
139 
140 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
141 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
142 #define	PDU_EXPDATASN(X)		(X)->ip_prv0
143 #define	PDU_TOTAL_TRANSFER_LEN(X)	(X)->ip_prv1
144 #define	PDU_R2TSN(X)			(X)->ip_prv2
145 
146 int		cfiscsi_init(void);
147 static void	cfiscsi_online(void *arg);
148 static void	cfiscsi_offline(void *arg);
149 static int	cfiscsi_info(void *arg, struct sbuf *sb);
150 static int	cfiscsi_lun_enable(void *arg,
151 		    struct ctl_id target_id, int lun_id);
152 static int	cfiscsi_lun_disable(void *arg,
153 		    struct ctl_id target_id, int lun_id);
154 static int	cfiscsi_ioctl(struct cdev *dev,
155 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
156 static void	cfiscsi_datamove(union ctl_io *io);
157 static void	cfiscsi_datamove_in(union ctl_io *io);
158 static void	cfiscsi_datamove_out(union ctl_io *io);
159 static void	cfiscsi_done(union ctl_io *io);
160 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
161 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
162 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
163 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
164 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
165 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
166 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
167 static struct cfiscsi_data_wait	*cfiscsi_data_wait_new(
168 		    struct cfiscsi_session *cs, union ctl_io *io,
169 		    uint32_t initiator_task_tag,
170 		    uint32_t *target_transfer_tagp);
171 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
172 		    struct cfiscsi_data_wait *cdw);
173 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
174 		    *softc, const char *name, uint16_t tag);
175 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
176     struct cfiscsi_softc *softc, const char *name, const char *alias,
177     uint16_t tag);
178 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
179 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
180 
181 static struct cfiscsi_softc cfiscsi_softc;
182 extern struct ctl_softc *control_softc;
183 
184 static struct ctl_frontend cfiscsi_frontend =
185 {
186 	.name = "iscsi",
187 	.init = cfiscsi_init,
188 	.ioctl = cfiscsi_ioctl,
189 };
190 CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend);
191 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
192 
193 static struct icl_pdu *
194 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
195 {
196 
197 	return (icl_pdu_new(request->ip_conn, flags));
198 }
199 
200 static bool
201 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
202 {
203 	const struct iscsi_bhs_scsi_command *bhssc;
204 	struct cfiscsi_session *cs;
205 	uint32_t cmdsn, expstatsn;
206 
207 	cs = PDU_SESSION(request);
208 
209 	/*
210 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
211 	 * The purpose of the timeout is to reset the connection when it stalls;
212 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
213 	 * in some queue.
214 	 *
215 	 * XXX: Locking?
216 	 */
217 	cs->cs_timeout = 0;
218 
219 	/*
220 	 * Data-Out PDUs don't contain CmdSN.
221 	 */
222 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
223 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
224 		return (false);
225 
226 	/*
227 	 * We're only using fields common for all the request
228 	 * (initiator -> target) PDUs.
229 	 */
230 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
231 	cmdsn = ntohl(bhssc->bhssc_cmdsn);
232 	expstatsn = ntohl(bhssc->bhssc_expstatsn);
233 
234 	CFISCSI_SESSION_LOCK(cs);
235 #if 0
236 	if (expstatsn != cs->cs_statsn) {
237 		CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
238 		    "while current StatSN is %d", expstatsn,
239 		    cs->cs_statsn);
240 	}
241 #endif
242 
243 	if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
244 		/*
245 		 * The target MUST silently ignore any non-immediate command
246 		 * outside of this range.
247 		 */
248 		if (ISCSI_SNLT(cmdsn, cs->cs_cmdsn) ||
249 		    ISCSI_SNGT(cmdsn, cs->cs_cmdsn + maxcmdsn_delta)) {
250 			CFISCSI_SESSION_UNLOCK(cs);
251 			CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
252 			    "while expected %u", cmdsn, cs->cs_cmdsn);
253 			return (true);
254 		}
255 
256 		/*
257 		 * We don't support multiple connections now, so any
258 		 * discontinuity in CmdSN means lost PDUs.  Since we don't
259 		 * support PDU retransmission -- terminate the connection.
260 		 */
261 		if (cmdsn != cs->cs_cmdsn) {
262 			CFISCSI_SESSION_UNLOCK(cs);
263 			CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
264 			    "while expected %u; dropping connection",
265 			    cmdsn, cs->cs_cmdsn);
266 			cfiscsi_session_terminate(cs);
267 			return (true);
268 		}
269 		cs->cs_cmdsn++;
270 	}
271 
272 	CFISCSI_SESSION_UNLOCK(cs);
273 
274 	return (false);
275 }
276 
277 static void
278 cfiscsi_pdu_handle(struct icl_pdu *request)
279 {
280 	struct cfiscsi_session *cs;
281 	bool ignore;
282 
283 	cs = PDU_SESSION(request);
284 
285 	ignore = cfiscsi_pdu_update_cmdsn(request);
286 	if (ignore) {
287 		icl_pdu_free(request);
288 		return;
289 	}
290 
291 	/*
292 	 * Handle the PDU; this includes e.g. receiving the remaining
293 	 * part of PDU and submitting the SCSI command to CTL
294 	 * or queueing a reply.  The handling routine is responsible
295 	 * for freeing the PDU when it's no longer needed.
296 	 */
297 	switch (request->ip_bhs->bhs_opcode &
298 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
299 	case ISCSI_BHS_OPCODE_NOP_OUT:
300 		cfiscsi_pdu_handle_nop_out(request);
301 		break;
302 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
303 		cfiscsi_pdu_handle_scsi_command(request);
304 		break;
305 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
306 		cfiscsi_pdu_handle_task_request(request);
307 		break;
308 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
309 		cfiscsi_pdu_handle_data_out(request);
310 		break;
311 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
312 		cfiscsi_pdu_handle_logout_request(request);
313 		break;
314 	default:
315 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
316 		    "opcode 0x%x; dropping connection",
317 		    request->ip_bhs->bhs_opcode);
318 		icl_pdu_free(request);
319 		cfiscsi_session_terminate(cs);
320 	}
321 
322 }
323 
324 static void
325 cfiscsi_receive_callback(struct icl_pdu *request)
326 {
327 	struct cfiscsi_session *cs;
328 
329 	cs = PDU_SESSION(request);
330 
331 #ifdef ICL_KERNEL_PROXY
332 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
333 		if (cs->cs_login_pdu == NULL)
334 			cs->cs_login_pdu = request;
335 		else
336 			icl_pdu_free(request);
337 		cv_signal(&cs->cs_login_cv);
338 		return;
339 	}
340 #endif
341 
342 	cfiscsi_pdu_handle(request);
343 }
344 
345 static void
346 cfiscsi_error_callback(struct icl_conn *ic)
347 {
348 	struct cfiscsi_session *cs;
349 
350 	cs = CONN_SESSION(ic);
351 
352 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
353 	cfiscsi_session_terminate(cs);
354 }
355 
356 static int
357 cfiscsi_pdu_prepare(struct icl_pdu *response)
358 {
359 	struct cfiscsi_session *cs;
360 	struct iscsi_bhs_scsi_response *bhssr;
361 	bool advance_statsn = true;
362 
363 	cs = PDU_SESSION(response);
364 
365 	CFISCSI_SESSION_LOCK_ASSERT(cs);
366 
367 	/*
368 	 * We're only using fields common for all the response
369 	 * (target -> initiator) PDUs.
370 	 */
371 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
372 
373 	/*
374 	 * 10.8.3: "The StatSN for this connection is not advanced
375 	 * after this PDU is sent."
376 	 */
377 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
378 		advance_statsn = false;
379 
380 	/*
381 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
382 	 * StatSN for the connection is not advanced after this PDU is sent."
383 	 */
384 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
385 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
386 		advance_statsn = false;
387 
388 	/*
389 	 * See the comment below - StatSN is not meaningful and must
390 	 * not be advanced.
391 	 */
392 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
393 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
394 		advance_statsn = false;
395 
396 	/*
397 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
398 	 * only have meaningful content if the S bit is set to 1."
399 	 */
400 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
401 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
402 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
403 	bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
404 	bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta);
405 
406 	if (advance_statsn)
407 		cs->cs_statsn++;
408 
409 	return (0);
410 }
411 
412 static void
413 cfiscsi_pdu_queue(struct icl_pdu *response)
414 {
415 	struct cfiscsi_session *cs;
416 
417 	cs = PDU_SESSION(response);
418 
419 	CFISCSI_SESSION_LOCK(cs);
420 	cfiscsi_pdu_prepare(response);
421 	icl_pdu_queue(response);
422 	CFISCSI_SESSION_UNLOCK(cs);
423 }
424 
425 static uint32_t
426 cfiscsi_decode_lun(uint64_t encoded)
427 {
428 	uint8_t lun[8];
429 	uint32_t result;
430 
431 	/*
432 	 * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number,
433 	 * but is in fact an evil, multidimensional structure defined
434 	 * in SCSI Architecture Model 5 (SAM-5), section 4.6.
435 	 */
436 	memcpy(lun, &encoded, sizeof(lun));
437 	switch (lun[0] & 0xC0) {
438 	case 0x00:
439 		if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 ||
440 		    lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) {
441 			CFISCSI_WARN("malformed LUN "
442 			    "(peripheral device addressing method): 0x%jx",
443 			    (uintmax_t)encoded);
444 			result = 0xffffffff;
445 			break;
446 		}
447 		result = lun[1];
448 		break;
449 	case 0x40:
450 		if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 ||
451 		    lun[6] != 0 || lun[7] != 0) {
452 			CFISCSI_WARN("malformed LUN "
453 			    "(flat address space addressing method): 0x%jx",
454 			    (uintmax_t)encoded);
455 			result = 0xffffffff;
456 			break;
457 		}
458 		result = ((lun[0] & 0x3f) << 8) + lun[1];
459 		break;
460 	case 0xC0:
461 		if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 ||
462 		    lun[6] != 0 || lun[7] != 0) {
463 			CFISCSI_WARN("malformed LUN (extended flat "
464 			    "address space addressing method): 0x%jx",
465 			    (uintmax_t)encoded);
466 			result = 0xffffffff;
467 			break;
468 		}
469 		result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
470 	default:
471 		CFISCSI_WARN("unsupported LUN format 0x%jx",
472 		    (uintmax_t)encoded);
473 		result = 0xffffffff;
474 		break;
475 	}
476 
477 	return (result);
478 }
479 
480 static void
481 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
482 {
483 	struct cfiscsi_session *cs;
484 	struct iscsi_bhs_nop_out *bhsno;
485 	struct iscsi_bhs_nop_in *bhsni;
486 	struct icl_pdu *response;
487 	void *data = NULL;
488 	size_t datasize;
489 	int error;
490 
491 	cs = PDU_SESSION(request);
492 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
493 
494 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
495 		/*
496 		 * Nothing to do, iscsi_pdu_update_statsn() already
497 		 * zeroed the timeout.
498 		 */
499 		icl_pdu_free(request);
500 		return;
501 	}
502 
503 	datasize = icl_pdu_data_segment_length(request);
504 	if (datasize > 0) {
505 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
506 		if (data == NULL) {
507 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
508 			    "dropping connection");
509 			icl_pdu_free(request);
510 			cfiscsi_session_terminate(cs);
511 			return;
512 		}
513 		icl_pdu_get_data(request, 0, data, datasize);
514 	}
515 
516 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
517 	if (response == NULL) {
518 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
519 		    "droppping connection");
520 		free(data, M_CFISCSI);
521 		icl_pdu_free(request);
522 		cfiscsi_session_terminate(cs);
523 		return;
524 	}
525 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
526 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
527 	bhsni->bhsni_flags = 0x80;
528 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
529 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
530 	if (datasize > 0) {
531 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
532 		if (error != 0) {
533 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
534 			    "dropping connection");
535 			free(data, M_CFISCSI);
536 			icl_pdu_free(request);
537 			icl_pdu_free(response);
538 			cfiscsi_session_terminate(cs);
539 			return;
540 		}
541 		free(data, M_CFISCSI);
542 	}
543 
544 	icl_pdu_free(request);
545 	cfiscsi_pdu_queue(response);
546 }
547 
548 static void
549 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
550 {
551 	struct iscsi_bhs_scsi_command *bhssc;
552 	struct cfiscsi_session *cs;
553 	union ctl_io *io;
554 	int error;
555 
556 	cs = PDU_SESSION(request);
557 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
558 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
559 	//    bhssc->bhssc_initiator_task_tag);
560 
561 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
562 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
563 		    "ImmediateData=No; dropping connection");
564 		icl_pdu_free(request);
565 		cfiscsi_session_terminate(cs);
566 		return;
567 	}
568 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
569 	ctl_zero_io(io);
570 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
571 	io->io_hdr.io_type = CTL_IO_SCSI;
572 	io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
573 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
574 	io->io_hdr.nexus.targ_target.id = 0;
575 	io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun);
576 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
577 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
578 	case BHSSC_FLAGS_ATTR_UNTAGGED:
579 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
580 		break;
581 	case BHSSC_FLAGS_ATTR_SIMPLE:
582 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
583 		break;
584 	case BHSSC_FLAGS_ATTR_ORDERED:
585         	io->scsiio.tag_type = CTL_TAG_ORDERED;
586 		break;
587 	case BHSSC_FLAGS_ATTR_HOQ:
588         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
589 		break;
590 	case BHSSC_FLAGS_ATTR_ACA:
591 		io->scsiio.tag_type = CTL_TAG_ACA;
592 		break;
593 	default:
594 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
595 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
596 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
597 		break;
598 	}
599 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
600 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
601 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
602 	error = ctl_queue(io);
603 	if (error != CTL_RETVAL_COMPLETE) {
604 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
605 		    "dropping connection", error);
606 		ctl_free_io(io);
607 		refcount_release(&cs->cs_outstanding_ctl_pdus);
608 		icl_pdu_free(request);
609 		cfiscsi_session_terminate(cs);
610 	}
611 }
612 
613 static void
614 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
615 {
616 	struct iscsi_bhs_task_management_request *bhstmr;
617 	struct iscsi_bhs_task_management_response *bhstmr2;
618 	struct icl_pdu *response;
619 	struct cfiscsi_session *cs;
620 	union ctl_io *io;
621 	int error;
622 
623 	cs = PDU_SESSION(request);
624 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
625 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
626 	ctl_zero_io(io);
627 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
628 	io->io_hdr.io_type = CTL_IO_TASK;
629 	io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
630 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
631 	io->io_hdr.nexus.targ_target.id = 0;
632 	io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun);
633 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
634 
635 	switch (bhstmr->bhstmr_function & ~0x80) {
636 	case BHSTMR_FUNCTION_ABORT_TASK:
637 #if 0
638 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
639 #endif
640 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
641 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
642 		break;
643 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
644 #if 0
645 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
646 #endif
647 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
648 		break;
649 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
650 #if 0
651 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
652 #endif
653 		io->taskio.task_action = CTL_TASK_LUN_RESET;
654 		break;
655 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
656 #if 0
657 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
658 #endif
659 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
660 		break;
661 	default:
662 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
663 		    bhstmr->bhstmr_function & ~0x80);
664 		ctl_free_io(io);
665 
666 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
667 		if (response == NULL) {
668 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
669 			    "dropping connection");
670 			icl_pdu_free(request);
671 			cfiscsi_session_terminate(cs);
672 			return;
673 		}
674 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
675 		    response->ip_bhs;
676 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
677 		bhstmr2->bhstmr_flags = 0x80;
678 		bhstmr2->bhstmr_response =
679 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
680 		bhstmr2->bhstmr_initiator_task_tag =
681 		    bhstmr->bhstmr_initiator_task_tag;
682 		icl_pdu_free(request);
683 		cfiscsi_pdu_queue(response);
684 		return;
685 	}
686 
687 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
688 	error = ctl_queue(io);
689 	if (error != CTL_RETVAL_COMPLETE) {
690 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
691 		    "dropping connection", error);
692 		ctl_free_io(io);
693 		refcount_release(&cs->cs_outstanding_ctl_pdus);
694 		icl_pdu_free(request);
695 		cfiscsi_session_terminate(cs);
696 	}
697 }
698 
699 static bool
700 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
701 {
702 	struct iscsi_bhs_data_out *bhsdo;
703 	struct cfiscsi_session *cs;
704 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
705 	size_t copy_len, len, off, buffer_offset;
706 	int ctl_sg_count;
707 	union ctl_io *io;
708 
709 	cs = PDU_SESSION(request);
710 
711 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
712 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
713 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
714 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
715 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
716 
717 	/*
718 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
719 	 */
720 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
721 
722 	io = cdw->cdw_ctl_io;
723 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
724 	    ("CTL_FLAG_DATA_IN"));
725 
726 #if 0
727 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
728 	    request->ip_data_len, io->scsiio.kern_total_len);
729 #endif
730 
731 	if (io->scsiio.kern_sg_entries > 0) {
732 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
733 		ctl_sg_count = io->scsiio.kern_sg_entries;
734 	} else {
735 		ctl_sglist = &ctl_sg_entry;
736 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
737 		ctl_sglist->len = io->scsiio.kern_data_len;
738 		ctl_sg_count = 1;
739 	}
740 
741 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
742 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
743 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
744 	else
745 		buffer_offset = 0;
746 	len = icl_pdu_data_segment_length(request);
747 
748 	/*
749 	 * Make sure the offset, as sent by the initiator, matches the offset
750 	 * we're supposed to be at in the scatter-gather list.
751 	 */
752 	if (buffer_offset >
753 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
754 	    buffer_offset + len <=
755 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
756 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
757 		    "expected %zd; dropping connection", buffer_offset,
758 		    (size_t)io->scsiio.kern_rel_offset +
759 		    (size_t)io->scsiio.ext_data_filled);
760 		ctl_set_data_phase_error(&io->scsiio);
761 		cfiscsi_session_terminate(cs);
762 		return (true);
763 	}
764 
765 	/*
766 	 * This is the offset within the PDU data segment, as opposed
767 	 * to buffer_offset, which is the offset within the task (SCSI
768 	 * command).
769 	 */
770 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
771 	    buffer_offset;
772 
773 	/*
774 	 * Iterate over the scatter/gather segments, filling them with data
775 	 * from the PDU data segment.  Note that this can get called multiple
776 	 * times for one SCSI command; the cdw structure holds state for the
777 	 * scatter/gather list.
778 	 */
779 	for (;;) {
780 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
781 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
782 		if (cdw->cdw_sg_len == 0) {
783 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
784 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
785 		}
786 		KASSERT(off <= len, ("len > off"));
787 		copy_len = len - off;
788 		if (copy_len > cdw->cdw_sg_len)
789 			copy_len = cdw->cdw_sg_len;
790 
791 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
792 		cdw->cdw_sg_addr += copy_len;
793 		cdw->cdw_sg_len -= copy_len;
794 		off += copy_len;
795 		io->scsiio.ext_data_filled += copy_len;
796 
797 		if (cdw->cdw_sg_len == 0) {
798 			/*
799 			 * End of current segment.
800 			 */
801 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
802 				/*
803 				 * Last segment in scatter/gather list.
804 				 */
805 				break;
806 			}
807 			cdw->cdw_sg_index++;
808 		}
809 
810 		if (off == len) {
811 			/*
812 			 * End of PDU payload.
813 			 */
814 			break;
815 		}
816 	}
817 
818 	if (len > off) {
819 		/*
820 		 * In case of unsolicited data, it's possible that the buffer
821 		 * provided by CTL is smaller than negotiated FirstBurstLength.
822 		 * Just ignore the superfluous data; will ask for them with R2T
823 		 * on next call to cfiscsi_datamove().
824 		 *
825 		 * This obviously can only happen with SCSI Command PDU.
826 		 */
827 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
828 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
829 			return (true);
830 
831 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
832 		    "expected %zd; dropping connection",
833 		    icl_pdu_data_segment_length(request), off);
834 		ctl_set_data_phase_error(&io->scsiio);
835 		cfiscsi_session_terminate(cs);
836 		return (true);
837 	}
838 
839 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
840 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
841 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
842 		    "the F flag; flags = 0x%x; dropping connection",
843 		    bhsdo->bhsdo_flags);
844 		ctl_set_data_phase_error(&io->scsiio);
845 		cfiscsi_session_terminate(cs);
846 		return (true);
847 	}
848 
849 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
850 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
851 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
852 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
853 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
854 			    "transmitted size was %zd bytes instead of %d; "
855 			    "dropping connection",
856 			    (size_t)io->scsiio.ext_data_filled,
857 			    cdw->cdw_r2t_end);
858 			ctl_set_data_phase_error(&io->scsiio);
859 			cfiscsi_session_terminate(cs);
860 			return (true);
861 		} else {
862 			/*
863 			 * For SCSI Command PDU, this just means we need to
864 			 * solicit more data by sending R2T.
865 			 */
866 			return (false);
867 		}
868 	}
869 
870 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
871 #if 0
872 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
873 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
874 #endif
875 
876 		return (true);
877 	}
878 
879 	return (false);
880 }
881 
882 static void
883 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
884 {
885 	struct iscsi_bhs_data_out *bhsdo;
886 	struct cfiscsi_session *cs;
887 	struct cfiscsi_data_wait *cdw = NULL;
888 	union ctl_io *io;
889 	bool done;
890 
891 	cs = PDU_SESSION(request);
892 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
893 
894 	CFISCSI_SESSION_LOCK(cs);
895 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
896 #if 0
897 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
898 		    "ttt 0x%x, itt 0x%x",
899 		    bhsdo->bhsdo_target_transfer_tag,
900 		    bhsdo->bhsdo_initiator_task_tag,
901 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
902 #endif
903 		if (bhsdo->bhsdo_target_transfer_tag ==
904 		    cdw->cdw_target_transfer_tag)
905 			break;
906 	}
907 	CFISCSI_SESSION_UNLOCK(cs);
908 	if (cdw == NULL) {
909 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
910 		    "0x%x, not found; dropping connection",
911 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
912 		icl_pdu_free(request);
913 		cfiscsi_session_terminate(cs);
914 		return;
915 	}
916 
917 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
918 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
919 		    "DataSN %u, while expected %u; dropping connection",
920 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
921 		icl_pdu_free(request);
922 		cfiscsi_session_terminate(cs);
923 		return;
924 	}
925 	cdw->cdw_datasn++;
926 
927 	io = cdw->cdw_ctl_io;
928 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
929 	    ("CTL_FLAG_DATA_IN"));
930 
931 	done = cfiscsi_handle_data_segment(request, cdw);
932 	if (done) {
933 		CFISCSI_SESSION_LOCK(cs);
934 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
935 		CFISCSI_SESSION_UNLOCK(cs);
936 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
937 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
938 		cfiscsi_data_wait_free(cs, cdw);
939 		if (done)
940 			io->scsiio.be_move_done(io);
941 		else
942 			cfiscsi_datamove_out(io);
943 	}
944 
945 	icl_pdu_free(request);
946 }
947 
948 static void
949 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
950 {
951 	struct iscsi_bhs_logout_request *bhslr;
952 	struct iscsi_bhs_logout_response *bhslr2;
953 	struct icl_pdu *response;
954 	struct cfiscsi_session *cs;
955 
956 	cs = PDU_SESSION(request);
957 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
958 	switch (bhslr->bhslr_reason & 0x7f) {
959 	case BHSLR_REASON_CLOSE_SESSION:
960 	case BHSLR_REASON_CLOSE_CONNECTION:
961 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
962 		if (response == NULL) {
963 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
964 			icl_pdu_free(request);
965 			cfiscsi_session_terminate(cs);
966 			return;
967 		}
968 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
969 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
970 		bhslr2->bhslr_flags = 0x80;
971 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
972 		bhslr2->bhslr_initiator_task_tag =
973 		    bhslr->bhslr_initiator_task_tag;
974 		icl_pdu_free(request);
975 		cfiscsi_pdu_queue(response);
976 		cfiscsi_session_terminate(cs);
977 		break;
978 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
979 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
980 		if (response == NULL) {
981 			CFISCSI_SESSION_WARN(cs,
982 			    "failed to allocate memory; dropping connection");
983 			icl_pdu_free(request);
984 			cfiscsi_session_terminate(cs);
985 			return;
986 		}
987 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
988 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
989 		bhslr2->bhslr_flags = 0x80;
990 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
991 		bhslr2->bhslr_initiator_task_tag =
992 		    bhslr->bhslr_initiator_task_tag;
993 		icl_pdu_free(request);
994 		cfiscsi_pdu_queue(response);
995 		break;
996 	default:
997 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
998 		    bhslr->bhslr_reason);
999 		icl_pdu_free(request);
1000 		cfiscsi_session_terminate(cs);
1001 		break;
1002 	}
1003 }
1004 
1005 static void
1006 cfiscsi_callout(void *context)
1007 {
1008 	struct icl_pdu *cp;
1009 	struct iscsi_bhs_nop_in *bhsni;
1010 	struct cfiscsi_session *cs;
1011 
1012 	cs = context;
1013 
1014 	if (cs->cs_terminating)
1015 		return;
1016 
1017 	callout_schedule(&cs->cs_callout, 1 * hz);
1018 
1019 	atomic_add_int(&cs->cs_timeout, 1);
1020 
1021 #ifdef ICL_KERNEL_PROXY
1022 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1023 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1024 			CFISCSI_SESSION_WARN(cs, "login timed out after "
1025 			    "%d seconds; dropping connection", cs->cs_timeout);
1026 			cfiscsi_session_terminate(cs);
1027 		}
1028 		return;
1029 	}
1030 #endif
1031 
1032 	if (ping_timeout <= 0) {
1033 		/*
1034 		 * Pings are disabled.  Don't send NOP-In in this case;
1035 		 * user might have disabled pings to work around problems
1036 		 * with certain initiators that can't properly handle
1037 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1038 		 * triggering reconnection, should the user decide to
1039 		 * reenable them.
1040 		 */
1041 		cs->cs_timeout = 0;
1042 		return;
1043 	}
1044 
1045 	if (cs->cs_timeout >= ping_timeout) {
1046 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1047 		    "dropping connection",  ping_timeout);
1048 		cfiscsi_session_terminate(cs);
1049 		return;
1050 	}
1051 
1052 	/*
1053 	 * If the ping was reset less than one second ago - which means
1054 	 * that we've received some PDU during the last second - assume
1055 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1056 	 *
1057 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1058 	 * earlier in this routine.)
1059 	 */
1060 	if (cs->cs_timeout < 2)
1061 		return;
1062 
1063 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1064 	if (cp == NULL) {
1065 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1066 		return;
1067 	}
1068 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1069 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1070 	bhsni->bhsni_flags = 0x80;
1071 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1072 
1073 	cfiscsi_pdu_queue(cp);
1074 }
1075 
1076 static struct cfiscsi_data_wait *
1077 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1078     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1079 {
1080 	struct cfiscsi_data_wait *cdw;
1081 	int error;
1082 
1083 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1084 	if (cdw == NULL) {
1085 		CFISCSI_SESSION_WARN(cs,
1086 		    "failed to allocate %zd bytes", sizeof(*cdw));
1087 		return (NULL);
1088 	}
1089 
1090 	error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1091 	    &cdw->cdw_icl_prv);
1092 	if (error != 0) {
1093 		CFISCSI_SESSION_WARN(cs,
1094 		    "icl_conn_transfer_setup() failed with error %d", error);
1095 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1096 		return (NULL);
1097 	}
1098 
1099 	cdw->cdw_ctl_io = io;
1100 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1101 	cdw->cdw_initiator_task_tag = initiator_task_tag;
1102 
1103 	return (cdw);
1104 }
1105 
1106 static void
1107 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1108     struct cfiscsi_data_wait *cdw)
1109 {
1110 
1111 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1112 	uma_zfree(cfiscsi_data_wait_zone, cdw);
1113 }
1114 
1115 static void
1116 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1117 {
1118 	struct cfiscsi_data_wait *cdw;
1119 	union ctl_io *io;
1120 	int error, last, wait;
1121 
1122 	if (cs->cs_target == NULL)
1123 		return;		/* No target yet, so nothing to do. */
1124 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1125 	ctl_zero_io(io);
1126 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs;
1127 	io->io_hdr.io_type = CTL_IO_TASK;
1128 	io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1129 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1130 	io->io_hdr.nexus.targ_target.id = 0;
1131 	io->io_hdr.nexus.targ_lun = 0;
1132 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1133 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1134 	wait = cs->cs_outstanding_ctl_pdus;
1135 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1136 	error = ctl_queue(io);
1137 	if (error != CTL_RETVAL_COMPLETE) {
1138 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1139 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1140 		ctl_free_io(io);
1141 	}
1142 
1143 	CFISCSI_SESSION_LOCK(cs);
1144 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1145 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1146 		CFISCSI_SESSION_UNLOCK(cs);
1147 		/*
1148 		 * Set nonzero port status; this prevents backends from
1149 		 * assuming that the data transfer actually succeeded
1150 		 * and writing uninitialized data to disk.
1151 		 */
1152 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1153 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1154 		cfiscsi_data_wait_free(cs, cdw);
1155 		CFISCSI_SESSION_LOCK(cs);
1156 	}
1157 	CFISCSI_SESSION_UNLOCK(cs);
1158 
1159 	/*
1160 	 * Wait for CTL to terminate all the tasks.
1161 	 */
1162 	if (wait > 0)
1163 		CFISCSI_SESSION_WARN(cs,
1164 		    "waiting for CTL to terminate %d tasks", wait);
1165 	for (;;) {
1166 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1167 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1168 		if (last != 0)
1169 			break;
1170 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1171 		    0, "cfiscsi_terminate", hz / 100);
1172 	}
1173 	if (wait > 0)
1174 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1175 }
1176 
1177 static void
1178 cfiscsi_maintenance_thread(void *arg)
1179 {
1180 	struct cfiscsi_session *cs;
1181 
1182 	cs = arg;
1183 
1184 	for (;;) {
1185 		CFISCSI_SESSION_LOCK(cs);
1186 		if (cs->cs_terminating == false)
1187 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1188 		CFISCSI_SESSION_UNLOCK(cs);
1189 
1190 		if (cs->cs_terminating) {
1191 
1192 			/*
1193 			 * We used to wait up to 30 seconds to deliver queued
1194 			 * PDUs to the initiator.  We also tried hard to deliver
1195 			 * SCSI Responses for the aborted PDUs.  We don't do
1196 			 * that anymore.  We might need to revisit that.
1197 			 */
1198 			callout_drain(&cs->cs_callout);
1199 			icl_conn_close(cs->cs_conn);
1200 
1201 			/*
1202 			 * At this point ICL receive thread is no longer
1203 			 * running; no new tasks can be queued.
1204 			 */
1205 			cfiscsi_session_terminate_tasks(cs);
1206 			cfiscsi_session_delete(cs);
1207 			kthread_exit();
1208 			return;
1209 		}
1210 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1211 	}
1212 }
1213 
1214 static void
1215 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1216 {
1217 
1218 	if (cs->cs_terminating)
1219 		return;
1220 	cs->cs_terminating = true;
1221 	cv_signal(&cs->cs_maintenance_cv);
1222 #ifdef ICL_KERNEL_PROXY
1223 	cv_signal(&cs->cs_login_cv);
1224 #endif
1225 }
1226 
1227 static int
1228 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1229 {
1230 	struct cfiscsi_target *ct;
1231 	char *name;
1232 	int i;
1233 
1234 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1235 
1236 	ct = cs->cs_target;
1237 	name = strdup(cs->cs_initiator_id, M_CTL);
1238 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1239 	if (i < 0) {
1240 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1241 		    i);
1242 		cs->cs_ctl_initid = -1;
1243 		return (1);
1244 	}
1245 	cs->cs_ctl_initid = i;
1246 #if 0
1247 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1248 #endif
1249 
1250 	return (0);
1251 }
1252 
1253 static void
1254 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1255 {
1256 	int error;
1257 
1258 	if (cs->cs_ctl_initid == -1)
1259 		return;
1260 
1261 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1262 	if (error != 0) {
1263 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1264 		    error);
1265 	}
1266 	cs->cs_ctl_initid = -1;
1267 }
1268 
1269 static struct cfiscsi_session *
1270 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1271 {
1272 	struct cfiscsi_session *cs;
1273 	int error;
1274 
1275 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1276 	if (cs == NULL) {
1277 		CFISCSI_WARN("malloc failed");
1278 		return (NULL);
1279 	}
1280 	cs->cs_ctl_initid = -1;
1281 
1282 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1283 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1284 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1285 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1286 #ifdef ICL_KERNEL_PROXY
1287 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1288 #endif
1289 
1290 	cs->cs_conn = icl_new_conn(offload, "cfiscsi", &cs->cs_lock);
1291 	if (cs->cs_conn == NULL) {
1292 		free(cs, M_CFISCSI);
1293 		return (NULL);
1294 	}
1295 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1296 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1297 	cs->cs_conn->ic_prv0 = cs;
1298 
1299 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1300 	if (error != 0) {
1301 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1302 		free(cs, M_CFISCSI);
1303 		return (NULL);
1304 	}
1305 
1306 	mtx_lock(&softc->lock);
1307 	cs->cs_id = ++softc->last_session_id;
1308 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1309 	mtx_unlock(&softc->lock);
1310 
1311 	/*
1312 	 * Start pinging the initiator.
1313 	 */
1314 	callout_init(&cs->cs_callout, 1);
1315 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1316 
1317 	return (cs);
1318 }
1319 
1320 static void
1321 cfiscsi_session_delete(struct cfiscsi_session *cs)
1322 {
1323 	struct cfiscsi_softc *softc;
1324 
1325 	softc = &cfiscsi_softc;
1326 
1327 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1328 	    ("destroying session with outstanding CTL pdus"));
1329 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1330 	    ("destroying session with non-empty queue"));
1331 
1332 	cfiscsi_session_unregister_initiator(cs);
1333 	if (cs->cs_target != NULL)
1334 		cfiscsi_target_release(cs->cs_target);
1335 	icl_conn_close(cs->cs_conn);
1336 	icl_conn_free(cs->cs_conn);
1337 
1338 	mtx_lock(&softc->lock);
1339 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1340 	cv_signal(&softc->sessions_cv);
1341 	mtx_unlock(&softc->lock);
1342 
1343 	free(cs, M_CFISCSI);
1344 }
1345 
1346 int
1347 cfiscsi_init(void)
1348 {
1349 	struct cfiscsi_softc *softc;
1350 	int retval;
1351 
1352 	softc = &cfiscsi_softc;
1353 	retval = 0;
1354 	bzero(softc, sizeof(*softc));
1355 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1356 
1357 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1358 #ifdef ICL_KERNEL_PROXY
1359 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1360 #endif
1361 	TAILQ_INIT(&softc->sessions);
1362 	TAILQ_INIT(&softc->targets);
1363 
1364 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1365 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1366 	    UMA_ALIGN_PTR, 0);
1367 
1368 	return (0);
1369 }
1370 
1371 #ifdef ICL_KERNEL_PROXY
1372 static void
1373 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1374 {
1375 	struct cfiscsi_session *cs;
1376 
1377 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1378 	if (cs == NULL) {
1379 		CFISCSI_WARN("failed to create session");
1380 		return;
1381 	}
1382 
1383 	icl_conn_handoff_sock(cs->cs_conn, so);
1384 	cs->cs_initiator_sa = sa;
1385 	cs->cs_portal_id = portal_id;
1386 	cs->cs_waiting_for_ctld = true;
1387 	cv_signal(&cfiscsi_softc.accept_cv);
1388 }
1389 #endif
1390 
1391 static void
1392 cfiscsi_online(void *arg)
1393 {
1394 	struct cfiscsi_softc *softc;
1395 	struct cfiscsi_target *ct;
1396 	int online;
1397 
1398 	ct = (struct cfiscsi_target *)arg;
1399 	softc = ct->ct_softc;
1400 
1401 	mtx_lock(&softc->lock);
1402 	if (ct->ct_online) {
1403 		mtx_unlock(&softc->lock);
1404 		return;
1405 	}
1406 	ct->ct_online = 1;
1407 	online = softc->online++;
1408 	mtx_unlock(&softc->lock);
1409 	if (online > 0)
1410 		return;
1411 
1412 #ifdef ICL_KERNEL_PROXY
1413 	if (softc->listener != NULL)
1414 		icl_listen_free(softc->listener);
1415 	softc->listener = icl_listen_new(cfiscsi_accept);
1416 #endif
1417 }
1418 
1419 static void
1420 cfiscsi_offline(void *arg)
1421 {
1422 	struct cfiscsi_softc *softc;
1423 	struct cfiscsi_target *ct;
1424 	struct cfiscsi_session *cs;
1425 	int online;
1426 
1427 	ct = (struct cfiscsi_target *)arg;
1428 	softc = ct->ct_softc;
1429 
1430 	mtx_lock(&softc->lock);
1431 	if (!ct->ct_online) {
1432 		mtx_unlock(&softc->lock);
1433 		return;
1434 	}
1435 	ct->ct_online = 0;
1436 	online = --softc->online;
1437 
1438 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1439 		if (cs->cs_target == ct)
1440 			cfiscsi_session_terminate(cs);
1441 	}
1442 	do {
1443 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1444 			if (cs->cs_target == ct)
1445 				break;
1446 		}
1447 		if (cs != NULL)
1448 			cv_wait(&softc->sessions_cv, &softc->lock);
1449 	} while (cs != NULL && ct->ct_online == 0);
1450 	mtx_unlock(&softc->lock);
1451 	if (online > 0)
1452 		return;
1453 
1454 #ifdef ICL_KERNEL_PROXY
1455 	icl_listen_free(softc->listener);
1456 	softc->listener = NULL;
1457 #endif
1458 }
1459 
1460 static int
1461 cfiscsi_info(void *arg, struct sbuf *sb)
1462 {
1463 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1464 	int retval;
1465 
1466 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1467 	    ct->ct_state);
1468 	return (retval);
1469 }
1470 
1471 static void
1472 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1473 {
1474 	struct cfiscsi_softc *softc;
1475 	struct cfiscsi_session *cs, *cs2;
1476 	struct cfiscsi_target *ct;
1477 	struct ctl_iscsi_handoff_params *cihp;
1478 	int error;
1479 
1480 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1481 	softc = &cfiscsi_softc;
1482 
1483 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1484 	    cihp->initiator_name, cihp->initiator_addr,
1485 	    cihp->target_name);
1486 
1487 	ct = cfiscsi_target_find(softc, cihp->target_name,
1488 	    cihp->portal_group_tag);
1489 	if (ct == NULL) {
1490 		ci->status = CTL_ISCSI_ERROR;
1491 		snprintf(ci->error_str, sizeof(ci->error_str),
1492 		    "%s: target not found", __func__);
1493 		return;
1494 	}
1495 
1496 #ifdef ICL_KERNEL_PROXY
1497 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1498 		snprintf(ci->error_str, sizeof(ci->error_str),
1499 		    "both socket and connection_id set");
1500 		ci->status = CTL_ISCSI_ERROR;
1501 		cfiscsi_target_release(ct);
1502 		return;
1503 	}
1504 	if (cihp->socket == 0) {
1505 		mtx_lock(&cfiscsi_softc.lock);
1506 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1507 			if (cs->cs_id == cihp->connection_id)
1508 				break;
1509 		}
1510 		if (cs == NULL) {
1511 			mtx_unlock(&cfiscsi_softc.lock);
1512 			snprintf(ci->error_str, sizeof(ci->error_str),
1513 			    "connection not found");
1514 			ci->status = CTL_ISCSI_ERROR;
1515 			cfiscsi_target_release(ct);
1516 			return;
1517 		}
1518 		mtx_unlock(&cfiscsi_softc.lock);
1519 	} else {
1520 #endif
1521 		cs = cfiscsi_session_new(softc, cihp->offload);
1522 		if (cs == NULL) {
1523 			ci->status = CTL_ISCSI_ERROR;
1524 			snprintf(ci->error_str, sizeof(ci->error_str),
1525 			    "%s: cfiscsi_session_new failed", __func__);
1526 			cfiscsi_target_release(ct);
1527 			return;
1528 		}
1529 #ifdef ICL_KERNEL_PROXY
1530 	}
1531 #endif
1532 
1533 	/*
1534 	 * First PDU of Full Feature phase has the same CmdSN as the last
1535 	 * PDU from the Login Phase received from the initiator.  Thus,
1536 	 * the -1 below.
1537 	 */
1538 	cs->cs_cmdsn = cihp->cmdsn;
1539 	cs->cs_statsn = cihp->statsn;
1540 	cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length;
1541 	cs->cs_max_burst_length = cihp->max_burst_length;
1542 	cs->cs_immediate_data = !!cihp->immediate_data;
1543 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1544 		cs->cs_conn->ic_header_crc32c = true;
1545 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1546 		cs->cs_conn->ic_data_crc32c = true;
1547 
1548 	strlcpy(cs->cs_initiator_name,
1549 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1550 	strlcpy(cs->cs_initiator_addr,
1551 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1552 	strlcpy(cs->cs_initiator_alias,
1553 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1554 	memcpy(cs->cs_initiator_isid,
1555 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1556 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1557 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1558 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1559 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1560 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1561 
1562 	mtx_lock(&softc->lock);
1563 	if (ct->ct_online == 0) {
1564 		mtx_unlock(&softc->lock);
1565 		cfiscsi_session_terminate(cs);
1566 		cfiscsi_target_release(ct);
1567 		ci->status = CTL_ISCSI_ERROR;
1568 		snprintf(ci->error_str, sizeof(ci->error_str),
1569 		    "%s: port offline", __func__);
1570 		return;
1571 	}
1572 	cs->cs_target = ct;
1573 	mtx_unlock(&softc->lock);
1574 
1575 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1576 restart:
1577 	if (!cs->cs_terminating) {
1578 		mtx_lock(&softc->lock);
1579 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1580 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1581 			    cs->cs_target == cs2->cs_target &&
1582 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1583 				cfiscsi_session_terminate(cs2);
1584 				mtx_unlock(&softc->lock);
1585 				pause("cfiscsi_reinstate", 1);
1586 				goto restart;
1587 			}
1588 		}
1589 		mtx_unlock(&softc->lock);
1590 	}
1591 
1592 	/*
1593 	 * Register initiator with CTL.
1594 	 */
1595 	cfiscsi_session_register_initiator(cs);
1596 
1597 #ifdef ICL_KERNEL_PROXY
1598 	if (cihp->socket > 0) {
1599 #endif
1600 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1601 		if (error != 0) {
1602 			cfiscsi_session_terminate(cs);
1603 			refcount_release(&cs->cs_outstanding_ctl_pdus);
1604 			ci->status = CTL_ISCSI_ERROR;
1605 			snprintf(ci->error_str, sizeof(ci->error_str),
1606 			    "%s: icl_conn_handoff failed with error %d",
1607 			    __func__, error);
1608 			return;
1609 		}
1610 #ifdef ICL_KERNEL_PROXY
1611 	}
1612 #endif
1613 
1614 #ifdef ICL_KERNEL_PROXY
1615 	cs->cs_login_phase = false;
1616 
1617 	/*
1618 	 * First PDU of the Full Feature phase has likely already arrived.
1619 	 * We have to pick it up and execute properly.
1620 	 */
1621 	if (cs->cs_login_pdu != NULL) {
1622 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1623 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1624 		cs->cs_login_pdu = NULL;
1625 	}
1626 #endif
1627 
1628 	refcount_release(&cs->cs_outstanding_ctl_pdus);
1629 	ci->status = CTL_ISCSI_OK;
1630 }
1631 
1632 static void
1633 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1634 {
1635 	struct ctl_iscsi_list_params *cilp;
1636 	struct cfiscsi_session *cs;
1637 	struct cfiscsi_softc *softc;
1638 	struct sbuf *sb;
1639 	int error;
1640 
1641 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1642 	softc = &cfiscsi_softc;
1643 
1644 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1645 	if (sb == NULL) {
1646 		ci->status = CTL_ISCSI_ERROR;
1647 		snprintf(ci->error_str, sizeof(ci->error_str),
1648 		    "Unable to allocate %d bytes for iSCSI session list",
1649 		    cilp->alloc_len);
1650 		return;
1651 	}
1652 
1653 	sbuf_printf(sb, "<ctlislist>\n");
1654 	mtx_lock(&softc->lock);
1655 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1656 #ifdef ICL_KERNEL_PROXY
1657 		if (cs->cs_target == NULL)
1658 			continue;
1659 #endif
1660 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1661 		    "<initiator>%s</initiator>"
1662 		    "<initiator_addr>%s</initiator_addr>"
1663 		    "<initiator_alias>%s</initiator_alias>"
1664 		    "<target>%s</target>"
1665 		    "<target_alias>%s</target_alias>"
1666 		    "<target_portal_group_tag>%u</target_portal_group_tag>"
1667 		    "<header_digest>%s</header_digest>"
1668 		    "<data_digest>%s</data_digest>"
1669 		    "<max_data_segment_length>%zd</max_data_segment_length>"
1670 		    "<immediate_data>%d</immediate_data>"
1671 		    "<iser>%d</iser>"
1672 		    "<offload>%s</offload>"
1673 		    "</connection>\n",
1674 		    cs->cs_id,
1675 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1676 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1677 		    cs->cs_target->ct_tag,
1678 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1679 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1680 		    cs->cs_max_data_segment_length,
1681 		    cs->cs_immediate_data,
1682 		    cs->cs_conn->ic_iser,
1683 		    cs->cs_conn->ic_offload);
1684 		if (error != 0)
1685 			break;
1686 	}
1687 	mtx_unlock(&softc->lock);
1688 	error = sbuf_printf(sb, "</ctlislist>\n");
1689 	if (error != 0) {
1690 		sbuf_delete(sb);
1691 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1692 		snprintf(ci->error_str, sizeof(ci->error_str),
1693 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1694 		return;
1695 	}
1696 	sbuf_finish(sb);
1697 
1698 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1699 	cilp->fill_len = sbuf_len(sb) + 1;
1700 	ci->status = CTL_ISCSI_OK;
1701 	sbuf_delete(sb);
1702 }
1703 
1704 static void
1705 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1706 {
1707 	struct icl_pdu *response;
1708 	struct iscsi_bhs_asynchronous_message *bhsam;
1709 	struct ctl_iscsi_logout_params *cilp;
1710 	struct cfiscsi_session *cs;
1711 	struct cfiscsi_softc *softc;
1712 	int found = 0;
1713 
1714 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1715 	softc = &cfiscsi_softc;
1716 
1717 	mtx_lock(&softc->lock);
1718 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1719 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1720 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1721 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1722 			continue;
1723 
1724 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1725 		if (response == NULL) {
1726 			ci->status = CTL_ISCSI_ERROR;
1727 			snprintf(ci->error_str, sizeof(ci->error_str),
1728 			    "Unable to allocate memory");
1729 			mtx_unlock(&softc->lock);
1730 			return;
1731 		}
1732 		bhsam =
1733 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1734 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1735 		bhsam->bhsam_flags = 0x80;
1736 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1737 		bhsam->bhsam_parameter3 = htons(10);
1738 		cfiscsi_pdu_queue(response);
1739 		found++;
1740 	}
1741 	mtx_unlock(&softc->lock);
1742 
1743 	if (found == 0) {
1744 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1745 		snprintf(ci->error_str, sizeof(ci->error_str),
1746 		    "No matching connections found");
1747 		return;
1748 	}
1749 
1750 	ci->status = CTL_ISCSI_OK;
1751 }
1752 
1753 static void
1754 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1755 {
1756 	struct icl_pdu *response;
1757 	struct iscsi_bhs_asynchronous_message *bhsam;
1758 	struct ctl_iscsi_terminate_params *citp;
1759 	struct cfiscsi_session *cs;
1760 	struct cfiscsi_softc *softc;
1761 	int found = 0;
1762 
1763 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1764 	softc = &cfiscsi_softc;
1765 
1766 	mtx_lock(&softc->lock);
1767 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1768 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1769 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1770 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1771 			continue;
1772 
1773 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1774 		if (response == NULL) {
1775 			/*
1776 			 * Oh well.  Just terminate the connection.
1777 			 */
1778 		} else {
1779 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1780 			    response->ip_bhs;
1781 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1782 			bhsam->bhsam_flags = 0x80;
1783 			bhsam->bhsam_0xffffffff = 0xffffffff;
1784 			bhsam->bhsam_async_event =
1785 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1786 			cfiscsi_pdu_queue(response);
1787 		}
1788 		cfiscsi_session_terminate(cs);
1789 		found++;
1790 	}
1791 	mtx_unlock(&softc->lock);
1792 
1793 	if (found == 0) {
1794 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1795 		snprintf(ci->error_str, sizeof(ci->error_str),
1796 		    "No matching connections found");
1797 		return;
1798 	}
1799 
1800 	ci->status = CTL_ISCSI_OK;
1801 }
1802 
1803 static void
1804 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1805 {
1806 	struct ctl_iscsi_limits_params *cilp;
1807 	int error;
1808 
1809 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1810 
1811 	error = icl_limits(cilp->offload, &cilp->data_segment_limit);
1812 	if (error != 0) {
1813 		ci->status = CTL_ISCSI_ERROR;
1814 		snprintf(ci->error_str, sizeof(ci->error_str),
1815 			"%s: icl_limits failed with error %d",
1816 			__func__, error);
1817 		return;
1818 	}
1819 
1820 	ci->status = CTL_ISCSI_OK;
1821 }
1822 
1823 #ifdef ICL_KERNEL_PROXY
1824 static void
1825 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1826 {
1827 	struct ctl_iscsi_listen_params *cilp;
1828 	struct sockaddr *sa;
1829 	int error;
1830 
1831 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1832 
1833 	if (cfiscsi_softc.listener == NULL) {
1834 		CFISCSI_DEBUG("no listener");
1835 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1836 		ci->status = CTL_ISCSI_ERROR;
1837 		return;
1838 	}
1839 
1840 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1841 	if (error != 0) {
1842 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1843 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1844 		ci->status = CTL_ISCSI_ERROR;
1845 		return;
1846 	}
1847 
1848 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1849 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1850 	if (error != 0) {
1851 		free(sa, M_SONAME);
1852 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1853 		snprintf(ci->error_str, sizeof(ci->error_str),
1854 		    "icl_listen_add failed, error %d", error);
1855 		ci->status = CTL_ISCSI_ERROR;
1856 		return;
1857 	}
1858 
1859 	ci->status = CTL_ISCSI_OK;
1860 }
1861 
1862 static void
1863 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1864 {
1865 	struct ctl_iscsi_accept_params *ciap;
1866 	struct cfiscsi_session *cs;
1867 	int error;
1868 
1869 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1870 
1871 	mtx_lock(&cfiscsi_softc.lock);
1872 	for (;;) {
1873 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1874 			if (cs->cs_waiting_for_ctld)
1875 				break;
1876 		}
1877 		if (cs != NULL)
1878 			break;
1879 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1880 		if (error != 0) {
1881 			mtx_unlock(&cfiscsi_softc.lock);
1882 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1883 			ci->status = CTL_ISCSI_ERROR;
1884 			return;
1885 		}
1886 	}
1887 	mtx_unlock(&cfiscsi_softc.lock);
1888 
1889 	cs->cs_waiting_for_ctld = false;
1890 	cs->cs_login_phase = true;
1891 
1892 	ciap->connection_id = cs->cs_id;
1893 	ciap->portal_id = cs->cs_portal_id;
1894 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1895 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1896 	    cs->cs_initiator_sa->sa_len);
1897 	if (error != 0) {
1898 		snprintf(ci->error_str, sizeof(ci->error_str),
1899 		    "copyout failed with error %d", error);
1900 		ci->status = CTL_ISCSI_ERROR;
1901 		return;
1902 	}
1903 
1904 	ci->status = CTL_ISCSI_OK;
1905 }
1906 
1907 static void
1908 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1909 {
1910 	struct ctl_iscsi_send_params *cisp;
1911 	struct cfiscsi_session *cs;
1912 	struct icl_pdu *ip;
1913 	size_t datalen;
1914 	void *data;
1915 	int error;
1916 
1917 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1918 
1919 	mtx_lock(&cfiscsi_softc.lock);
1920 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1921 		if (cs->cs_id == cisp->connection_id)
1922 			break;
1923 	}
1924 	if (cs == NULL) {
1925 		mtx_unlock(&cfiscsi_softc.lock);
1926 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1927 		ci->status = CTL_ISCSI_ERROR;
1928 		return;
1929 	}
1930 	mtx_unlock(&cfiscsi_softc.lock);
1931 
1932 #if 0
1933 	if (cs->cs_login_phase == false)
1934 		return (EBUSY);
1935 #endif
1936 
1937 	if (cs->cs_terminating) {
1938 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1939 		ci->status = CTL_ISCSI_ERROR;
1940 		return;
1941 	}
1942 
1943 	datalen = cisp->data_segment_len;
1944 	/*
1945 	 * XXX
1946 	 */
1947 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1948 	if (datalen > 65535) {
1949 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1950 		ci->status = CTL_ISCSI_ERROR;
1951 		return;
1952 	}
1953 	if (datalen > 0) {
1954 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
1955 		error = copyin(cisp->data_segment, data, datalen);
1956 		if (error != 0) {
1957 			free(data, M_CFISCSI);
1958 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1959 			ci->status = CTL_ISCSI_ERROR;
1960 			return;
1961 		}
1962 	}
1963 
1964 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
1965 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1966 	if (datalen > 0) {
1967 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1968 		free(data, M_CFISCSI);
1969 	}
1970 	CFISCSI_SESSION_LOCK(cs);
1971 	icl_pdu_queue(ip);
1972 	CFISCSI_SESSION_UNLOCK(cs);
1973 	ci->status = CTL_ISCSI_OK;
1974 }
1975 
1976 static void
1977 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1978 {
1979 	struct ctl_iscsi_receive_params *cirp;
1980 	struct cfiscsi_session *cs;
1981 	struct icl_pdu *ip;
1982 	void *data;
1983 	int error;
1984 
1985 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
1986 
1987 	mtx_lock(&cfiscsi_softc.lock);
1988 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1989 		if (cs->cs_id == cirp->connection_id)
1990 			break;
1991 	}
1992 	if (cs == NULL) {
1993 		mtx_unlock(&cfiscsi_softc.lock);
1994 		snprintf(ci->error_str, sizeof(ci->error_str),
1995 		    "connection not found");
1996 		ci->status = CTL_ISCSI_ERROR;
1997 		return;
1998 	}
1999 	mtx_unlock(&cfiscsi_softc.lock);
2000 
2001 #if 0
2002 	if (is->is_login_phase == false)
2003 		return (EBUSY);
2004 #endif
2005 
2006 	CFISCSI_SESSION_LOCK(cs);
2007 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2008 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2009 		if (error != 0) {
2010 			CFISCSI_SESSION_UNLOCK(cs);
2011 			snprintf(ci->error_str, sizeof(ci->error_str),
2012 			    "interrupted by signal");
2013 			ci->status = CTL_ISCSI_ERROR;
2014 			return;
2015 		}
2016 	}
2017 
2018 	if (cs->cs_terminating) {
2019 		CFISCSI_SESSION_UNLOCK(cs);
2020 		snprintf(ci->error_str, sizeof(ci->error_str),
2021 		    "connection terminating");
2022 		ci->status = CTL_ISCSI_ERROR;
2023 		return;
2024 	}
2025 	ip = cs->cs_login_pdu;
2026 	cs->cs_login_pdu = NULL;
2027 	CFISCSI_SESSION_UNLOCK(cs);
2028 
2029 	if (ip->ip_data_len > cirp->data_segment_len) {
2030 		icl_pdu_free(ip);
2031 		snprintf(ci->error_str, sizeof(ci->error_str),
2032 		    "data segment too big");
2033 		ci->status = CTL_ISCSI_ERROR;
2034 		return;
2035 	}
2036 
2037 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2038 	if (ip->ip_data_len > 0) {
2039 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2040 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2041 		copyout(data, cirp->data_segment, ip->ip_data_len);
2042 		free(data, M_CFISCSI);
2043 	}
2044 
2045 	icl_pdu_free(ip);
2046 	ci->status = CTL_ISCSI_OK;
2047 }
2048 
2049 #endif /* !ICL_KERNEL_PROXY */
2050 
2051 static void
2052 cfiscsi_ioctl_port_create(struct ctl_req *req)
2053 {
2054 	struct cfiscsi_target *ct;
2055 	struct ctl_port *port;
2056 	const char *target, *alias, *tags;
2057 	struct scsi_vpd_id_descriptor *desc;
2058 	ctl_options_t opts;
2059 	int retval, len, idlen;
2060 	uint16_t tag;
2061 
2062 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2063 	target = ctl_get_opt(&opts, "cfiscsi_target");
2064 	alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
2065 	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
2066 	if (target == NULL || tags == NULL) {
2067 		req->status = CTL_LUN_ERROR;
2068 		snprintf(req->error_str, sizeof(req->error_str),
2069 		    "Missing required argument");
2070 		ctl_free_opts(&opts);
2071 		return;
2072 	}
2073 	tag = strtol(tags, (char **)NULL, 10);
2074 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2075 	if (ct == NULL) {
2076 		req->status = CTL_LUN_ERROR;
2077 		snprintf(req->error_str, sizeof(req->error_str),
2078 		    "failed to create target \"%s\"", target);
2079 		ctl_free_opts(&opts);
2080 		return;
2081 	}
2082 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2083 		req->status = CTL_LUN_ERROR;
2084 		snprintf(req->error_str, sizeof(req->error_str),
2085 		    "target \"%s\" already exists", target);
2086 		cfiscsi_target_release(ct);
2087 		ctl_free_opts(&opts);
2088 		return;
2089 	}
2090 	port = &ct->ct_port;
2091 	// WAT
2092 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2093 		goto done;
2094 
2095 	port->frontend = &cfiscsi_frontend;
2096 	port->port_type = CTL_PORT_ISCSI;
2097 	/* XXX KDM what should the real number be here? */
2098 	port->num_requested_ctl_io = 4096;
2099 	port->port_name = "iscsi";
2100 	port->physical_port = tag;
2101 	port->virtual_port = ct->ct_target_id;
2102 	port->port_online = cfiscsi_online;
2103 	port->port_offline = cfiscsi_offline;
2104 	port->port_info = cfiscsi_info;
2105 	port->onoff_arg = ct;
2106 	port->lun_enable = cfiscsi_lun_enable;
2107 	port->lun_disable = cfiscsi_lun_disable;
2108 	port->targ_lun_arg = ct;
2109 	port->fe_datamove = cfiscsi_datamove;
2110 	port->fe_done = cfiscsi_done;
2111 
2112 	/* XXX KDM what should we report here? */
2113 	/* XXX These should probably be fetched from CTL. */
2114 	port->max_targets = 1;
2115 	port->max_target_id = 15;
2116 
2117 	port->options = opts;
2118 	STAILQ_INIT(&opts);
2119 
2120 	/* Generate Port ID. */
2121 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2122 	idlen = roundup2(idlen, 4);
2123 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2124 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2125 	    M_CTL, M_WAITOK | M_ZERO);
2126 	port->port_devid->len = len;
2127 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2128 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2129 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2130 	    SVPD_ID_TYPE_SCSI_NAME;
2131 	desc->length = idlen;
2132 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2133 
2134 	/* Generate Target ID. */
2135 	idlen = strlen(target) + 1;
2136 	idlen = roundup2(idlen, 4);
2137 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2138 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2139 	    M_CTL, M_WAITOK | M_ZERO);
2140 	port->target_devid->len = len;
2141 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2142 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2143 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2144 	    SVPD_ID_TYPE_SCSI_NAME;
2145 	desc->length = idlen;
2146 	strlcpy(desc->identifier, target, idlen);
2147 
2148 	retval = ctl_port_register(port);
2149 	if (retval != 0) {
2150 		ctl_free_opts(&port->options);
2151 		cfiscsi_target_release(ct);
2152 		free(port->port_devid, M_CFISCSI);
2153 		free(port->target_devid, M_CFISCSI);
2154 		req->status = CTL_LUN_ERROR;
2155 		snprintf(req->error_str, sizeof(req->error_str),
2156 		    "ctl_port_register() failed with error %d", retval);
2157 		return;
2158 	}
2159 done:
2160 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2161 	req->status = CTL_LUN_OK;
2162 	memcpy(req->kern_args[0].kvalue, &port->targ_port,
2163 	    sizeof(port->targ_port)); //XXX
2164 }
2165 
2166 static void
2167 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2168 {
2169 	struct cfiscsi_target *ct;
2170 	const char *target, *tags;
2171 	ctl_options_t opts;
2172 	uint16_t tag;
2173 
2174 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2175 	target = ctl_get_opt(&opts, "cfiscsi_target");
2176 	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
2177 	if (target == NULL || tags == NULL) {
2178 		ctl_free_opts(&opts);
2179 		req->status = CTL_LUN_ERROR;
2180 		snprintf(req->error_str, sizeof(req->error_str),
2181 		    "Missing required argument");
2182 		return;
2183 	}
2184 	tag = strtol(tags, (char **)NULL, 10);
2185 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2186 	if (ct == NULL) {
2187 		ctl_free_opts(&opts);
2188 		req->status = CTL_LUN_ERROR;
2189 		snprintf(req->error_str, sizeof(req->error_str),
2190 		    "can't find target \"%s\"", target);
2191 		return;
2192 	}
2193 	if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
2194 		ctl_free_opts(&opts);
2195 		req->status = CTL_LUN_ERROR;
2196 		snprintf(req->error_str, sizeof(req->error_str),
2197 		    "target \"%s\" is already dying", target);
2198 		return;
2199 	}
2200 	ctl_free_opts(&opts);
2201 
2202 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2203 	ctl_port_offline(&ct->ct_port);
2204 	cfiscsi_target_release(ct);
2205 	cfiscsi_target_release(ct);
2206 	req->status = CTL_LUN_OK;
2207 }
2208 
2209 static int
2210 cfiscsi_ioctl(struct cdev *dev,
2211     u_long cmd, caddr_t addr, int flag, struct thread *td)
2212 {
2213 	struct ctl_iscsi *ci;
2214 	struct ctl_req *req;
2215 
2216 	if (cmd == CTL_PORT_REQ) {
2217 		req = (struct ctl_req *)addr;
2218 		switch (req->reqtype) {
2219 		case CTL_REQ_CREATE:
2220 			cfiscsi_ioctl_port_create(req);
2221 			break;
2222 		case CTL_REQ_REMOVE:
2223 			cfiscsi_ioctl_port_remove(req);
2224 			break;
2225 		default:
2226 			req->status = CTL_LUN_ERROR;
2227 			snprintf(req->error_str, sizeof(req->error_str),
2228 			    "Unsupported request type %d", req->reqtype);
2229 		}
2230 		return (0);
2231 	}
2232 
2233 	if (cmd != CTL_ISCSI)
2234 		return (ENOTTY);
2235 
2236 	ci = (struct ctl_iscsi *)addr;
2237 	switch (ci->type) {
2238 	case CTL_ISCSI_HANDOFF:
2239 		cfiscsi_ioctl_handoff(ci);
2240 		break;
2241 	case CTL_ISCSI_LIST:
2242 		cfiscsi_ioctl_list(ci);
2243 		break;
2244 	case CTL_ISCSI_LOGOUT:
2245 		cfiscsi_ioctl_logout(ci);
2246 		break;
2247 	case CTL_ISCSI_TERMINATE:
2248 		cfiscsi_ioctl_terminate(ci);
2249 		break;
2250 	case CTL_ISCSI_LIMITS:
2251 		cfiscsi_ioctl_limits(ci);
2252 		break;
2253 #ifdef ICL_KERNEL_PROXY
2254 	case CTL_ISCSI_LISTEN:
2255 		cfiscsi_ioctl_listen(ci);
2256 		break;
2257 	case CTL_ISCSI_ACCEPT:
2258 		cfiscsi_ioctl_accept(ci);
2259 		break;
2260 	case CTL_ISCSI_SEND:
2261 		cfiscsi_ioctl_send(ci);
2262 		break;
2263 	case CTL_ISCSI_RECEIVE:
2264 		cfiscsi_ioctl_receive(ci);
2265 		break;
2266 #else
2267 	case CTL_ISCSI_LISTEN:
2268 	case CTL_ISCSI_ACCEPT:
2269 	case CTL_ISCSI_SEND:
2270 	case CTL_ISCSI_RECEIVE:
2271 		ci->status = CTL_ISCSI_ERROR;
2272 		snprintf(ci->error_str, sizeof(ci->error_str),
2273 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2274 		    __func__);
2275 		break;
2276 #endif /* !ICL_KERNEL_PROXY */
2277 	default:
2278 		ci->status = CTL_ISCSI_ERROR;
2279 		snprintf(ci->error_str, sizeof(ci->error_str),
2280 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2281 		break;
2282 	}
2283 
2284 	return (0);
2285 }
2286 
2287 static void
2288 cfiscsi_target_hold(struct cfiscsi_target *ct)
2289 {
2290 
2291 	refcount_acquire(&ct->ct_refcount);
2292 }
2293 
2294 static void
2295 cfiscsi_target_release(struct cfiscsi_target *ct)
2296 {
2297 	struct cfiscsi_softc *softc;
2298 
2299 	softc = ct->ct_softc;
2300 	mtx_lock(&softc->lock);
2301 	if (refcount_release(&ct->ct_refcount)) {
2302 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2303 		mtx_unlock(&softc->lock);
2304 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2305 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2306 			if (ctl_port_deregister(&ct->ct_port) != 0)
2307 				printf("%s: ctl_port_deregister() failed\n",
2308 				    __func__);
2309 		}
2310 		free(ct, M_CFISCSI);
2311 
2312 		return;
2313 	}
2314 	mtx_unlock(&softc->lock);
2315 }
2316 
2317 static struct cfiscsi_target *
2318 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2319 {
2320 	struct cfiscsi_target *ct;
2321 
2322 	mtx_lock(&softc->lock);
2323 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2324 		if (ct->ct_tag != tag ||
2325 		    strcmp(name, ct->ct_name) != 0 ||
2326 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2327 			continue;
2328 		cfiscsi_target_hold(ct);
2329 		mtx_unlock(&softc->lock);
2330 		return (ct);
2331 	}
2332 	mtx_unlock(&softc->lock);
2333 
2334 	return (NULL);
2335 }
2336 
2337 static struct cfiscsi_target *
2338 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2339     const char *alias, uint16_t tag)
2340 {
2341 	struct cfiscsi_target *ct, *newct;
2342 
2343 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2344 		return (NULL);
2345 
2346 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2347 
2348 	mtx_lock(&softc->lock);
2349 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2350 		if (ct->ct_tag != tag ||
2351 		    strcmp(name, ct->ct_name) != 0 ||
2352 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2353 			continue;
2354 		cfiscsi_target_hold(ct);
2355 		mtx_unlock(&softc->lock);
2356 		free(newct, M_CFISCSI);
2357 		return (ct);
2358 	}
2359 
2360 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2361 	if (alias != NULL)
2362 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2363 	newct->ct_tag = tag;
2364 	refcount_init(&newct->ct_refcount, 1);
2365 	newct->ct_softc = softc;
2366 	if (TAILQ_EMPTY(&softc->targets))
2367 		softc->last_target_id = 0;
2368 	newct->ct_target_id = ++softc->last_target_id;
2369 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2370 	mtx_unlock(&softc->lock);
2371 
2372 	return (newct);
2373 }
2374 
2375 static int
2376 cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
2377 {
2378 
2379 	return (0);
2380 }
2381 
2382 static int
2383 cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
2384 {
2385 
2386 	return (0);
2387 }
2388 
2389 static void
2390 cfiscsi_datamove_in(union ctl_io *io)
2391 {
2392 	struct cfiscsi_session *cs;
2393 	struct icl_pdu *request, *response;
2394 	const struct iscsi_bhs_scsi_command *bhssc;
2395 	struct iscsi_bhs_data_in *bhsdi;
2396 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2397 	size_t len, expected_len, sg_len, buffer_offset;
2398 	const char *sg_addr;
2399 	int ctl_sg_count, error, i;
2400 
2401 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2402 	cs = PDU_SESSION(request);
2403 
2404 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2405 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2406 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2407 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2408 
2409 	if (io->scsiio.kern_sg_entries > 0) {
2410 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2411 		ctl_sg_count = io->scsiio.kern_sg_entries;
2412 	} else {
2413 		ctl_sglist = &ctl_sg_entry;
2414 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2415 		ctl_sglist->len = io->scsiio.kern_data_len;
2416 		ctl_sg_count = 1;
2417 	}
2418 
2419 	/*
2420 	 * This is the total amount of data to be transferred within the current
2421 	 * SCSI command.  We need to record it so that we can properly report
2422 	 * underflow/underflow.
2423 	 */
2424 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2425 
2426 	/*
2427 	 * This is the offset within the current SCSI command; for the first
2428 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2429 	 * it will be the sum of lengths of previous ones.
2430 	 */
2431 	buffer_offset = io->scsiio.kern_rel_offset;
2432 
2433 	/*
2434 	 * This is the transfer length expected by the initiator.  In theory,
2435 	 * it could be different from the correct amount of data from the SCSI
2436 	 * point of view, even if that doesn't make any sense.
2437 	 */
2438 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2439 #if 0
2440 	if (expected_len != io->scsiio.kern_total_len) {
2441 		CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2442 		    "actual length %zd", expected_len,
2443 		    (size_t)io->scsiio.kern_total_len);
2444 	}
2445 #endif
2446 
2447 	if (buffer_offset >= expected_len) {
2448 #if 0
2449 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2450 		    "already sent the expected len", buffer_offset);
2451 #endif
2452 		io->scsiio.be_move_done(io);
2453 		return;
2454 	}
2455 
2456 	i = 0;
2457 	sg_addr = NULL;
2458 	sg_len = 0;
2459 	response = NULL;
2460 	bhsdi = NULL;
2461 	for (;;) {
2462 		if (response == NULL) {
2463 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2464 			if (response == NULL) {
2465 				CFISCSI_SESSION_WARN(cs, "failed to "
2466 				    "allocate memory; dropping connection");
2467 				ctl_set_busy(&io->scsiio);
2468 				io->scsiio.be_move_done(io);
2469 				cfiscsi_session_terminate(cs);
2470 				return;
2471 			}
2472 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2473 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2474 			bhsdi->bhsdi_initiator_task_tag =
2475 			    bhssc->bhssc_initiator_task_tag;
2476 			bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2477 			PDU_EXPDATASN(request)++;
2478 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2479 		}
2480 
2481 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2482 		if (sg_len == 0) {
2483 			sg_addr = ctl_sglist[i].addr;
2484 			sg_len = ctl_sglist[i].len;
2485 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2486 		}
2487 
2488 		len = sg_len;
2489 
2490 		/*
2491 		 * Truncate to maximum data segment length.
2492 		 */
2493 		KASSERT(response->ip_data_len < cs->cs_max_data_segment_length,
2494 		    ("ip_data_len %zd >= max_data_segment_length %zd",
2495 		    response->ip_data_len, cs->cs_max_data_segment_length));
2496 		if (response->ip_data_len + len >
2497 		    cs->cs_max_data_segment_length) {
2498 			len = cs->cs_max_data_segment_length -
2499 			    response->ip_data_len;
2500 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2501 			    len, sg_len));
2502 		}
2503 
2504 		/*
2505 		 * Truncate to expected data transfer length.
2506 		 */
2507 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2508 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2509 		    buffer_offset, response->ip_data_len, expected_len));
2510 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2511 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2512 			    "to expected data transfer length %zd",
2513 			    buffer_offset + response->ip_data_len + len, expected_len);
2514 			len = expected_len - (buffer_offset + response->ip_data_len);
2515 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2516 			    len, sg_len));
2517 		}
2518 
2519 		error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2520 		if (error != 0) {
2521 			CFISCSI_SESSION_WARN(cs, "failed to "
2522 			    "allocate memory; dropping connection");
2523 			icl_pdu_free(response);
2524 			ctl_set_busy(&io->scsiio);
2525 			io->scsiio.be_move_done(io);
2526 			cfiscsi_session_terminate(cs);
2527 			return;
2528 		}
2529 		sg_addr += len;
2530 		sg_len -= len;
2531 
2532 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2533 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2534 		    buffer_offset, response->ip_data_len, expected_len));
2535 		if (buffer_offset + response->ip_data_len == expected_len) {
2536 			/*
2537 			 * Already have the amount of data the initiator wanted.
2538 			 */
2539 			break;
2540 		}
2541 
2542 		if (sg_len == 0) {
2543 			/*
2544 			 * End of scatter-gather segment;
2545 			 * proceed to the next one...
2546 			 */
2547 			if (i == ctl_sg_count - 1) {
2548 				/*
2549 				 * ... unless this was the last one.
2550 				 */
2551 				break;
2552 			}
2553 			i++;
2554 		}
2555 
2556 		if (response->ip_data_len == cs->cs_max_data_segment_length) {
2557 			/*
2558 			 * Can't stuff more data into the current PDU;
2559 			 * queue it.  Note that's not enough to check
2560 			 * for kern_data_resid == 0 instead; there
2561 			 * may be several Data-In PDUs for the final
2562 			 * call to cfiscsi_datamove(), and we want
2563 			 * to set the F flag only on the last of them.
2564 			 */
2565 			buffer_offset += response->ip_data_len;
2566 			if (buffer_offset == io->scsiio.kern_total_len ||
2567 			    buffer_offset == expected_len) {
2568 				buffer_offset -= response->ip_data_len;
2569 				break;
2570 			}
2571 			cfiscsi_pdu_queue(response);
2572 			response = NULL;
2573 			bhsdi = NULL;
2574 		}
2575 	}
2576 	if (response != NULL) {
2577 		buffer_offset += response->ip_data_len;
2578 		if (buffer_offset == io->scsiio.kern_total_len ||
2579 		    buffer_offset == expected_len) {
2580 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2581 			if (io->io_hdr.status == CTL_SUCCESS) {
2582 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2583 				if (PDU_TOTAL_TRANSFER_LEN(request) <
2584 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2585 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2586 					bhsdi->bhsdi_residual_count =
2587 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2588 					    PDU_TOTAL_TRANSFER_LEN(request));
2589 				} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2590 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2591 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2592 					bhsdi->bhsdi_residual_count =
2593 					    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2594 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2595 				}
2596 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2597 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2598 			}
2599 		}
2600 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2601 		cfiscsi_pdu_queue(response);
2602 	}
2603 
2604 	io->scsiio.be_move_done(io);
2605 }
2606 
2607 static void
2608 cfiscsi_datamove_out(union ctl_io *io)
2609 {
2610 	struct cfiscsi_session *cs;
2611 	struct icl_pdu *request, *response;
2612 	const struct iscsi_bhs_scsi_command *bhssc;
2613 	struct iscsi_bhs_r2t *bhsr2t;
2614 	struct cfiscsi_data_wait *cdw;
2615 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2616 	uint32_t expected_len, r2t_off, r2t_len;
2617 	uint32_t target_transfer_tag;
2618 	bool done;
2619 
2620 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2621 	cs = PDU_SESSION(request);
2622 
2623 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2624 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2625 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2626 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2627 
2628 	/*
2629 	 * We need to record it so that we can properly report
2630 	 * underflow/underflow.
2631 	 */
2632 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2633 
2634 	/*
2635 	 * Report write underflow as error since CTL and backends don't
2636 	 * really support it, and SCSI does not tell how to do it right.
2637 	 */
2638 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2639 	if (io->scsiio.kern_rel_offset + io->scsiio.kern_data_len >
2640 	    expected_len) {
2641 		io->scsiio.io_hdr.port_status = 43;
2642 		io->scsiio.be_move_done(io);
2643 		return;
2644 	}
2645 
2646 	target_transfer_tag =
2647 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2648 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2649 	    &target_transfer_tag);
2650 	if (cdw == NULL) {
2651 		CFISCSI_SESSION_WARN(cs, "failed to "
2652 		    "allocate memory; dropping connection");
2653 		ctl_set_busy(&io->scsiio);
2654 		io->scsiio.be_move_done(io);
2655 		cfiscsi_session_terminate(cs);
2656 		return;
2657 	}
2658 #if 0
2659 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2660 	    "task tag 0x%x, target transfer tag 0x%x",
2661 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2662 #endif
2663 
2664 	cdw->cdw_ctl_io = io;
2665 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2666 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2667 	cdw->cdw_r2t_end = io->scsiio.kern_data_len;
2668 	cdw->cdw_datasn = 0;
2669 
2670 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2671 	if (io->scsiio.kern_sg_entries > 0) {
2672 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2673 	} else {
2674 		ctl_sglist = &ctl_sg_entry;
2675 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2676 		ctl_sglist->len = io->scsiio.kern_data_len;
2677 	}
2678 	cdw->cdw_sg_index = 0;
2679 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2680 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2681 	r2t_off = io->scsiio.ext_data_filled;
2682 	while (r2t_off > 0) {
2683 		if (r2t_off >= cdw->cdw_sg_len) {
2684 			r2t_off -= cdw->cdw_sg_len;
2685 			cdw->cdw_sg_index++;
2686 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2687 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2688 			continue;
2689 		}
2690 		cdw->cdw_sg_addr += r2t_off;
2691 		cdw->cdw_sg_len -= r2t_off;
2692 		r2t_off = 0;
2693 	}
2694 
2695 	if (cs->cs_immediate_data &&
2696 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2697 	    icl_pdu_data_segment_length(request)) {
2698 		done = cfiscsi_handle_data_segment(request, cdw);
2699 		if (done) {
2700 			cfiscsi_data_wait_free(cs, cdw);
2701 			io->scsiio.be_move_done(io);
2702 			return;
2703 		}
2704 	}
2705 
2706 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2707 	r2t_len = MIN(io->scsiio.kern_data_len - io->scsiio.ext_data_filled,
2708 	    cs->cs_max_burst_length);
2709 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2710 
2711 	CFISCSI_SESSION_LOCK(cs);
2712 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2713 	CFISCSI_SESSION_UNLOCK(cs);
2714 
2715 	/*
2716 	 * XXX: We should limit the number of outstanding R2T PDUs
2717 	 * 	per task to MaxOutstandingR2T.
2718 	 */
2719 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2720 	if (response == NULL) {
2721 		CFISCSI_SESSION_WARN(cs, "failed to "
2722 		    "allocate memory; dropping connection");
2723 		ctl_set_busy(&io->scsiio);
2724 		io->scsiio.be_move_done(io);
2725 		cfiscsi_session_terminate(cs);
2726 		return;
2727 	}
2728 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2729 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2730 	bhsr2t->bhsr2t_flags = 0x80;
2731 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2732 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2733 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2734 	/*
2735 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2736 	 *	be running concurrently on several CPUs for a given
2737 	 *	command.
2738 	 */
2739 	bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2740 	PDU_R2TSN(request)++;
2741 	/*
2742 	 * This is the offset within the current SCSI command;
2743 	 * i.e. for the first call of datamove(), it will be 0,
2744 	 * and for subsequent ones it will be the sum of lengths
2745 	 * of previous ones.
2746 	 *
2747 	 * The ext_data_filled is to account for unsolicited
2748 	 * (immediate) data that might have already arrived.
2749 	 */
2750 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2751 	/*
2752 	 * This is the total length (sum of S/G lengths) this call
2753 	 * to cfiscsi_datamove() is supposed to handle, limited by
2754 	 * MaxBurstLength.
2755 	 */
2756 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2757 	cfiscsi_pdu_queue(response);
2758 }
2759 
2760 static void
2761 cfiscsi_datamove(union ctl_io *io)
2762 {
2763 
2764 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2765 		cfiscsi_datamove_in(io);
2766 	else {
2767 		/* We hadn't received anything during this datamove yet. */
2768 		io->scsiio.ext_data_filled = 0;
2769 		cfiscsi_datamove_out(io);
2770 	}
2771 }
2772 
2773 static void
2774 cfiscsi_scsi_command_done(union ctl_io *io)
2775 {
2776 	struct icl_pdu *request, *response;
2777 	struct iscsi_bhs_scsi_command *bhssc;
2778 	struct iscsi_bhs_scsi_response *bhssr;
2779 #ifdef DIAGNOSTIC
2780 	struct cfiscsi_data_wait *cdw;
2781 #endif
2782 	struct cfiscsi_session *cs;
2783 	uint16_t sense_length;
2784 
2785 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2786 	cs = PDU_SESSION(request);
2787 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2788 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2789 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2790 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2791 
2792 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2793 	//    bhssc->bhssc_initiator_task_tag);
2794 
2795 #ifdef DIAGNOSTIC
2796 	CFISCSI_SESSION_LOCK(cs);
2797 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2798 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2799 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2800 	CFISCSI_SESSION_UNLOCK(cs);
2801 #endif
2802 
2803 	/*
2804 	 * Do not return status for aborted commands.
2805 	 * There are exceptions, but none supported by CTL yet.
2806 	 */
2807 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2808 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2809 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2810 		ctl_free_io(io);
2811 		icl_pdu_free(request);
2812 		return;
2813 	}
2814 
2815 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2816 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2817 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2818 	bhssr->bhssr_flags = 0x80;
2819 	/*
2820 	 * XXX: We don't deal with bidirectional under/overflows;
2821 	 *	does anything actually support those?
2822 	 */
2823 	if (PDU_TOTAL_TRANSFER_LEN(request) <
2824 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2825 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2826 		bhssr->bhssr_residual_count =
2827 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2828 		    PDU_TOTAL_TRANSFER_LEN(request));
2829 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2830 		//    ntohl(bhssr->bhssr_residual_count));
2831 	} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2832 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2833 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2834 		bhssr->bhssr_residual_count =
2835 		    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2836 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2837 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2838 		//    ntohl(bhssr->bhssr_residual_count));
2839 	}
2840 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2841 	bhssr->bhssr_status = io->scsiio.scsi_status;
2842 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2843 	bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2844 
2845 	if (io->scsiio.sense_len > 0) {
2846 #if 0
2847 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2848 		    io->scsiio.sense_len);
2849 #endif
2850 		sense_length = htons(io->scsiio.sense_len);
2851 		icl_pdu_append_data(response,
2852 		    &sense_length, sizeof(sense_length), M_WAITOK);
2853 		icl_pdu_append_data(response,
2854 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2855 	}
2856 
2857 	ctl_free_io(io);
2858 	icl_pdu_free(request);
2859 	cfiscsi_pdu_queue(response);
2860 }
2861 
2862 static void
2863 cfiscsi_task_management_done(union ctl_io *io)
2864 {
2865 	struct icl_pdu *request, *response;
2866 	struct iscsi_bhs_task_management_request *bhstmr;
2867 	struct iscsi_bhs_task_management_response *bhstmr2;
2868 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2869 	struct cfiscsi_session *cs;
2870 
2871 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2872 	cs = PDU_SESSION(request);
2873 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2874 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2875 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2876 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2877 
2878 #if 0
2879 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2880 	    bhstmr->bhstmr_initiator_task_tag,
2881 	    bhstmr->bhstmr_referenced_task_tag);
2882 #endif
2883 
2884 	if ((bhstmr->bhstmr_function & ~0x80) ==
2885 	    BHSTMR_FUNCTION_ABORT_TASK) {
2886 		/*
2887 		 * Make sure we no longer wait for Data-Out for this command.
2888 		 */
2889 		CFISCSI_SESSION_LOCK(cs);
2890 		TAILQ_FOREACH_SAFE(cdw,
2891 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2892 			if (bhstmr->bhstmr_referenced_task_tag !=
2893 			    cdw->cdw_initiator_task_tag)
2894 				continue;
2895 
2896 #if 0
2897 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2898 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2899 #endif
2900 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2901 			    cdw, cdw_next);
2902 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2903 			cfiscsi_data_wait_free(cs, cdw);
2904 		}
2905 		CFISCSI_SESSION_UNLOCK(cs);
2906 	}
2907 
2908 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2909 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2910 	    response->ip_bhs;
2911 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2912 	bhstmr2->bhstmr_flags = 0x80;
2913 	if (io->io_hdr.status == CTL_SUCCESS) {
2914 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2915 	} else {
2916 		/*
2917 		 * XXX: How to figure out what exactly went wrong?  iSCSI spec
2918 		 * 	expects us to provide detailed error, e.g. "Task does
2919 		 * 	not exist" or "LUN does not exist".
2920 		 */
2921 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
2922 		bhstmr2->bhstmr_response =
2923 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2924 	}
2925 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2926 
2927 	ctl_free_io(io);
2928 	icl_pdu_free(request);
2929 	cfiscsi_pdu_queue(response);
2930 }
2931 
2932 static void
2933 cfiscsi_done(union ctl_io *io)
2934 {
2935 	struct icl_pdu *request;
2936 	struct cfiscsi_session *cs;
2937 
2938 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2939 		("invalid CTL status %#x", io->io_hdr.status));
2940 
2941 	if (io->io_hdr.io_type == CTL_IO_TASK &&
2942 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
2943 		/*
2944 		 * Implicit task termination has just completed; nothing to do.
2945 		 */
2946 		cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2947 		cs->cs_tasks_aborted = true;
2948 		refcount_release(&cs->cs_outstanding_ctl_pdus);
2949 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
2950 		ctl_free_io(io);
2951 		return;
2952 	}
2953 
2954 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2955 	cs = PDU_SESSION(request);
2956 	refcount_release(&cs->cs_outstanding_ctl_pdus);
2957 
2958 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
2959 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
2960 		cfiscsi_scsi_command_done(io);
2961 		break;
2962 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
2963 		cfiscsi_task_management_done(io);
2964 		break;
2965 	default:
2966 		panic("cfiscsi_done called with wrong opcode 0x%x",
2967 		    request->ip_bhs->bhs_opcode);
2968 	}
2969 }
2970