xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision eac7052fdebb90caf2f653e06187bdbca837b9c7)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 The FreeBSD Foundation
5  * All rights reserved.
6  *
7  * This software was developed by Edward Tomasz Napierala under sponsorship
8  * from the FreeBSD Foundation.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  *
31  * $FreeBSD$
32  */
33 
34 /*
35  * CTL frontend for the iSCSI protocol.
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include <sys/param.h>
42 #include <sys/capsicum.h>
43 #include <sys/condvar.h>
44 #include <sys/endian.h>
45 #include <sys/file.h>
46 #include <sys/kernel.h>
47 #include <sys/kthread.h>
48 #include <sys/lock.h>
49 #include <sys/malloc.h>
50 #include <sys/module.h>
51 #include <sys/mutex.h>
52 #include <sys/queue.h>
53 #include <sys/sbuf.h>
54 #include <sys/socket.h>
55 #include <sys/sysctl.h>
56 #include <sys/systm.h>
57 #include <sys/uio.h>
58 #include <sys/unistd.h>
59 #include <sys/nv.h>
60 #include <sys/dnv.h>
61 #include <vm/uma.h>
62 
63 #include <cam/scsi/scsi_all.h>
64 #include <cam/scsi/scsi_da.h>
65 #include <cam/ctl/ctl_io.h>
66 #include <cam/ctl/ctl.h>
67 #include <cam/ctl/ctl_backend.h>
68 #include <cam/ctl/ctl_error.h>
69 #include <cam/ctl/ctl_frontend.h>
70 #include <cam/ctl/ctl_debug.h>
71 #include <cam/ctl/ctl_ha.h>
72 #include <cam/ctl/ctl_ioctl.h>
73 #include <cam/ctl/ctl_private.h>
74 
75 #include <dev/iscsi/icl.h>
76 #include <dev/iscsi/icl_wrappers.h>
77 #include <dev/iscsi/iscsi_proto.h>
78 #include <cam/ctl/ctl_frontend_iscsi.h>
79 
80 #ifdef ICL_KERNEL_PROXY
81 #include <sys/socketvar.h>
82 #endif
83 
84 #ifdef ICL_KERNEL_PROXY
85 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
86 #endif
87 
88 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
89 static uma_zone_t cfiscsi_data_wait_zone;
90 
91 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
92     "CAM Target Layer iSCSI Frontend");
93 static int debug = 1;
94 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
95     &debug, 1, "Enable debug messages");
96 static int ping_timeout = 5;
97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
98     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
99 static int login_timeout = 60;
100 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
101     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
102 static int maxtags = 256;
103 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
104     &maxtags, 0, "Max number of requests queued by initiator");
105 
106 #define	CFISCSI_DEBUG(X, ...)						\
107 	do {								\
108 		if (debug > 1) {					\
109 			printf("%s: " X "\n",				\
110 			    __func__, ## __VA_ARGS__);			\
111 		}							\
112 	} while (0)
113 
114 #define	CFISCSI_WARN(X, ...)						\
115 	do {								\
116 		if (debug > 0) {					\
117 			printf("WARNING: %s: " X "\n",			\
118 			    __func__, ## __VA_ARGS__);			\
119 		}							\
120 	} while (0)
121 
122 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
123 	do {								\
124 		if (debug > 1) {					\
125 			printf("%s: %s (%s): " X "\n",			\
126 			    __func__, S->cs_initiator_addr,		\
127 			    S->cs_initiator_name, ## __VA_ARGS__);	\
128 		}							\
129 	} while (0)
130 
131 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
132 	do  {								\
133 		if (debug > 0) {					\
134 			printf("WARNING: %s (%s): " X "\n",		\
135 			    S->cs_initiator_addr,			\
136 			    S->cs_initiator_name, ## __VA_ARGS__);	\
137 		}							\
138 	} while (0)
139 
140 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
141 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
142 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
143 
144 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
145 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
146 
147 struct cfiscsi_priv {
148 	void		*request;
149 	uint32_t	 expdatasn;
150 	uint32_t	 r2tsn;
151 };
152 #define	PRIV(io)	\
153     ((struct cfiscsi_priv *)&(io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND])
154 #define	PRIV_REQUEST(io)		PRIV(io)->request
155 #define	PRIV_EXPDATASN(io)		PRIV(io)->expdatasn
156 #define	PRIV_R2TSN(io)			PRIV(io)->r2tsn
157 
158 static int	cfiscsi_init(void);
159 static int	cfiscsi_shutdown(void);
160 static void	cfiscsi_online(void *arg);
161 static void	cfiscsi_offline(void *arg);
162 static int	cfiscsi_info(void *arg, struct sbuf *sb);
163 static int	cfiscsi_ioctl(struct cdev *dev,
164 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
165 static void	cfiscsi_datamove(union ctl_io *io);
166 static void	cfiscsi_datamove_in(union ctl_io *io);
167 static void	cfiscsi_datamove_out(union ctl_io *io);
168 static void	cfiscsi_done(union ctl_io *io);
169 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
170 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
171 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
172 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
173 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
174 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
175 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
176 static struct cfiscsi_data_wait	*cfiscsi_data_wait_new(
177 		    struct cfiscsi_session *cs, union ctl_io *io,
178 		    uint32_t initiator_task_tag,
179 		    uint32_t *target_transfer_tagp);
180 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
181 		    struct cfiscsi_data_wait *cdw);
182 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
183 		    *softc, const char *name, uint16_t tag);
184 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
185     struct cfiscsi_softc *softc, const char *name, const char *alias,
186     uint16_t tag);
187 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
188 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
189 
190 static struct cfiscsi_softc cfiscsi_softc;
191 
192 static struct ctl_frontend cfiscsi_frontend =
193 {
194 	.name = "iscsi",
195 	.init = cfiscsi_init,
196 	.ioctl = cfiscsi_ioctl,
197 	.shutdown = cfiscsi_shutdown,
198 };
199 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
200 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
201 
202 static struct icl_pdu *
203 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
204 {
205 
206 	return (icl_pdu_new(request->ip_conn, flags));
207 }
208 
209 static bool
210 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
211 {
212 	const struct iscsi_bhs_scsi_command *bhssc;
213 	struct cfiscsi_session *cs;
214 	uint32_t cmdsn, curcmdsn;
215 
216 	cs = PDU_SESSION(request);
217 
218 	/*
219 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
220 	 * The purpose of the timeout is to reset the connection when it stalls;
221 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
222 	 * in some queue.
223 	 */
224 	cs->cs_timeout = 0;
225 
226 	/*
227 	 * Immediate commands carry cmdsn, but it is neither incremented nor
228 	 * verified.
229 	 */
230 	if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE)
231 		return (false);
232 
233 	/*
234 	 * Data-Out PDUs don't contain CmdSN.
235 	 */
236 	if (request->ip_bhs->bhs_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
237 		return (false);
238 
239 	/*
240 	 * We're only using fields common for all the request
241 	 * (initiator -> target) PDUs.
242 	 */
243 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
244 	curcmdsn = cmdsn = ntohl(bhssc->bhssc_cmdsn);
245 
246 	/*
247 	 * Increment session cmdsn and exit if we received the expected value.
248 	 */
249 	do {
250 		if (atomic_fcmpset_32(&cs->cs_cmdsn, &curcmdsn, cmdsn + 1))
251 			return (false);
252 	} while (curcmdsn == cmdsn);
253 
254 	/*
255 	 * The target MUST silently ignore any non-immediate command outside
256 	 * of this range.
257 	 */
258 	if (ISCSI_SNLT(cmdsn, curcmdsn) ||
259 	    ISCSI_SNGT(cmdsn, curcmdsn - 1 + maxtags)) {
260 		CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
261 		    "while expected %u", cmdsn, curcmdsn);
262 		return (true);
263 	}
264 
265 	/*
266 	 * We don't support multiple connections now, so any discontinuity in
267 	 * CmdSN means lost PDUs.  Since we don't support PDU retransmission --
268 	 * terminate the connection.
269 	 */
270 	CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
271 	    "while expected %u; dropping connection",
272 	    cmdsn, curcmdsn);
273 	cfiscsi_session_terminate(cs);
274 	return (true);
275 }
276 
277 static void
278 cfiscsi_pdu_handle(struct icl_pdu *request)
279 {
280 	struct cfiscsi_session *cs;
281 	bool ignore;
282 
283 	cs = PDU_SESSION(request);
284 
285 	ignore = cfiscsi_pdu_update_cmdsn(request);
286 	if (ignore) {
287 		icl_pdu_free(request);
288 		return;
289 	}
290 
291 	/*
292 	 * Handle the PDU; this includes e.g. receiving the remaining
293 	 * part of PDU and submitting the SCSI command to CTL
294 	 * or queueing a reply.  The handling routine is responsible
295 	 * for freeing the PDU when it's no longer needed.
296 	 */
297 	switch (request->ip_bhs->bhs_opcode &
298 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
299 	case ISCSI_BHS_OPCODE_NOP_OUT:
300 		cfiscsi_pdu_handle_nop_out(request);
301 		break;
302 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
303 		cfiscsi_pdu_handle_scsi_command(request);
304 		break;
305 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
306 		cfiscsi_pdu_handle_task_request(request);
307 		break;
308 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
309 		cfiscsi_pdu_handle_data_out(request);
310 		break;
311 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
312 		cfiscsi_pdu_handle_logout_request(request);
313 		break;
314 	default:
315 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
316 		    "opcode 0x%x; dropping connection",
317 		    request->ip_bhs->bhs_opcode);
318 		icl_pdu_free(request);
319 		cfiscsi_session_terminate(cs);
320 	}
321 
322 }
323 
324 static void
325 cfiscsi_receive_callback(struct icl_pdu *request)
326 {
327 #ifdef ICL_KERNEL_PROXY
328 	struct cfiscsi_session *cs;
329 
330 	cs = PDU_SESSION(request);
331 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
332 		if (cs->cs_login_pdu == NULL)
333 			cs->cs_login_pdu = request;
334 		else
335 			icl_pdu_free(request);
336 		cv_signal(&cs->cs_login_cv);
337 		return;
338 	}
339 #endif
340 
341 	cfiscsi_pdu_handle(request);
342 }
343 
344 static void
345 cfiscsi_error_callback(struct icl_conn *ic)
346 {
347 	struct cfiscsi_session *cs;
348 
349 	cs = CONN_SESSION(ic);
350 
351 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
352 	cfiscsi_session_terminate(cs);
353 }
354 
355 static int
356 cfiscsi_pdu_prepare(struct icl_pdu *response)
357 {
358 	struct cfiscsi_session *cs;
359 	struct iscsi_bhs_scsi_response *bhssr;
360 	bool advance_statsn = true;
361 	uint32_t cmdsn;
362 
363 	cs = PDU_SESSION(response);
364 
365 	CFISCSI_SESSION_LOCK_ASSERT(cs);
366 
367 	/*
368 	 * We're only using fields common for all the response
369 	 * (target -> initiator) PDUs.
370 	 */
371 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
372 
373 	/*
374 	 * 10.8.3: "The StatSN for this connection is not advanced
375 	 * after this PDU is sent."
376 	 */
377 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
378 		advance_statsn = false;
379 
380 	/*
381 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
382 	 * StatSN for the connection is not advanced after this PDU is sent."
383 	 */
384 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
385 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
386 		advance_statsn = false;
387 
388 	/*
389 	 * See the comment below - StatSN is not meaningful and must
390 	 * not be advanced.
391 	 */
392 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
393 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
394 		advance_statsn = false;
395 
396 	/*
397 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
398 	 * only have meaningful content if the S bit is set to 1."
399 	 */
400 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
401 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
402 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
403 	cmdsn = cs->cs_cmdsn;
404 	bhssr->bhssr_expcmdsn = htonl(cmdsn);
405 	bhssr->bhssr_maxcmdsn = htonl(cmdsn - 1 +
406 	    imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
407 
408 	if (advance_statsn)
409 		cs->cs_statsn++;
410 
411 	return (0);
412 }
413 
414 static void
415 cfiscsi_pdu_queue(struct icl_pdu *response)
416 {
417 	struct cfiscsi_session *cs;
418 
419 	cs = PDU_SESSION(response);
420 
421 	CFISCSI_SESSION_LOCK(cs);
422 	cfiscsi_pdu_prepare(response);
423 	icl_pdu_queue(response);
424 	CFISCSI_SESSION_UNLOCK(cs);
425 }
426 
427  static void
428 cfiscsi_pdu_queue_cb(struct icl_pdu *response, icl_pdu_cb cb)
429 {
430 	struct cfiscsi_session *cs = PDU_SESSION(response);
431 
432 	CFISCSI_SESSION_LOCK(cs);
433 	cfiscsi_pdu_prepare(response);
434 	icl_pdu_queue_cb(response, cb);
435 	CFISCSI_SESSION_UNLOCK(cs);
436 }
437 
438 static void
439 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
440 {
441 	struct cfiscsi_session *cs;
442 	struct iscsi_bhs_nop_out *bhsno;
443 	struct iscsi_bhs_nop_in *bhsni;
444 	struct icl_pdu *response;
445 	void *data = NULL;
446 	size_t datasize;
447 	int error;
448 
449 	cs = PDU_SESSION(request);
450 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
451 
452 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
453 		/*
454 		 * Nothing to do, iscsi_pdu_update_statsn() already
455 		 * zeroed the timeout.
456 		 */
457 		icl_pdu_free(request);
458 		return;
459 	}
460 
461 	datasize = icl_pdu_data_segment_length(request);
462 	if (datasize > 0) {
463 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
464 		if (data == NULL) {
465 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
466 			    "dropping connection");
467 			icl_pdu_free(request);
468 			cfiscsi_session_terminate(cs);
469 			return;
470 		}
471 		icl_pdu_get_data(request, 0, data, datasize);
472 	}
473 
474 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
475 	if (response == NULL) {
476 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
477 		    "droppping connection");
478 		free(data, M_CFISCSI);
479 		icl_pdu_free(request);
480 		cfiscsi_session_terminate(cs);
481 		return;
482 	}
483 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
484 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
485 	bhsni->bhsni_flags = 0x80;
486 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
487 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
488 	if (datasize > 0) {
489 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
490 		if (error != 0) {
491 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
492 			    "dropping connection");
493 			free(data, M_CFISCSI);
494 			icl_pdu_free(request);
495 			icl_pdu_free(response);
496 			cfiscsi_session_terminate(cs);
497 			return;
498 		}
499 		free(data, M_CFISCSI);
500 	}
501 
502 	icl_pdu_free(request);
503 	cfiscsi_pdu_queue(response);
504 }
505 
506 static void
507 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
508 {
509 	struct iscsi_bhs_scsi_command *bhssc;
510 	struct cfiscsi_session *cs;
511 	union ctl_io *io;
512 	int error;
513 
514 	cs = PDU_SESSION(request);
515 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
516 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
517 	//    bhssc->bhssc_initiator_task_tag);
518 
519 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
520 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
521 		    "ImmediateData=No; dropping connection");
522 		icl_pdu_free(request);
523 		cfiscsi_session_terminate(cs);
524 		return;
525 	}
526 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
527 	ctl_zero_io(io);
528 	PRIV_REQUEST(io) = request;
529 	io->io_hdr.io_type = CTL_IO_SCSI;
530 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
531 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
532 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
533 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
534 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
535 	case BHSSC_FLAGS_ATTR_UNTAGGED:
536 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
537 		break;
538 	case BHSSC_FLAGS_ATTR_SIMPLE:
539 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
540 		break;
541 	case BHSSC_FLAGS_ATTR_ORDERED:
542         	io->scsiio.tag_type = CTL_TAG_ORDERED;
543 		break;
544 	case BHSSC_FLAGS_ATTR_HOQ:
545         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
546 		break;
547 	case BHSSC_FLAGS_ATTR_ACA:
548 		io->scsiio.tag_type = CTL_TAG_ACA;
549 		break;
550 	default:
551 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
552 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
553 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
554 		break;
555 	}
556 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
557 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
558 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
559 	error = ctl_queue(io);
560 	if (error != CTL_RETVAL_COMPLETE) {
561 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
562 		    "dropping connection", error);
563 		ctl_free_io(io);
564 		refcount_release(&cs->cs_outstanding_ctl_pdus);
565 		icl_pdu_free(request);
566 		cfiscsi_session_terminate(cs);
567 	}
568 }
569 
570 static void
571 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
572 {
573 	struct iscsi_bhs_task_management_request *bhstmr;
574 	struct iscsi_bhs_task_management_response *bhstmr2;
575 	struct icl_pdu *response;
576 	struct cfiscsi_session *cs;
577 	union ctl_io *io;
578 	int error;
579 
580 	cs = PDU_SESSION(request);
581 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
582 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
583 	ctl_zero_io(io);
584 	PRIV_REQUEST(io) = request;
585 	io->io_hdr.io_type = CTL_IO_TASK;
586 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
587 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
588 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
589 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
590 
591 	switch (bhstmr->bhstmr_function & ~0x80) {
592 	case BHSTMR_FUNCTION_ABORT_TASK:
593 #if 0
594 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
595 #endif
596 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
597 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
598 		break;
599 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
600 #if 0
601 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
602 #endif
603 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
604 		break;
605 	case BHSTMR_FUNCTION_CLEAR_TASK_SET:
606 #if 0
607 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
608 #endif
609 		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
610 		break;
611 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
612 #if 0
613 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
614 #endif
615 		io->taskio.task_action = CTL_TASK_LUN_RESET;
616 		break;
617 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
618 #if 0
619 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
620 #endif
621 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
622 		break;
623 	case BHSTMR_FUNCTION_TARGET_COLD_RESET:
624 #if 0
625 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
626 #endif
627 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
628 		break;
629 	case BHSTMR_FUNCTION_QUERY_TASK:
630 #if 0
631 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
632 #endif
633 		io->taskio.task_action = CTL_TASK_QUERY_TASK;
634 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
635 		break;
636 	case BHSTMR_FUNCTION_QUERY_TASK_SET:
637 #if 0
638 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
639 #endif
640 		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
641 		break;
642 	case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
643 #if 0
644 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
645 #endif
646 		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
647 		break;
648 	case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
649 #if 0
650 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
651 #endif
652 		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
653 		break;
654 	default:
655 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
656 		    bhstmr->bhstmr_function & ~0x80);
657 		ctl_free_io(io);
658 
659 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
660 		if (response == NULL) {
661 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
662 			    "dropping connection");
663 			icl_pdu_free(request);
664 			cfiscsi_session_terminate(cs);
665 			return;
666 		}
667 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
668 		    response->ip_bhs;
669 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
670 		bhstmr2->bhstmr_flags = 0x80;
671 		bhstmr2->bhstmr_response =
672 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
673 		bhstmr2->bhstmr_initiator_task_tag =
674 		    bhstmr->bhstmr_initiator_task_tag;
675 		icl_pdu_free(request);
676 		cfiscsi_pdu_queue(response);
677 		return;
678 	}
679 
680 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
681 	error = ctl_queue(io);
682 	if (error != CTL_RETVAL_COMPLETE) {
683 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
684 		    "dropping connection", error);
685 		ctl_free_io(io);
686 		refcount_release(&cs->cs_outstanding_ctl_pdus);
687 		icl_pdu_free(request);
688 		cfiscsi_session_terminate(cs);
689 	}
690 }
691 
692 static bool
693 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
694 {
695 	struct iscsi_bhs_data_out *bhsdo;
696 	struct cfiscsi_session *cs;
697 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
698 	size_t copy_len, len, off, buffer_offset;
699 	int ctl_sg_count;
700 	union ctl_io *io;
701 
702 	cs = PDU_SESSION(request);
703 
704 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
705 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
706 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
707 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
708 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
709 
710 	/*
711 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
712 	 */
713 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
714 
715 	io = cdw->cdw_ctl_io;
716 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
717 	    ("CTL_FLAG_DATA_IN"));
718 
719 #if 0
720 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
721 	    request->ip_data_len, io->scsiio.kern_total_len);
722 #endif
723 
724 	if (io->scsiio.kern_sg_entries > 0) {
725 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
726 		ctl_sg_count = io->scsiio.kern_sg_entries;
727 	} else {
728 		ctl_sglist = &ctl_sg_entry;
729 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
730 		ctl_sglist->len = io->scsiio.kern_data_len;
731 		ctl_sg_count = 1;
732 	}
733 
734 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
735 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
736 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
737 	else
738 		buffer_offset = 0;
739 	len = icl_pdu_data_segment_length(request);
740 
741 	/*
742 	 * Make sure the offset, as sent by the initiator, matches the offset
743 	 * we're supposed to be at in the scatter-gather list.
744 	 */
745 	if (buffer_offset >
746 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
747 	    buffer_offset + len <=
748 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
749 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
750 		    "expected %zd; dropping connection", buffer_offset,
751 		    (size_t)io->scsiio.kern_rel_offset +
752 		    (size_t)io->scsiio.ext_data_filled);
753 		ctl_set_data_phase_error(&io->scsiio);
754 		cfiscsi_session_terminate(cs);
755 		return (true);
756 	}
757 
758 	/*
759 	 * This is the offset within the PDU data segment, as opposed
760 	 * to buffer_offset, which is the offset within the task (SCSI
761 	 * command).
762 	 */
763 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
764 	    buffer_offset;
765 
766 	/*
767 	 * Iterate over the scatter/gather segments, filling them with data
768 	 * from the PDU data segment.  Note that this can get called multiple
769 	 * times for one SCSI command; the cdw structure holds state for the
770 	 * scatter/gather list.
771 	 */
772 	for (;;) {
773 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
774 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
775 		if (cdw->cdw_sg_len == 0) {
776 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
777 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
778 		}
779 		KASSERT(off <= len, ("len > off"));
780 		copy_len = len - off;
781 		if (copy_len > cdw->cdw_sg_len)
782 			copy_len = cdw->cdw_sg_len;
783 
784 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
785 		cdw->cdw_sg_addr += copy_len;
786 		cdw->cdw_sg_len -= copy_len;
787 		off += copy_len;
788 		io->scsiio.ext_data_filled += copy_len;
789 		io->scsiio.kern_data_resid -= copy_len;
790 
791 		if (cdw->cdw_sg_len == 0) {
792 			/*
793 			 * End of current segment.
794 			 */
795 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
796 				/*
797 				 * Last segment in scatter/gather list.
798 				 */
799 				break;
800 			}
801 			cdw->cdw_sg_index++;
802 		}
803 
804 		if (off == len) {
805 			/*
806 			 * End of PDU payload.
807 			 */
808 			break;
809 		}
810 	}
811 
812 	if (len > off) {
813 		/*
814 		 * In case of unsolicited data, it's possible that the buffer
815 		 * provided by CTL is smaller than negotiated FirstBurstLength.
816 		 * Just ignore the superfluous data; will ask for them with R2T
817 		 * on next call to cfiscsi_datamove().
818 		 *
819 		 * This obviously can only happen with SCSI Command PDU.
820 		 */
821 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
822 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
823 			return (true);
824 
825 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
826 		    "expected %zd; dropping connection",
827 		    icl_pdu_data_segment_length(request), off);
828 		ctl_set_data_phase_error(&io->scsiio);
829 		cfiscsi_session_terminate(cs);
830 		return (true);
831 	}
832 
833 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
834 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
835 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
836 		    "the F flag; flags = 0x%x; dropping connection",
837 		    bhsdo->bhsdo_flags);
838 		ctl_set_data_phase_error(&io->scsiio);
839 		cfiscsi_session_terminate(cs);
840 		return (true);
841 	}
842 
843 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
844 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
845 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
846 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
847 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
848 			    "transmitted size was %zd bytes instead of %d; "
849 			    "dropping connection",
850 			    (size_t)io->scsiio.ext_data_filled,
851 			    cdw->cdw_r2t_end);
852 			ctl_set_data_phase_error(&io->scsiio);
853 			cfiscsi_session_terminate(cs);
854 			return (true);
855 		} else {
856 			/*
857 			 * For SCSI Command PDU, this just means we need to
858 			 * solicit more data by sending R2T.
859 			 */
860 			return (false);
861 		}
862 	}
863 
864 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
865 #if 0
866 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
867 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
868 #endif
869 
870 		return (true);
871 	}
872 
873 	return (false);
874 }
875 
876 static void
877 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
878 {
879 	struct iscsi_bhs_data_out *bhsdo;
880 	struct cfiscsi_session *cs;
881 	struct cfiscsi_data_wait *cdw = NULL;
882 	union ctl_io *io;
883 	bool done;
884 
885 	cs = PDU_SESSION(request);
886 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
887 
888 	CFISCSI_SESSION_LOCK(cs);
889 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
890 #if 0
891 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
892 		    "ttt 0x%x, itt 0x%x",
893 		    bhsdo->bhsdo_target_transfer_tag,
894 		    bhsdo->bhsdo_initiator_task_tag,
895 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
896 #endif
897 		if (bhsdo->bhsdo_target_transfer_tag ==
898 		    cdw->cdw_target_transfer_tag)
899 			break;
900 	}
901 	CFISCSI_SESSION_UNLOCK(cs);
902 	if (cdw == NULL) {
903 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
904 		    "0x%x, not found; dropping connection",
905 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
906 		icl_pdu_free(request);
907 		cfiscsi_session_terminate(cs);
908 		return;
909 	}
910 
911 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
912 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
913 		    "DataSN %u, while expected %u; dropping connection",
914 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
915 		icl_pdu_free(request);
916 		cfiscsi_session_terminate(cs);
917 		return;
918 	}
919 	cdw->cdw_datasn++;
920 
921 	io = cdw->cdw_ctl_io;
922 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
923 	    ("CTL_FLAG_DATA_IN"));
924 
925 	done = cfiscsi_handle_data_segment(request, cdw);
926 	if (done) {
927 		CFISCSI_SESSION_LOCK(cs);
928 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
929 		CFISCSI_SESSION_UNLOCK(cs);
930 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
931 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
932 		cfiscsi_data_wait_free(cs, cdw);
933 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
934 		if (done)
935 			io->scsiio.be_move_done(io);
936 		else
937 			cfiscsi_datamove_out(io);
938 	}
939 
940 	icl_pdu_free(request);
941 }
942 
943 static void
944 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
945 {
946 	struct iscsi_bhs_logout_request *bhslr;
947 	struct iscsi_bhs_logout_response *bhslr2;
948 	struct icl_pdu *response;
949 	struct cfiscsi_session *cs;
950 
951 	cs = PDU_SESSION(request);
952 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
953 	switch (bhslr->bhslr_reason & 0x7f) {
954 	case BHSLR_REASON_CLOSE_SESSION:
955 	case BHSLR_REASON_CLOSE_CONNECTION:
956 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
957 		if (response == NULL) {
958 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
959 			icl_pdu_free(request);
960 			cfiscsi_session_terminate(cs);
961 			return;
962 		}
963 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
964 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
965 		bhslr2->bhslr_flags = 0x80;
966 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
967 		bhslr2->bhslr_initiator_task_tag =
968 		    bhslr->bhslr_initiator_task_tag;
969 		icl_pdu_free(request);
970 		cfiscsi_pdu_queue(response);
971 		cfiscsi_session_terminate(cs);
972 		break;
973 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
974 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
975 		if (response == NULL) {
976 			CFISCSI_SESSION_WARN(cs,
977 			    "failed to allocate memory; dropping connection");
978 			icl_pdu_free(request);
979 			cfiscsi_session_terminate(cs);
980 			return;
981 		}
982 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
983 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
984 		bhslr2->bhslr_flags = 0x80;
985 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
986 		bhslr2->bhslr_initiator_task_tag =
987 		    bhslr->bhslr_initiator_task_tag;
988 		icl_pdu_free(request);
989 		cfiscsi_pdu_queue(response);
990 		break;
991 	default:
992 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
993 		    bhslr->bhslr_reason);
994 		icl_pdu_free(request);
995 		cfiscsi_session_terminate(cs);
996 		break;
997 	}
998 }
999 
1000 static void
1001 cfiscsi_callout(void *context)
1002 {
1003 	struct icl_pdu *cp;
1004 	struct iscsi_bhs_nop_in *bhsni;
1005 	struct cfiscsi_session *cs;
1006 
1007 	cs = context;
1008 
1009 	if (cs->cs_terminating)
1010 		return;
1011 
1012 	callout_schedule(&cs->cs_callout, 1 * hz);
1013 
1014 	atomic_add_int(&cs->cs_timeout, 1);
1015 
1016 #ifdef ICL_KERNEL_PROXY
1017 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1018 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1019 			CFISCSI_SESSION_WARN(cs, "login timed out after "
1020 			    "%d seconds; dropping connection", cs->cs_timeout);
1021 			cfiscsi_session_terminate(cs);
1022 		}
1023 		return;
1024 	}
1025 #endif
1026 
1027 	if (ping_timeout <= 0) {
1028 		/*
1029 		 * Pings are disabled.  Don't send NOP-In in this case;
1030 		 * user might have disabled pings to work around problems
1031 		 * with certain initiators that can't properly handle
1032 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1033 		 * triggering reconnection, should the user decide to
1034 		 * reenable them.
1035 		 */
1036 		cs->cs_timeout = 0;
1037 		return;
1038 	}
1039 
1040 	if (cs->cs_timeout >= ping_timeout) {
1041 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1042 		    "dropping connection",  ping_timeout);
1043 		cfiscsi_session_terminate(cs);
1044 		return;
1045 	}
1046 
1047 	/*
1048 	 * If the ping was reset less than one second ago - which means
1049 	 * that we've received some PDU during the last second - assume
1050 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1051 	 *
1052 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1053 	 * earlier in this routine.)
1054 	 */
1055 	if (cs->cs_timeout < 2)
1056 		return;
1057 
1058 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1059 	if (cp == NULL) {
1060 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1061 		return;
1062 	}
1063 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1064 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1065 	bhsni->bhsni_flags = 0x80;
1066 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1067 
1068 	cfiscsi_pdu_queue(cp);
1069 }
1070 
1071 static struct cfiscsi_data_wait *
1072 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1073     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1074 {
1075 	struct cfiscsi_data_wait *cdw;
1076 	int error;
1077 
1078 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1079 	if (cdw == NULL) {
1080 		CFISCSI_SESSION_WARN(cs,
1081 		    "failed to allocate %zd bytes", sizeof(*cdw));
1082 		return (NULL);
1083 	}
1084 
1085 	error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1086 	    &cdw->cdw_icl_prv);
1087 	if (error != 0) {
1088 		CFISCSI_SESSION_WARN(cs,
1089 		    "icl_conn_transfer_setup() failed with error %d", error);
1090 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1091 		return (NULL);
1092 	}
1093 
1094 	cdw->cdw_ctl_io = io;
1095 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1096 	cdw->cdw_initiator_task_tag = initiator_task_tag;
1097 
1098 	return (cdw);
1099 }
1100 
1101 static void
1102 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1103     struct cfiscsi_data_wait *cdw)
1104 {
1105 
1106 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1107 	uma_zfree(cfiscsi_data_wait_zone, cdw);
1108 }
1109 
1110 static void
1111 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1112 {
1113 	struct cfiscsi_data_wait *cdw;
1114 	union ctl_io *io;
1115 	int error, last, wait;
1116 
1117 	if (cs->cs_target == NULL)
1118 		return;		/* No target yet, so nothing to do. */
1119 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1120 	ctl_zero_io(io);
1121 	PRIV_REQUEST(io) = cs;
1122 	io->io_hdr.io_type = CTL_IO_TASK;
1123 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1124 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1125 	io->io_hdr.nexus.targ_lun = 0;
1126 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1127 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1128 	wait = cs->cs_outstanding_ctl_pdus;
1129 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1130 	error = ctl_queue(io);
1131 	if (error != CTL_RETVAL_COMPLETE) {
1132 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1133 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1134 		ctl_free_io(io);
1135 	}
1136 
1137 	CFISCSI_SESSION_LOCK(cs);
1138 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1139 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1140 		CFISCSI_SESSION_UNLOCK(cs);
1141 		/*
1142 		 * Set nonzero port status; this prevents backends from
1143 		 * assuming that the data transfer actually succeeded
1144 		 * and writing uninitialized data to disk.
1145 		 */
1146 		cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1147 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1148 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1149 		cfiscsi_data_wait_free(cs, cdw);
1150 		CFISCSI_SESSION_LOCK(cs);
1151 	}
1152 	CFISCSI_SESSION_UNLOCK(cs);
1153 
1154 	/*
1155 	 * Wait for CTL to terminate all the tasks.
1156 	 */
1157 	if (wait > 0)
1158 		CFISCSI_SESSION_WARN(cs,
1159 		    "waiting for CTL to terminate %d tasks", wait);
1160 	for (;;) {
1161 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1162 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1163 		if (last != 0)
1164 			break;
1165 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1166 		    0, "cfiscsi_terminate", hz / 100);
1167 	}
1168 	if (wait > 0)
1169 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1170 }
1171 
1172 static void
1173 cfiscsi_maintenance_thread(void *arg)
1174 {
1175 	struct cfiscsi_session *cs;
1176 
1177 	cs = arg;
1178 
1179 	for (;;) {
1180 		CFISCSI_SESSION_LOCK(cs);
1181 		if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1182 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1183 		CFISCSI_SESSION_UNLOCK(cs);
1184 
1185 		if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1186 			/*
1187 			 * We used to wait up to 30 seconds to deliver queued
1188 			 * PDUs to the initiator.  We also tried hard to deliver
1189 			 * SCSI Responses for the aborted PDUs.  We don't do
1190 			 * that anymore.  We might need to revisit that.
1191 			 */
1192 			callout_drain(&cs->cs_callout);
1193 			icl_conn_close(cs->cs_conn);
1194 
1195 			/*
1196 			 * At this point ICL receive thread is no longer
1197 			 * running; no new tasks can be queued.
1198 			 */
1199 			cfiscsi_session_terminate_tasks(cs);
1200 			cfiscsi_session_delete(cs);
1201 			kthread_exit();
1202 			return;
1203 		}
1204 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1205 	}
1206 }
1207 
1208 static void
1209 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1210 {
1211 
1212 	cs->cs_terminating = true;
1213 	cv_signal(&cs->cs_maintenance_cv);
1214 #ifdef ICL_KERNEL_PROXY
1215 	cv_signal(&cs->cs_login_cv);
1216 #endif
1217 }
1218 
1219 static int
1220 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1221 {
1222 	struct cfiscsi_target *ct;
1223 	char *name;
1224 	int i;
1225 
1226 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1227 
1228 	ct = cs->cs_target;
1229 	name = strdup(cs->cs_initiator_id, M_CTL);
1230 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1231 	if (i < 0) {
1232 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1233 		    i);
1234 		cs->cs_ctl_initid = -1;
1235 		return (1);
1236 	}
1237 	cs->cs_ctl_initid = i;
1238 #if 0
1239 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1240 #endif
1241 
1242 	return (0);
1243 }
1244 
1245 static void
1246 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1247 {
1248 	int error;
1249 
1250 	if (cs->cs_ctl_initid == -1)
1251 		return;
1252 
1253 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1254 	if (error != 0) {
1255 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1256 		    error);
1257 	}
1258 	cs->cs_ctl_initid = -1;
1259 }
1260 
1261 static struct cfiscsi_session *
1262 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1263 {
1264 	struct cfiscsi_session *cs;
1265 	int error;
1266 
1267 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1268 	if (cs == NULL) {
1269 		CFISCSI_WARN("malloc failed");
1270 		return (NULL);
1271 	}
1272 	cs->cs_ctl_initid = -1;
1273 
1274 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1275 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1276 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1277 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1278 #ifdef ICL_KERNEL_PROXY
1279 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1280 #endif
1281 
1282 	/*
1283 	 * The purpose of this is to avoid racing with session shutdown.
1284 	 * Otherwise we could have the maintenance thread call icl_conn_close()
1285 	 * before we call icl_conn_handoff().
1286 	 */
1287 	cs->cs_handoff_in_progress = true;
1288 
1289 	cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1290 	if (cs->cs_conn == NULL) {
1291 		free(cs, M_CFISCSI);
1292 		return (NULL);
1293 	}
1294 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1295 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1296 	cs->cs_conn->ic_prv0 = cs;
1297 
1298 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1299 	if (error != 0) {
1300 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1301 		free(cs, M_CFISCSI);
1302 		return (NULL);
1303 	}
1304 
1305 	mtx_lock(&softc->lock);
1306 	cs->cs_id = ++softc->last_session_id;
1307 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1308 	mtx_unlock(&softc->lock);
1309 
1310 	/*
1311 	 * Start pinging the initiator.
1312 	 */
1313 	callout_init(&cs->cs_callout, 1);
1314 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1315 
1316 	return (cs);
1317 }
1318 
1319 static void
1320 cfiscsi_session_delete(struct cfiscsi_session *cs)
1321 {
1322 	struct cfiscsi_softc *softc;
1323 
1324 	softc = &cfiscsi_softc;
1325 
1326 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1327 	    ("destroying session with outstanding CTL pdus"));
1328 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1329 	    ("destroying session with non-empty queue"));
1330 
1331 	mtx_lock(&softc->lock);
1332 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1333 	mtx_unlock(&softc->lock);
1334 
1335 	cfiscsi_session_unregister_initiator(cs);
1336 	if (cs->cs_target != NULL)
1337 		cfiscsi_target_release(cs->cs_target);
1338 	icl_conn_close(cs->cs_conn);
1339 	icl_conn_free(cs->cs_conn);
1340 	free(cs, M_CFISCSI);
1341 	cv_signal(&softc->sessions_cv);
1342 }
1343 
1344 static int
1345 cfiscsi_init(void)
1346 {
1347 	struct cfiscsi_softc *softc;
1348 
1349 	softc = &cfiscsi_softc;
1350 	bzero(softc, sizeof(*softc));
1351 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1352 
1353 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1354 #ifdef ICL_KERNEL_PROXY
1355 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1356 #endif
1357 	TAILQ_INIT(&softc->sessions);
1358 	TAILQ_INIT(&softc->targets);
1359 
1360 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1361 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1362 	    UMA_ALIGN_PTR, 0);
1363 
1364 	return (0);
1365 }
1366 
1367 static int
1368 cfiscsi_shutdown(void)
1369 {
1370 	struct cfiscsi_softc *softc = &cfiscsi_softc;
1371 
1372 	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1373 		return (EBUSY);
1374 
1375 	uma_zdestroy(cfiscsi_data_wait_zone);
1376 #ifdef ICL_KERNEL_PROXY
1377 	cv_destroy(&softc->accept_cv);
1378 #endif
1379 	cv_destroy(&softc->sessions_cv);
1380 	mtx_destroy(&softc->lock);
1381 	return (0);
1382 }
1383 
1384 #ifdef ICL_KERNEL_PROXY
1385 static void
1386 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1387 {
1388 	struct cfiscsi_session *cs;
1389 
1390 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1391 	if (cs == NULL) {
1392 		CFISCSI_WARN("failed to create session");
1393 		return;
1394 	}
1395 
1396 	icl_conn_handoff_sock(cs->cs_conn, so);
1397 	cs->cs_initiator_sa = sa;
1398 	cs->cs_portal_id = portal_id;
1399 	cs->cs_handoff_in_progress = false;
1400 	cs->cs_waiting_for_ctld = true;
1401 	cv_signal(&cfiscsi_softc.accept_cv);
1402 
1403 	CFISCSI_SESSION_LOCK(cs);
1404 	/*
1405 	 * Wake up the maintenance thread if we got scheduled for termination
1406 	 * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1407 	 */
1408 	if (cs->cs_terminating)
1409 		cfiscsi_session_terminate(cs);
1410 	CFISCSI_SESSION_UNLOCK(cs);
1411 }
1412 #endif
1413 
1414 static void
1415 cfiscsi_online(void *arg)
1416 {
1417 	struct cfiscsi_softc *softc;
1418 	struct cfiscsi_target *ct;
1419 	int online;
1420 
1421 	ct = (struct cfiscsi_target *)arg;
1422 	softc = ct->ct_softc;
1423 
1424 	mtx_lock(&softc->lock);
1425 	if (ct->ct_online) {
1426 		mtx_unlock(&softc->lock);
1427 		return;
1428 	}
1429 	ct->ct_online = 1;
1430 	online = softc->online++;
1431 	mtx_unlock(&softc->lock);
1432 	if (online > 0)
1433 		return;
1434 
1435 #ifdef ICL_KERNEL_PROXY
1436 	if (softc->listener != NULL)
1437 		icl_listen_free(softc->listener);
1438 	softc->listener = icl_listen_new(cfiscsi_accept);
1439 #endif
1440 }
1441 
1442 static void
1443 cfiscsi_offline(void *arg)
1444 {
1445 	struct cfiscsi_softc *softc;
1446 	struct cfiscsi_target *ct;
1447 	struct cfiscsi_session *cs;
1448 	int error, online;
1449 
1450 	ct = (struct cfiscsi_target *)arg;
1451 	softc = ct->ct_softc;
1452 
1453 	mtx_lock(&softc->lock);
1454 	if (!ct->ct_online) {
1455 		mtx_unlock(&softc->lock);
1456 		return;
1457 	}
1458 	ct->ct_online = 0;
1459 	online = --softc->online;
1460 
1461 	do {
1462 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1463 			if (cs->cs_target == ct)
1464 				cfiscsi_session_terminate(cs);
1465 		}
1466 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1467 			if (cs->cs_target == ct)
1468 				break;
1469 		}
1470 		if (cs != NULL) {
1471 			error = cv_wait_sig(&softc->sessions_cv, &softc->lock);
1472 			if (error != 0) {
1473 				CFISCSI_SESSION_DEBUG(cs,
1474 				    "cv_wait failed with error %d\n", error);
1475 				break;
1476 			}
1477 		}
1478 	} while (cs != NULL && ct->ct_online == 0);
1479 	mtx_unlock(&softc->lock);
1480 	if (online > 0)
1481 		return;
1482 
1483 #ifdef ICL_KERNEL_PROXY
1484 	icl_listen_free(softc->listener);
1485 	softc->listener = NULL;
1486 #endif
1487 }
1488 
1489 static int
1490 cfiscsi_info(void *arg, struct sbuf *sb)
1491 {
1492 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1493 	int retval;
1494 
1495 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1496 	    ct->ct_state);
1497 	return (retval);
1498 }
1499 
1500 static void
1501 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1502 {
1503 	struct cfiscsi_softc *softc;
1504 	struct cfiscsi_session *cs, *cs2;
1505 	struct cfiscsi_target *ct;
1506 	struct ctl_iscsi_handoff_params *cihp;
1507 	int error;
1508 
1509 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1510 	softc = &cfiscsi_softc;
1511 
1512 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1513 	    cihp->initiator_name, cihp->initiator_addr,
1514 	    cihp->target_name);
1515 
1516 	ct = cfiscsi_target_find(softc, cihp->target_name,
1517 	    cihp->portal_group_tag);
1518 	if (ct == NULL) {
1519 		ci->status = CTL_ISCSI_ERROR;
1520 		snprintf(ci->error_str, sizeof(ci->error_str),
1521 		    "%s: target not found", __func__);
1522 		return;
1523 	}
1524 
1525 #ifdef ICL_KERNEL_PROXY
1526 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1527 		snprintf(ci->error_str, sizeof(ci->error_str),
1528 		    "both socket and connection_id set");
1529 		ci->status = CTL_ISCSI_ERROR;
1530 		cfiscsi_target_release(ct);
1531 		return;
1532 	}
1533 	if (cihp->socket == 0) {
1534 		mtx_lock(&cfiscsi_softc.lock);
1535 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1536 			if (cs->cs_id == cihp->connection_id)
1537 				break;
1538 		}
1539 		if (cs == NULL) {
1540 			mtx_unlock(&cfiscsi_softc.lock);
1541 			snprintf(ci->error_str, sizeof(ci->error_str),
1542 			    "connection not found");
1543 			ci->status = CTL_ISCSI_ERROR;
1544 			cfiscsi_target_release(ct);
1545 			return;
1546 		}
1547 		mtx_unlock(&cfiscsi_softc.lock);
1548 	} else {
1549 #endif
1550 		cs = cfiscsi_session_new(softc, cihp->offload);
1551 		if (cs == NULL) {
1552 			ci->status = CTL_ISCSI_ERROR;
1553 			snprintf(ci->error_str, sizeof(ci->error_str),
1554 			    "%s: cfiscsi_session_new failed", __func__);
1555 			cfiscsi_target_release(ct);
1556 			return;
1557 		}
1558 #ifdef ICL_KERNEL_PROXY
1559 	}
1560 #endif
1561 
1562 	/*
1563 	 * First PDU of Full Feature phase has the same CmdSN as the last
1564 	 * PDU from the Login Phase received from the initiator.  Thus,
1565 	 * the -1 below.
1566 	 */
1567 	cs->cs_cmdsn = cihp->cmdsn;
1568 	cs->cs_statsn = cihp->statsn;
1569 	cs->cs_max_recv_data_segment_length = cihp->max_recv_data_segment_length;
1570 	cs->cs_max_send_data_segment_length = cihp->max_send_data_segment_length;
1571 	cs->cs_max_burst_length = cihp->max_burst_length;
1572 	cs->cs_first_burst_length = cihp->first_burst_length;
1573 	cs->cs_immediate_data = !!cihp->immediate_data;
1574 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1575 		cs->cs_conn->ic_header_crc32c = true;
1576 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1577 		cs->cs_conn->ic_data_crc32c = true;
1578 
1579 	strlcpy(cs->cs_initiator_name,
1580 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1581 	strlcpy(cs->cs_initiator_addr,
1582 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1583 	strlcpy(cs->cs_initiator_alias,
1584 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1585 	memcpy(cs->cs_initiator_isid,
1586 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1587 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1588 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1589 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1590 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1591 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1592 
1593 	mtx_lock(&softc->lock);
1594 	if (ct->ct_online == 0) {
1595 		mtx_unlock(&softc->lock);
1596 		CFISCSI_SESSION_LOCK(cs);
1597 		cs->cs_handoff_in_progress = false;
1598 		cfiscsi_session_terminate(cs);
1599 		CFISCSI_SESSION_UNLOCK(cs);
1600 		cfiscsi_target_release(ct);
1601 		ci->status = CTL_ISCSI_ERROR;
1602 		snprintf(ci->error_str, sizeof(ci->error_str),
1603 		    "%s: port offline", __func__);
1604 		return;
1605 	}
1606 	cs->cs_target = ct;
1607 	mtx_unlock(&softc->lock);
1608 
1609 restart:
1610 	if (!cs->cs_terminating) {
1611 		mtx_lock(&softc->lock);
1612 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1613 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1614 			    cs->cs_target == cs2->cs_target &&
1615 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1616 				if (strcmp(cs->cs_initiator_addr,
1617 				    cs2->cs_initiator_addr) != 0) {
1618 					CFISCSI_SESSION_WARN(cs2,
1619 					    "session reinstatement from "
1620 					    "different address %s",
1621 					    cs->cs_initiator_addr);
1622 				} else {
1623 					CFISCSI_SESSION_DEBUG(cs2,
1624 					    "session reinstatement");
1625 				}
1626 				cfiscsi_session_terminate(cs2);
1627 				mtx_unlock(&softc->lock);
1628 				pause("cfiscsi_reinstate", 1);
1629 				goto restart;
1630 			}
1631 		}
1632 		mtx_unlock(&softc->lock);
1633 	}
1634 
1635 	/*
1636 	 * Register initiator with CTL.
1637 	 */
1638 	cfiscsi_session_register_initiator(cs);
1639 
1640 #ifdef ICL_KERNEL_PROXY
1641 	if (cihp->socket > 0) {
1642 #endif
1643 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1644 		if (error != 0) {
1645 			CFISCSI_SESSION_LOCK(cs);
1646 			cs->cs_handoff_in_progress = false;
1647 			cfiscsi_session_terminate(cs);
1648 			CFISCSI_SESSION_UNLOCK(cs);
1649 			ci->status = CTL_ISCSI_ERROR;
1650 			snprintf(ci->error_str, sizeof(ci->error_str),
1651 			    "%s: icl_conn_handoff failed with error %d",
1652 			    __func__, error);
1653 			return;
1654 		}
1655 #ifdef ICL_KERNEL_PROXY
1656 	}
1657 #endif
1658 
1659 #ifdef ICL_KERNEL_PROXY
1660 	cs->cs_login_phase = false;
1661 
1662 	/*
1663 	 * First PDU of the Full Feature phase has likely already arrived.
1664 	 * We have to pick it up and execute properly.
1665 	 */
1666 	if (cs->cs_login_pdu != NULL) {
1667 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1668 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1669 		cs->cs_login_pdu = NULL;
1670 	}
1671 #endif
1672 
1673 	CFISCSI_SESSION_LOCK(cs);
1674 	cs->cs_handoff_in_progress = false;
1675 
1676 	/*
1677 	 * Wake up the maintenance thread if we got scheduled for termination.
1678 	 */
1679 	if (cs->cs_terminating)
1680 		cfiscsi_session_terminate(cs);
1681 	CFISCSI_SESSION_UNLOCK(cs);
1682 
1683 	ci->status = CTL_ISCSI_OK;
1684 }
1685 
1686 static void
1687 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1688 {
1689 	struct ctl_iscsi_list_params *cilp;
1690 	struct cfiscsi_session *cs;
1691 	struct cfiscsi_softc *softc;
1692 	struct sbuf *sb;
1693 	int error;
1694 
1695 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1696 	softc = &cfiscsi_softc;
1697 
1698 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1699 	if (sb == NULL) {
1700 		ci->status = CTL_ISCSI_ERROR;
1701 		snprintf(ci->error_str, sizeof(ci->error_str),
1702 		    "Unable to allocate %d bytes for iSCSI session list",
1703 		    cilp->alloc_len);
1704 		return;
1705 	}
1706 
1707 	sbuf_printf(sb, "<ctlislist>\n");
1708 	mtx_lock(&softc->lock);
1709 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1710 		if (cs->cs_target == NULL)
1711 			continue;
1712 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1713 		    "<initiator>%s</initiator>"
1714 		    "<initiator_addr>%s</initiator_addr>"
1715 		    "<initiator_alias>%s</initiator_alias>"
1716 		    "<target>%s</target>"
1717 		    "<target_alias>%s</target_alias>"
1718 		    "<target_portal_group_tag>%u</target_portal_group_tag>"
1719 		    "<header_digest>%s</header_digest>"
1720 		    "<data_digest>%s</data_digest>"
1721 		    "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1722 		    "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1723 		    "<max_burst_length>%d</max_burst_length>"
1724 		    "<first_burst_length>%d</first_burst_length>"
1725 		    "<immediate_data>%d</immediate_data>"
1726 		    "<iser>%d</iser>"
1727 		    "<offload>%s</offload>"
1728 		    "</connection>\n",
1729 		    cs->cs_id,
1730 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1731 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1732 		    cs->cs_target->ct_tag,
1733 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1734 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1735 		    cs->cs_max_recv_data_segment_length,
1736 		    cs->cs_max_send_data_segment_length,
1737 		    cs->cs_max_burst_length,
1738 		    cs->cs_first_burst_length,
1739 		    cs->cs_immediate_data,
1740 		    cs->cs_conn->ic_iser,
1741 		    cs->cs_conn->ic_offload);
1742 		if (error != 0)
1743 			break;
1744 	}
1745 	mtx_unlock(&softc->lock);
1746 	error = sbuf_printf(sb, "</ctlislist>\n");
1747 	if (error != 0) {
1748 		sbuf_delete(sb);
1749 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1750 		snprintf(ci->error_str, sizeof(ci->error_str),
1751 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1752 		return;
1753 	}
1754 	sbuf_finish(sb);
1755 
1756 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1757 	if (error != 0) {
1758 		sbuf_delete(sb);
1759 		snprintf(ci->error_str, sizeof(ci->error_str),
1760 		    "copyout failed with error %d", error);
1761 		ci->status = CTL_ISCSI_ERROR;
1762 		return;
1763 	}
1764 	cilp->fill_len = sbuf_len(sb) + 1;
1765 	ci->status = CTL_ISCSI_OK;
1766 	sbuf_delete(sb);
1767 }
1768 
1769 static void
1770 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1771 {
1772 	struct icl_pdu *response;
1773 	struct iscsi_bhs_asynchronous_message *bhsam;
1774 	struct ctl_iscsi_logout_params *cilp;
1775 	struct cfiscsi_session *cs;
1776 	struct cfiscsi_softc *softc;
1777 	int found = 0;
1778 
1779 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1780 	softc = &cfiscsi_softc;
1781 
1782 	mtx_lock(&softc->lock);
1783 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1784 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1785 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1786 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1787 			continue;
1788 
1789 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1790 		if (response == NULL) {
1791 			ci->status = CTL_ISCSI_ERROR;
1792 			snprintf(ci->error_str, sizeof(ci->error_str),
1793 			    "Unable to allocate memory");
1794 			mtx_unlock(&softc->lock);
1795 			return;
1796 		}
1797 		bhsam =
1798 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1799 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1800 		bhsam->bhsam_flags = 0x80;
1801 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1802 		bhsam->bhsam_parameter3 = htons(10);
1803 		cfiscsi_pdu_queue(response);
1804 		found++;
1805 	}
1806 	mtx_unlock(&softc->lock);
1807 
1808 	if (found == 0) {
1809 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1810 		snprintf(ci->error_str, sizeof(ci->error_str),
1811 		    "No matching connections found");
1812 		return;
1813 	}
1814 
1815 	ci->status = CTL_ISCSI_OK;
1816 }
1817 
1818 static void
1819 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1820 {
1821 	struct icl_pdu *response;
1822 	struct iscsi_bhs_asynchronous_message *bhsam;
1823 	struct ctl_iscsi_terminate_params *citp;
1824 	struct cfiscsi_session *cs;
1825 	struct cfiscsi_softc *softc;
1826 	int found = 0;
1827 
1828 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1829 	softc = &cfiscsi_softc;
1830 
1831 	mtx_lock(&softc->lock);
1832 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1833 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1834 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1835 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1836 			continue;
1837 
1838 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1839 		if (response == NULL) {
1840 			/*
1841 			 * Oh well.  Just terminate the connection.
1842 			 */
1843 		} else {
1844 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1845 			    response->ip_bhs;
1846 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1847 			bhsam->bhsam_flags = 0x80;
1848 			bhsam->bhsam_0xffffffff = 0xffffffff;
1849 			bhsam->bhsam_async_event =
1850 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1851 			cfiscsi_pdu_queue(response);
1852 		}
1853 		cfiscsi_session_terminate(cs);
1854 		found++;
1855 	}
1856 	mtx_unlock(&softc->lock);
1857 
1858 	if (found == 0) {
1859 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1860 		snprintf(ci->error_str, sizeof(ci->error_str),
1861 		    "No matching connections found");
1862 		return;
1863 	}
1864 
1865 	ci->status = CTL_ISCSI_OK;
1866 }
1867 
1868 static void
1869 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1870 {
1871 	struct ctl_iscsi_limits_params *cilp;
1872 	struct icl_drv_limits idl;
1873 	int error;
1874 
1875 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1876 
1877 	error = icl_limits(cilp->offload, false, &idl);
1878 	if (error != 0) {
1879 		ci->status = CTL_ISCSI_ERROR;
1880 		snprintf(ci->error_str, sizeof(ci->error_str),
1881 			"%s: icl_limits failed with error %d",
1882 			__func__, error);
1883 		return;
1884 	}
1885 
1886 	cilp->max_recv_data_segment_length =
1887 	    idl.idl_max_recv_data_segment_length;
1888 	cilp->max_send_data_segment_length =
1889 	    idl.idl_max_send_data_segment_length;
1890 	cilp->max_burst_length = idl.idl_max_burst_length;
1891 	cilp->first_burst_length = idl.idl_first_burst_length;
1892 
1893 	ci->status = CTL_ISCSI_OK;
1894 }
1895 
1896 #ifdef ICL_KERNEL_PROXY
1897 static void
1898 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1899 {
1900 	struct ctl_iscsi_listen_params *cilp;
1901 	struct sockaddr *sa;
1902 	int error;
1903 
1904 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1905 
1906 	if (cfiscsi_softc.listener == NULL) {
1907 		CFISCSI_DEBUG("no listener");
1908 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1909 		ci->status = CTL_ISCSI_ERROR;
1910 		return;
1911 	}
1912 
1913 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1914 	if (error != 0) {
1915 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1916 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1917 		ci->status = CTL_ISCSI_ERROR;
1918 		return;
1919 	}
1920 
1921 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1922 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1923 	if (error != 0) {
1924 		free(sa, M_SONAME);
1925 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1926 		snprintf(ci->error_str, sizeof(ci->error_str),
1927 		    "icl_listen_add failed, error %d", error);
1928 		ci->status = CTL_ISCSI_ERROR;
1929 		return;
1930 	}
1931 
1932 	ci->status = CTL_ISCSI_OK;
1933 }
1934 
1935 static void
1936 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1937 {
1938 	struct ctl_iscsi_accept_params *ciap;
1939 	struct cfiscsi_session *cs;
1940 	int error;
1941 
1942 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1943 
1944 	mtx_lock(&cfiscsi_softc.lock);
1945 	for (;;) {
1946 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1947 			if (cs->cs_waiting_for_ctld)
1948 				break;
1949 		}
1950 		if (cs != NULL)
1951 			break;
1952 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1953 		if (error != 0) {
1954 			mtx_unlock(&cfiscsi_softc.lock);
1955 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1956 			ci->status = CTL_ISCSI_ERROR;
1957 			return;
1958 		}
1959 	}
1960 	mtx_unlock(&cfiscsi_softc.lock);
1961 
1962 	cs->cs_waiting_for_ctld = false;
1963 	cs->cs_login_phase = true;
1964 
1965 	ciap->connection_id = cs->cs_id;
1966 	ciap->portal_id = cs->cs_portal_id;
1967 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1968 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1969 	    cs->cs_initiator_sa->sa_len);
1970 	if (error != 0) {
1971 		snprintf(ci->error_str, sizeof(ci->error_str),
1972 		    "copyout failed with error %d", error);
1973 		ci->status = CTL_ISCSI_ERROR;
1974 		return;
1975 	}
1976 
1977 	ci->status = CTL_ISCSI_OK;
1978 }
1979 
1980 static void
1981 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1982 {
1983 	struct ctl_iscsi_send_params *cisp;
1984 	struct cfiscsi_session *cs;
1985 	struct icl_pdu *ip;
1986 	size_t datalen;
1987 	void *data;
1988 	int error;
1989 
1990 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1991 
1992 	mtx_lock(&cfiscsi_softc.lock);
1993 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1994 		if (cs->cs_id == cisp->connection_id)
1995 			break;
1996 	}
1997 	if (cs == NULL) {
1998 		mtx_unlock(&cfiscsi_softc.lock);
1999 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
2000 		ci->status = CTL_ISCSI_ERROR;
2001 		return;
2002 	}
2003 	mtx_unlock(&cfiscsi_softc.lock);
2004 
2005 #if 0
2006 	if (cs->cs_login_phase == false)
2007 		return (EBUSY);
2008 #endif
2009 
2010 	if (cs->cs_terminating) {
2011 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
2012 		ci->status = CTL_ISCSI_ERROR;
2013 		return;
2014 	}
2015 
2016 	datalen = cisp->data_segment_len;
2017 	/*
2018 	 * XXX
2019 	 */
2020 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2021 	if (datalen > 65535) {
2022 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2023 		ci->status = CTL_ISCSI_ERROR;
2024 		return;
2025 	}
2026 	if (datalen > 0) {
2027 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
2028 		error = copyin(cisp->data_segment, data, datalen);
2029 		if (error != 0) {
2030 			free(data, M_CFISCSI);
2031 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2032 			ci->status = CTL_ISCSI_ERROR;
2033 			return;
2034 		}
2035 	}
2036 
2037 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2038 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2039 	if (datalen > 0) {
2040 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2041 		free(data, M_CFISCSI);
2042 	}
2043 	CFISCSI_SESSION_LOCK(cs);
2044 	icl_pdu_queue(ip);
2045 	CFISCSI_SESSION_UNLOCK(cs);
2046 	ci->status = CTL_ISCSI_OK;
2047 }
2048 
2049 static void
2050 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2051 {
2052 	struct ctl_iscsi_receive_params *cirp;
2053 	struct cfiscsi_session *cs;
2054 	struct icl_pdu *ip;
2055 	void *data;
2056 	int error;
2057 
2058 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2059 
2060 	mtx_lock(&cfiscsi_softc.lock);
2061 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2062 		if (cs->cs_id == cirp->connection_id)
2063 			break;
2064 	}
2065 	if (cs == NULL) {
2066 		mtx_unlock(&cfiscsi_softc.lock);
2067 		snprintf(ci->error_str, sizeof(ci->error_str),
2068 		    "connection not found");
2069 		ci->status = CTL_ISCSI_ERROR;
2070 		return;
2071 	}
2072 	mtx_unlock(&cfiscsi_softc.lock);
2073 
2074 #if 0
2075 	if (is->is_login_phase == false)
2076 		return (EBUSY);
2077 #endif
2078 
2079 	CFISCSI_SESSION_LOCK(cs);
2080 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2081 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2082 		if (error != 0) {
2083 			CFISCSI_SESSION_UNLOCK(cs);
2084 			snprintf(ci->error_str, sizeof(ci->error_str),
2085 			    "interrupted by signal");
2086 			ci->status = CTL_ISCSI_ERROR;
2087 			return;
2088 		}
2089 	}
2090 
2091 	if (cs->cs_terminating) {
2092 		CFISCSI_SESSION_UNLOCK(cs);
2093 		snprintf(ci->error_str, sizeof(ci->error_str),
2094 		    "connection terminating");
2095 		ci->status = CTL_ISCSI_ERROR;
2096 		return;
2097 	}
2098 	ip = cs->cs_login_pdu;
2099 	cs->cs_login_pdu = NULL;
2100 	CFISCSI_SESSION_UNLOCK(cs);
2101 
2102 	if (ip->ip_data_len > cirp->data_segment_len) {
2103 		icl_pdu_free(ip);
2104 		snprintf(ci->error_str, sizeof(ci->error_str),
2105 		    "data segment too big");
2106 		ci->status = CTL_ISCSI_ERROR;
2107 		return;
2108 	}
2109 
2110 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2111 	if (ip->ip_data_len > 0) {
2112 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2113 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2114 		copyout(data, cirp->data_segment, ip->ip_data_len);
2115 		free(data, M_CFISCSI);
2116 	}
2117 
2118 	icl_pdu_free(ip);
2119 	ci->status = CTL_ISCSI_OK;
2120 }
2121 
2122 #endif /* !ICL_KERNEL_PROXY */
2123 
2124 static void
2125 cfiscsi_ioctl_port_create(struct ctl_req *req)
2126 {
2127 	struct cfiscsi_target *ct;
2128 	struct ctl_port *port;
2129 	const char *target, *alias, *val;
2130 	struct scsi_vpd_id_descriptor *desc;
2131 	int retval, len, idlen;
2132 	uint16_t tag;
2133 
2134 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2135 	alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
2136 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2137 	    NULL);
2138 
2139 	if (target == NULL || val == NULL) {
2140 		req->status = CTL_LUN_ERROR;
2141 		snprintf(req->error_str, sizeof(req->error_str),
2142 		    "Missing required argument");
2143 		return;
2144 	}
2145 
2146 	tag = strtoul(val, NULL, 0);
2147 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2148 	if (ct == NULL) {
2149 		req->status = CTL_LUN_ERROR;
2150 		snprintf(req->error_str, sizeof(req->error_str),
2151 		    "failed to create target \"%s\"", target);
2152 		return;
2153 	}
2154 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2155 		req->status = CTL_LUN_ERROR;
2156 		snprintf(req->error_str, sizeof(req->error_str),
2157 		    "target \"%s\" for portal group tag %u already exists",
2158 		    target, tag);
2159 		cfiscsi_target_release(ct);
2160 		return;
2161 	}
2162 	port = &ct->ct_port;
2163 	// WAT
2164 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2165 		goto done;
2166 
2167 	port->frontend = &cfiscsi_frontend;
2168 	port->port_type = CTL_PORT_ISCSI;
2169 	/* XXX KDM what should the real number be here? */
2170 	port->num_requested_ctl_io = 4096;
2171 	port->port_name = "iscsi";
2172 	port->physical_port = (int)tag;
2173 	port->virtual_port = ct->ct_target_id;
2174 	port->port_online = cfiscsi_online;
2175 	port->port_offline = cfiscsi_offline;
2176 	port->port_info = cfiscsi_info;
2177 	port->onoff_arg = ct;
2178 	port->fe_datamove = cfiscsi_datamove;
2179 	port->fe_done = cfiscsi_done;
2180 	port->targ_port = -1;
2181 	port->options = nvlist_clone(req->args_nvl);
2182 
2183 	/* Generate Port ID. */
2184 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2185 	idlen = roundup2(idlen, 4);
2186 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2187 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2188 	    M_CTL, M_WAITOK | M_ZERO);
2189 	port->port_devid->len = len;
2190 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2191 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2192 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2193 	    SVPD_ID_TYPE_SCSI_NAME;
2194 	desc->length = idlen;
2195 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2196 
2197 	/* Generate Target ID. */
2198 	idlen = strlen(target) + 1;
2199 	idlen = roundup2(idlen, 4);
2200 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2201 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2202 	    M_CTL, M_WAITOK | M_ZERO);
2203 	port->target_devid->len = len;
2204 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2205 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2206 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2207 	    SVPD_ID_TYPE_SCSI_NAME;
2208 	desc->length = idlen;
2209 	strlcpy(desc->identifier, target, idlen);
2210 
2211 	retval = ctl_port_register(port);
2212 	if (retval != 0) {
2213 		free(port->port_devid, M_CFISCSI);
2214 		free(port->target_devid, M_CFISCSI);
2215 		cfiscsi_target_release(ct);
2216 		req->status = CTL_LUN_ERROR;
2217 		snprintf(req->error_str, sizeof(req->error_str),
2218 		    "ctl_port_register() failed with error %d", retval);
2219 		return;
2220 	}
2221 done:
2222 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2223 	req->status = CTL_LUN_OK;
2224 	req->result_nvl = nvlist_create(0);
2225 	nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
2226 }
2227 
2228 static void
2229 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2230 {
2231 	struct cfiscsi_target *ct;
2232 	const char *target, *val;
2233 	uint16_t tag;
2234 
2235 	target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2236 	val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2237 	    NULL);
2238 
2239 	if (target == NULL || val == NULL) {
2240 		req->status = CTL_LUN_ERROR;
2241 		snprintf(req->error_str, sizeof(req->error_str),
2242 		    "Missing required argument");
2243 		return;
2244 	}
2245 
2246 	tag = strtoul(val, NULL, 0);
2247 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2248 	if (ct == NULL) {
2249 		req->status = CTL_LUN_ERROR;
2250 		snprintf(req->error_str, sizeof(req->error_str),
2251 		    "can't find target \"%s\"", target);
2252 		return;
2253 	}
2254 
2255 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2256 	ctl_port_offline(&ct->ct_port);
2257 	cfiscsi_target_release(ct);
2258 	cfiscsi_target_release(ct);
2259 	req->status = CTL_LUN_OK;
2260 }
2261 
2262 static int
2263 cfiscsi_ioctl(struct cdev *dev,
2264     u_long cmd, caddr_t addr, int flag, struct thread *td)
2265 {
2266 	struct ctl_iscsi *ci;
2267 	struct ctl_req *req;
2268 
2269 	if (cmd == CTL_PORT_REQ) {
2270 		req = (struct ctl_req *)addr;
2271 		switch (req->reqtype) {
2272 		case CTL_REQ_CREATE:
2273 			cfiscsi_ioctl_port_create(req);
2274 			break;
2275 		case CTL_REQ_REMOVE:
2276 			cfiscsi_ioctl_port_remove(req);
2277 			break;
2278 		default:
2279 			req->status = CTL_LUN_ERROR;
2280 			snprintf(req->error_str, sizeof(req->error_str),
2281 			    "Unsupported request type %d", req->reqtype);
2282 		}
2283 		return (0);
2284 	}
2285 
2286 	if (cmd != CTL_ISCSI)
2287 		return (ENOTTY);
2288 
2289 	ci = (struct ctl_iscsi *)addr;
2290 	switch (ci->type) {
2291 	case CTL_ISCSI_HANDOFF:
2292 		cfiscsi_ioctl_handoff(ci);
2293 		break;
2294 	case CTL_ISCSI_LIST:
2295 		cfiscsi_ioctl_list(ci);
2296 		break;
2297 	case CTL_ISCSI_LOGOUT:
2298 		cfiscsi_ioctl_logout(ci);
2299 		break;
2300 	case CTL_ISCSI_TERMINATE:
2301 		cfiscsi_ioctl_terminate(ci);
2302 		break;
2303 	case CTL_ISCSI_LIMITS:
2304 		cfiscsi_ioctl_limits(ci);
2305 		break;
2306 #ifdef ICL_KERNEL_PROXY
2307 	case CTL_ISCSI_LISTEN:
2308 		cfiscsi_ioctl_listen(ci);
2309 		break;
2310 	case CTL_ISCSI_ACCEPT:
2311 		cfiscsi_ioctl_accept(ci);
2312 		break;
2313 	case CTL_ISCSI_SEND:
2314 		cfiscsi_ioctl_send(ci);
2315 		break;
2316 	case CTL_ISCSI_RECEIVE:
2317 		cfiscsi_ioctl_receive(ci);
2318 		break;
2319 #else
2320 	case CTL_ISCSI_LISTEN:
2321 	case CTL_ISCSI_ACCEPT:
2322 	case CTL_ISCSI_SEND:
2323 	case CTL_ISCSI_RECEIVE:
2324 		ci->status = CTL_ISCSI_ERROR;
2325 		snprintf(ci->error_str, sizeof(ci->error_str),
2326 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2327 		    __func__);
2328 		break;
2329 #endif /* !ICL_KERNEL_PROXY */
2330 	default:
2331 		ci->status = CTL_ISCSI_ERROR;
2332 		snprintf(ci->error_str, sizeof(ci->error_str),
2333 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2334 		break;
2335 	}
2336 
2337 	return (0);
2338 }
2339 
2340 static void
2341 cfiscsi_target_hold(struct cfiscsi_target *ct)
2342 {
2343 
2344 	refcount_acquire(&ct->ct_refcount);
2345 }
2346 
2347 static void
2348 cfiscsi_target_release(struct cfiscsi_target *ct)
2349 {
2350 	struct cfiscsi_softc *softc;
2351 
2352 	softc = ct->ct_softc;
2353 	mtx_lock(&softc->lock);
2354 	if (refcount_release(&ct->ct_refcount)) {
2355 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2356 		mtx_unlock(&softc->lock);
2357 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2358 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2359 			if (ctl_port_deregister(&ct->ct_port) != 0)
2360 				printf("%s: ctl_port_deregister() failed\n",
2361 				    __func__);
2362 		}
2363 		free(ct, M_CFISCSI);
2364 
2365 		return;
2366 	}
2367 	mtx_unlock(&softc->lock);
2368 }
2369 
2370 static struct cfiscsi_target *
2371 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2372 {
2373 	struct cfiscsi_target *ct;
2374 
2375 	mtx_lock(&softc->lock);
2376 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2377 		if (ct->ct_tag != tag ||
2378 		    strcmp(name, ct->ct_name) != 0 ||
2379 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2380 			continue;
2381 		cfiscsi_target_hold(ct);
2382 		mtx_unlock(&softc->lock);
2383 		return (ct);
2384 	}
2385 	mtx_unlock(&softc->lock);
2386 
2387 	return (NULL);
2388 }
2389 
2390 static struct cfiscsi_target *
2391 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2392     const char *alias, uint16_t tag)
2393 {
2394 	struct cfiscsi_target *ct, *newct;
2395 
2396 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2397 		return (NULL);
2398 
2399 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2400 
2401 	mtx_lock(&softc->lock);
2402 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2403 		if (ct->ct_tag != tag ||
2404 		    strcmp(name, ct->ct_name) != 0 ||
2405 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2406 			continue;
2407 		cfiscsi_target_hold(ct);
2408 		mtx_unlock(&softc->lock);
2409 		free(newct, M_CFISCSI);
2410 		return (ct);
2411 	}
2412 
2413 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2414 	if (alias != NULL)
2415 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2416 	newct->ct_tag = tag;
2417 	refcount_init(&newct->ct_refcount, 1);
2418 	newct->ct_softc = softc;
2419 	if (TAILQ_EMPTY(&softc->targets))
2420 		softc->last_target_id = 0;
2421 	newct->ct_target_id = ++softc->last_target_id;
2422 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2423 	mtx_unlock(&softc->lock);
2424 
2425 	return (newct);
2426 }
2427 
2428 static void
2429 cfiscsi_pdu_done(struct icl_pdu *ip, int error)
2430 {
2431 
2432 	if (error != 0)
2433 		; // XXX: Do something on error?
2434 	((ctl_ref)ip->ip_prv0)(ip->ip_prv1, -1);
2435 }
2436 
2437 static void
2438 cfiscsi_datamove_in(union ctl_io *io)
2439 {
2440 	struct cfiscsi_session *cs;
2441 	struct icl_pdu *request, *response;
2442 	const struct iscsi_bhs_scsi_command *bhssc;
2443 	struct iscsi_bhs_data_in *bhsdi;
2444 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2445 	size_t len, expected_len, sg_len, buffer_offset;
2446 	const char *sg_addr;
2447 	icl_pdu_cb cb;
2448 	int ctl_sg_count, error, i;
2449 
2450 	request = PRIV_REQUEST(io);
2451 	cs = PDU_SESSION(request);
2452 
2453 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2454 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2455 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2456 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2457 
2458 	if (io->scsiio.kern_sg_entries > 0) {
2459 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2460 		ctl_sg_count = io->scsiio.kern_sg_entries;
2461 	} else {
2462 		ctl_sglist = &ctl_sg_entry;
2463 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2464 		ctl_sglist->len = io->scsiio.kern_data_len;
2465 		ctl_sg_count = 1;
2466 	}
2467 
2468 	/*
2469 	 * This is the offset within the current SCSI command; for the first
2470 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2471 	 * it will be the sum of lengths of previous ones.
2472 	 */
2473 	buffer_offset = io->scsiio.kern_rel_offset;
2474 
2475 	/*
2476 	 * This is the transfer length expected by the initiator.  It can be
2477 	 * different from the amount of data from the SCSI point of view.
2478 	 */
2479 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2480 
2481 	/*
2482 	 * If the transfer is outside of expected length -- we are done.
2483 	 */
2484 	if (buffer_offset >= expected_len) {
2485 #if 0
2486 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2487 		    "already sent the expected len", buffer_offset);
2488 #endif
2489 		io->scsiio.be_move_done(io);
2490 		return;
2491 	}
2492 
2493 	if (io->scsiio.kern_data_ref != NULL)
2494 		cb = cfiscsi_pdu_done;
2495 	else
2496 		cb = NULL;
2497 
2498 	i = 0;
2499 	sg_addr = NULL;
2500 	sg_len = 0;
2501 	response = NULL;
2502 	bhsdi = NULL;
2503 	for (;;) {
2504 		if (response == NULL) {
2505 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2506 			if (response == NULL) {
2507 				CFISCSI_SESSION_WARN(cs, "failed to "
2508 				    "allocate memory; dropping connection");
2509 				ctl_set_busy(&io->scsiio);
2510 				io->scsiio.be_move_done(io);
2511 				cfiscsi_session_terminate(cs);
2512 				return;
2513 			}
2514 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2515 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2516 			bhsdi->bhsdi_initiator_task_tag =
2517 			    bhssc->bhssc_initiator_task_tag;
2518 			bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2519 			bhsdi->bhsdi_datasn = htonl(PRIV_EXPDATASN(io)++);
2520 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2521 		}
2522 
2523 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2524 		if (sg_len == 0) {
2525 			sg_addr = ctl_sglist[i].addr;
2526 			sg_len = ctl_sglist[i].len;
2527 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2528 		}
2529 
2530 		len = sg_len;
2531 
2532 		/*
2533 		 * Truncate to maximum data segment length.
2534 		 */
2535 		KASSERT(response->ip_data_len < cs->cs_max_send_data_segment_length,
2536 		    ("ip_data_len %zd >= max_send_data_segment_length %d",
2537 		    response->ip_data_len, cs->cs_max_send_data_segment_length));
2538 		if (response->ip_data_len + len >
2539 		    cs->cs_max_send_data_segment_length) {
2540 			len = cs->cs_max_send_data_segment_length -
2541 			    response->ip_data_len;
2542 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2543 			    len, sg_len));
2544 		}
2545 
2546 		/*
2547 		 * Truncate to expected data transfer length.
2548 		 */
2549 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2550 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2551 		    buffer_offset, response->ip_data_len, expected_len));
2552 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2553 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2554 			    "to expected data transfer length %zd",
2555 			    buffer_offset + response->ip_data_len + len, expected_len);
2556 			len = expected_len - (buffer_offset + response->ip_data_len);
2557 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2558 			    len, sg_len));
2559 		}
2560 
2561 		error = icl_pdu_append_data(response, sg_addr, len,
2562 		    M_NOWAIT | (cb ? ICL_NOCOPY : 0));
2563 		if (error != 0) {
2564 			CFISCSI_SESSION_WARN(cs, "failed to "
2565 			    "allocate memory; dropping connection");
2566 			icl_pdu_free(response);
2567 			ctl_set_busy(&io->scsiio);
2568 			io->scsiio.be_move_done(io);
2569 			cfiscsi_session_terminate(cs);
2570 			return;
2571 		}
2572 		sg_addr += len;
2573 		sg_len -= len;
2574 		io->scsiio.kern_data_resid -= len;
2575 
2576 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2577 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2578 		    buffer_offset, response->ip_data_len, expected_len));
2579 		if (buffer_offset + response->ip_data_len == expected_len) {
2580 			/*
2581 			 * Already have the amount of data the initiator wanted.
2582 			 */
2583 			break;
2584 		}
2585 
2586 		if (sg_len == 0) {
2587 			/*
2588 			 * End of scatter-gather segment;
2589 			 * proceed to the next one...
2590 			 */
2591 			if (i == ctl_sg_count - 1) {
2592 				/*
2593 				 * ... unless this was the last one.
2594 				 */
2595 				break;
2596 			}
2597 			i++;
2598 		}
2599 
2600 		if (response->ip_data_len == cs->cs_max_send_data_segment_length) {
2601 			/*
2602 			 * Can't stuff more data into the current PDU;
2603 			 * queue it.  Note that's not enough to check
2604 			 * for kern_data_resid == 0 instead; there
2605 			 * may be several Data-In PDUs for the final
2606 			 * call to cfiscsi_datamove(), and we want
2607 			 * to set the F flag only on the last of them.
2608 			 */
2609 			buffer_offset += response->ip_data_len;
2610 			if (buffer_offset == io->scsiio.kern_total_len ||
2611 			    buffer_offset == expected_len) {
2612 				buffer_offset -= response->ip_data_len;
2613 				break;
2614 			}
2615 			if (cb != NULL) {
2616 				response->ip_prv0 = io->scsiio.kern_data_ref;
2617 				response->ip_prv1 = io->scsiio.kern_data_arg;
2618 				io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2619 			}
2620 			cfiscsi_pdu_queue_cb(response, cb);
2621 			response = NULL;
2622 			bhsdi = NULL;
2623 		}
2624 	}
2625 	if (response != NULL) {
2626 		buffer_offset += response->ip_data_len;
2627 		if (buffer_offset == io->scsiio.kern_total_len ||
2628 		    buffer_offset == expected_len) {
2629 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2630 			if (io->io_hdr.status == CTL_SUCCESS) {
2631 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2632 				if (io->scsiio.kern_total_len <
2633 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2634 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2635 					bhsdi->bhsdi_residual_count =
2636 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2637 					    io->scsiio.kern_total_len);
2638 				} else if (io->scsiio.kern_total_len >
2639 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2640 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2641 					bhsdi->bhsdi_residual_count =
2642 					    htonl(io->scsiio.kern_total_len -
2643 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2644 				}
2645 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2646 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2647 			}
2648 		}
2649 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2650 		if (cb != NULL) {
2651 			response->ip_prv0 = io->scsiio.kern_data_ref;
2652 			response->ip_prv1 = io->scsiio.kern_data_arg;
2653 			io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2654 		}
2655 		cfiscsi_pdu_queue_cb(response, cb);
2656 	}
2657 
2658 	io->scsiio.be_move_done(io);
2659 }
2660 
2661 static void
2662 cfiscsi_datamove_out(union ctl_io *io)
2663 {
2664 	struct cfiscsi_session *cs;
2665 	struct icl_pdu *request, *response;
2666 	const struct iscsi_bhs_scsi_command *bhssc;
2667 	struct iscsi_bhs_r2t *bhsr2t;
2668 	struct cfiscsi_data_wait *cdw;
2669 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2670 	uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2671 	uint32_t target_transfer_tag;
2672 	bool done;
2673 
2674 	request = PRIV_REQUEST(io);
2675 	cs = PDU_SESSION(request);
2676 
2677 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2678 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2679 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2680 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2681 
2682 	/*
2683 	 * Complete write underflow.  Not a single byte to read.  Return.
2684 	 */
2685 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2686 	if (io->scsiio.kern_rel_offset >= expected_len) {
2687 		io->scsiio.be_move_done(io);
2688 		return;
2689 	}
2690 	datamove_len = MIN(io->scsiio.kern_data_len,
2691 	    expected_len - io->scsiio.kern_rel_offset);
2692 
2693 	target_transfer_tag =
2694 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2695 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2696 	    &target_transfer_tag);
2697 	if (cdw == NULL) {
2698 		CFISCSI_SESSION_WARN(cs, "failed to "
2699 		    "allocate memory; dropping connection");
2700 		ctl_set_busy(&io->scsiio);
2701 		io->scsiio.be_move_done(io);
2702 		cfiscsi_session_terminate(cs);
2703 		return;
2704 	}
2705 #if 0
2706 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2707 	    "task tag 0x%x, target transfer tag 0x%x",
2708 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2709 #endif
2710 
2711 	cdw->cdw_ctl_io = io;
2712 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2713 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2714 	cdw->cdw_r2t_end = datamove_len;
2715 	cdw->cdw_datasn = 0;
2716 
2717 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2718 	if (io->scsiio.kern_sg_entries > 0) {
2719 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2720 	} else {
2721 		ctl_sglist = &ctl_sg_entry;
2722 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2723 		ctl_sglist->len = datamove_len;
2724 	}
2725 	cdw->cdw_sg_index = 0;
2726 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2727 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2728 	r2t_off = io->scsiio.ext_data_filled;
2729 	while (r2t_off > 0) {
2730 		if (r2t_off >= cdw->cdw_sg_len) {
2731 			r2t_off -= cdw->cdw_sg_len;
2732 			cdw->cdw_sg_index++;
2733 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2734 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2735 			continue;
2736 		}
2737 		cdw->cdw_sg_addr += r2t_off;
2738 		cdw->cdw_sg_len -= r2t_off;
2739 		r2t_off = 0;
2740 	}
2741 
2742 	if (cs->cs_immediate_data &&
2743 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2744 	    icl_pdu_data_segment_length(request)) {
2745 		done = cfiscsi_handle_data_segment(request, cdw);
2746 		if (done) {
2747 			cfiscsi_data_wait_free(cs, cdw);
2748 			io->scsiio.be_move_done(io);
2749 			return;
2750 		}
2751 	}
2752 
2753 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2754 	r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2755 	    cs->cs_max_burst_length);
2756 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2757 
2758 	CFISCSI_SESSION_LOCK(cs);
2759 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2760 	CFISCSI_SESSION_UNLOCK(cs);
2761 
2762 	/*
2763 	 * XXX: We should limit the number of outstanding R2T PDUs
2764 	 * 	per task to MaxOutstandingR2T.
2765 	 */
2766 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2767 	if (response == NULL) {
2768 		CFISCSI_SESSION_WARN(cs, "failed to "
2769 		    "allocate memory; dropping connection");
2770 		ctl_set_busy(&io->scsiio);
2771 		io->scsiio.be_move_done(io);
2772 		cfiscsi_session_terminate(cs);
2773 		return;
2774 	}
2775 	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2776 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2777 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2778 	bhsr2t->bhsr2t_flags = 0x80;
2779 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2780 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2781 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2782 	/*
2783 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2784 	 *	be running concurrently on several CPUs for a given
2785 	 *	command.
2786 	 */
2787 	bhsr2t->bhsr2t_r2tsn = htonl(PRIV_R2TSN(io)++);
2788 	/*
2789 	 * This is the offset within the current SCSI command;
2790 	 * i.e. for the first call of datamove(), it will be 0,
2791 	 * and for subsequent ones it will be the sum of lengths
2792 	 * of previous ones.
2793 	 *
2794 	 * The ext_data_filled is to account for unsolicited
2795 	 * (immediate) data that might have already arrived.
2796 	 */
2797 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2798 	/*
2799 	 * This is the total length (sum of S/G lengths) this call
2800 	 * to cfiscsi_datamove() is supposed to handle, limited by
2801 	 * MaxBurstLength.
2802 	 */
2803 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2804 	cfiscsi_pdu_queue(response);
2805 }
2806 
2807 static void
2808 cfiscsi_datamove(union ctl_io *io)
2809 {
2810 
2811 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2812 		cfiscsi_datamove_in(io);
2813 	else {
2814 		/* We hadn't received anything during this datamove yet. */
2815 		io->scsiio.ext_data_filled = 0;
2816 		cfiscsi_datamove_out(io);
2817 	}
2818 }
2819 
2820 static void
2821 cfiscsi_scsi_command_done(union ctl_io *io)
2822 {
2823 	struct icl_pdu *request, *response;
2824 	struct iscsi_bhs_scsi_command *bhssc;
2825 	struct iscsi_bhs_scsi_response *bhssr;
2826 #ifdef DIAGNOSTIC
2827 	struct cfiscsi_data_wait *cdw;
2828 #endif
2829 	struct cfiscsi_session *cs;
2830 	uint16_t sense_length;
2831 
2832 	request = PRIV_REQUEST(io);
2833 	cs = PDU_SESSION(request);
2834 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2835 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2836 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2837 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2838 
2839 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2840 	//    bhssc->bhssc_initiator_task_tag);
2841 
2842 #ifdef DIAGNOSTIC
2843 	CFISCSI_SESSION_LOCK(cs);
2844 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2845 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2846 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2847 	CFISCSI_SESSION_UNLOCK(cs);
2848 #endif
2849 
2850 	/*
2851 	 * Do not return status for aborted commands.
2852 	 * There are exceptions, but none supported by CTL yet.
2853 	 */
2854 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2855 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2856 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2857 		ctl_free_io(io);
2858 		icl_pdu_free(request);
2859 		return;
2860 	}
2861 
2862 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2863 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2864 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2865 	bhssr->bhssr_flags = 0x80;
2866 	/*
2867 	 * XXX: We don't deal with bidirectional under/overflows;
2868 	 *	does anything actually support those?
2869 	 */
2870 	if (io->scsiio.kern_total_len <
2871 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2872 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2873 		bhssr->bhssr_residual_count =
2874 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2875 		    io->scsiio.kern_total_len);
2876 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2877 		//    ntohl(bhssr->bhssr_residual_count));
2878 	} else if (io->scsiio.kern_total_len >
2879 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2880 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2881 		bhssr->bhssr_residual_count = htonl(io->scsiio.kern_total_len -
2882 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2883 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2884 		//    ntohl(bhssr->bhssr_residual_count));
2885 	}
2886 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2887 	bhssr->bhssr_status = io->scsiio.scsi_status;
2888 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2889 	bhssr->bhssr_expdatasn = htonl(PRIV_EXPDATASN(io));
2890 
2891 	if (io->scsiio.sense_len > 0) {
2892 #if 0
2893 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2894 		    io->scsiio.sense_len);
2895 #endif
2896 		sense_length = htons(io->scsiio.sense_len);
2897 		icl_pdu_append_data(response,
2898 		    &sense_length, sizeof(sense_length), M_WAITOK);
2899 		icl_pdu_append_data(response,
2900 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2901 	}
2902 
2903 	ctl_free_io(io);
2904 	icl_pdu_free(request);
2905 	cfiscsi_pdu_queue(response);
2906 }
2907 
2908 static void
2909 cfiscsi_task_management_done(union ctl_io *io)
2910 {
2911 	struct icl_pdu *request, *response;
2912 	struct iscsi_bhs_task_management_request *bhstmr;
2913 	struct iscsi_bhs_task_management_response *bhstmr2;
2914 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2915 	struct cfiscsi_session *cs, *tcs;
2916 	struct cfiscsi_softc *softc;
2917 	int cold_reset = 0;
2918 
2919 	request = PRIV_REQUEST(io);
2920 	cs = PDU_SESSION(request);
2921 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2922 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2923 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2924 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2925 
2926 #if 0
2927 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2928 	    bhstmr->bhstmr_initiator_task_tag,
2929 	    bhstmr->bhstmr_referenced_task_tag);
2930 #endif
2931 
2932 	if ((bhstmr->bhstmr_function & ~0x80) ==
2933 	    BHSTMR_FUNCTION_ABORT_TASK) {
2934 		/*
2935 		 * Make sure we no longer wait for Data-Out for this command.
2936 		 */
2937 		CFISCSI_SESSION_LOCK(cs);
2938 		TAILQ_FOREACH_SAFE(cdw,
2939 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2940 			if (bhstmr->bhstmr_referenced_task_tag !=
2941 			    cdw->cdw_initiator_task_tag)
2942 				continue;
2943 
2944 #if 0
2945 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2946 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2947 #endif
2948 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2949 			    cdw, cdw_next);
2950 			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
2951 			cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
2952 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2953 			cfiscsi_data_wait_free(cs, cdw);
2954 		}
2955 		CFISCSI_SESSION_UNLOCK(cs);
2956 	}
2957 	if ((bhstmr->bhstmr_function & ~0x80) ==
2958 	    BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2959 	    io->io_hdr.status == CTL_SUCCESS)
2960 		cold_reset = 1;
2961 
2962 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2963 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2964 	    response->ip_bhs;
2965 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2966 	bhstmr2->bhstmr_flags = 0x80;
2967 	switch (io->taskio.task_status) {
2968 	case CTL_TASK_FUNCTION_COMPLETE:
2969 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2970 		break;
2971 	case CTL_TASK_FUNCTION_SUCCEEDED:
2972 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
2973 		break;
2974 	case CTL_TASK_LUN_DOES_NOT_EXIST:
2975 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
2976 		break;
2977 	case CTL_TASK_FUNCTION_NOT_SUPPORTED:
2978 	default:
2979 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2980 		break;
2981 	}
2982 	memcpy(bhstmr2->bhstmr_additional_reponse_information,
2983 	    io->taskio.task_resp, sizeof(io->taskio.task_resp));
2984 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2985 
2986 	ctl_free_io(io);
2987 	icl_pdu_free(request);
2988 	cfiscsi_pdu_queue(response);
2989 
2990 	if (cold_reset) {
2991 		softc = cs->cs_target->ct_softc;
2992 		mtx_lock(&softc->lock);
2993 		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
2994 			if (tcs->cs_target == cs->cs_target)
2995 				cfiscsi_session_terminate(tcs);
2996 		}
2997 		mtx_unlock(&softc->lock);
2998 	}
2999 }
3000 
3001 static void
3002 cfiscsi_done(union ctl_io *io)
3003 {
3004 	struct icl_pdu *request;
3005 	struct cfiscsi_session *cs;
3006 
3007 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
3008 		("invalid CTL status %#x", io->io_hdr.status));
3009 
3010 	if (io->io_hdr.io_type == CTL_IO_TASK &&
3011 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
3012 		/*
3013 		 * Implicit task termination has just completed; nothing to do.
3014 		 */
3015 		cs = PRIV_REQUEST(io);
3016 		cs->cs_tasks_aborted = true;
3017 		refcount_release(&cs->cs_outstanding_ctl_pdus);
3018 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3019 		ctl_free_io(io);
3020 		return;
3021 	}
3022 
3023 	request = PRIV_REQUEST(io);
3024 	cs = PDU_SESSION(request);
3025 
3026 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3027 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3028 		cfiscsi_scsi_command_done(io);
3029 		break;
3030 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
3031 		cfiscsi_task_management_done(io);
3032 		break;
3033 	default:
3034 		panic("cfiscsi_done called with wrong opcode 0x%x",
3035 		    request->ip_bhs->bhs_opcode);
3036 	}
3037 
3038 	refcount_release(&cs->cs_outstanding_ctl_pdus);
3039 }
3040