xref: /freebsd/sys/cam/ctl/ctl_frontend_iscsi.c (revision 4f52dfbb8d6c4d446500c5b097e3806ec219fbd4)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 The FreeBSD Foundation
5  * All rights reserved.
6  *
7  * This software was developed by Edward Tomasz Napierala under sponsorship
8  * from the FreeBSD Foundation.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  *
31  * $FreeBSD$
32  */
33 
34 /*
35  * CTL frontend for the iSCSI protocol.
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include <sys/param.h>
42 #include <sys/capsicum.h>
43 #include <sys/condvar.h>
44 #include <sys/endian.h>
45 #include <sys/file.h>
46 #include <sys/kernel.h>
47 #include <sys/kthread.h>
48 #include <sys/lock.h>
49 #include <sys/malloc.h>
50 #include <sys/module.h>
51 #include <sys/mutex.h>
52 #include <sys/queue.h>
53 #include <sys/sbuf.h>
54 #include <sys/socket.h>
55 #include <sys/sysctl.h>
56 #include <sys/systm.h>
57 #include <sys/uio.h>
58 #include <sys/unistd.h>
59 #include <vm/uma.h>
60 
61 #include <cam/scsi/scsi_all.h>
62 #include <cam/scsi/scsi_da.h>
63 #include <cam/ctl/ctl_io.h>
64 #include <cam/ctl/ctl.h>
65 #include <cam/ctl/ctl_backend.h>
66 #include <cam/ctl/ctl_error.h>
67 #include <cam/ctl/ctl_frontend.h>
68 #include <cam/ctl/ctl_debug.h>
69 #include <cam/ctl/ctl_ha.h>
70 #include <cam/ctl/ctl_ioctl.h>
71 #include <cam/ctl/ctl_private.h>
72 
73 #include <dev/iscsi/icl.h>
74 #include <dev/iscsi/icl_wrappers.h>
75 #include <dev/iscsi/iscsi_proto.h>
76 #include <cam/ctl/ctl_frontend_iscsi.h>
77 
78 #ifdef ICL_KERNEL_PROXY
79 #include <sys/socketvar.h>
80 #endif
81 
82 #ifdef ICL_KERNEL_PROXY
83 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
84 #endif
85 
86 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
87 static uma_zone_t cfiscsi_data_wait_zone;
88 
89 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
90     "CAM Target Layer iSCSI Frontend");
91 static int debug = 1;
92 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
93     &debug, 1, "Enable debug messages");
94 static int ping_timeout = 5;
95 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
96     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
97 static int login_timeout = 60;
98 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
99     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
100 static int maxtags = 256;
101 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
102     &maxtags, 0, "Max number of requests queued by initiator");
103 
104 #define	CFISCSI_DEBUG(X, ...)						\
105 	do {								\
106 		if (debug > 1) {					\
107 			printf("%s: " X "\n",				\
108 			    __func__, ## __VA_ARGS__);			\
109 		}							\
110 	} while (0)
111 
112 #define	CFISCSI_WARN(X, ...)						\
113 	do {								\
114 		if (debug > 0) {					\
115 			printf("WARNING: %s: " X "\n",			\
116 			    __func__, ## __VA_ARGS__);			\
117 		}							\
118 	} while (0)
119 
120 #define	CFISCSI_SESSION_DEBUG(S, X, ...)				\
121 	do {								\
122 		if (debug > 1) {					\
123 			printf("%s: %s (%s): " X "\n",			\
124 			    __func__, S->cs_initiator_addr,		\
125 			    S->cs_initiator_name, ## __VA_ARGS__);	\
126 		}							\
127 	} while (0)
128 
129 #define	CFISCSI_SESSION_WARN(S, X, ...)					\
130 	do  {								\
131 		if (debug > 0) {					\
132 			printf("WARNING: %s (%s): " X "\n",		\
133 			    S->cs_initiator_addr,			\
134 			    S->cs_initiator_name, ## __VA_ARGS__);	\
135 		}							\
136 	} while (0)
137 
138 #define CFISCSI_SESSION_LOCK(X)		mtx_lock(&X->cs_lock)
139 #define CFISCSI_SESSION_UNLOCK(X)	mtx_unlock(&X->cs_lock)
140 #define CFISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->cs_lock, MA_OWNED)
141 
142 #define	CONN_SESSION(X)			((struct cfiscsi_session *)(X)->ic_prv0)
143 #define	PDU_SESSION(X)			CONN_SESSION((X)->ip_conn)
144 #define	PDU_EXPDATASN(X)		(X)->ip_prv0
145 #define	PDU_TOTAL_TRANSFER_LEN(X)	(X)->ip_prv1
146 #define	PDU_R2TSN(X)			(X)->ip_prv2
147 
148 static int	cfiscsi_init(void);
149 static int	cfiscsi_shutdown(void);
150 static void	cfiscsi_online(void *arg);
151 static void	cfiscsi_offline(void *arg);
152 static int	cfiscsi_info(void *arg, struct sbuf *sb);
153 static int	cfiscsi_ioctl(struct cdev *dev,
154 		    u_long cmd, caddr_t addr, int flag, struct thread *td);
155 static void	cfiscsi_datamove(union ctl_io *io);
156 static void	cfiscsi_datamove_in(union ctl_io *io);
157 static void	cfiscsi_datamove_out(union ctl_io *io);
158 static void	cfiscsi_done(union ctl_io *io);
159 static bool	cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
160 static void	cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
161 static void	cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
162 static void	cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
163 static void	cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
164 static void	cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
165 static void	cfiscsi_session_terminate(struct cfiscsi_session *cs);
166 static struct cfiscsi_data_wait	*cfiscsi_data_wait_new(
167 		    struct cfiscsi_session *cs, union ctl_io *io,
168 		    uint32_t initiator_task_tag,
169 		    uint32_t *target_transfer_tagp);
170 static void	cfiscsi_data_wait_free(struct cfiscsi_session *cs,
171 		    struct cfiscsi_data_wait *cdw);
172 static struct cfiscsi_target	*cfiscsi_target_find(struct cfiscsi_softc
173 		    *softc, const char *name, uint16_t tag);
174 static struct cfiscsi_target	*cfiscsi_target_find_or_create(
175     struct cfiscsi_softc *softc, const char *name, const char *alias,
176     uint16_t tag);
177 static void	cfiscsi_target_release(struct cfiscsi_target *ct);
178 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
179 
180 static struct cfiscsi_softc cfiscsi_softc;
181 
182 static struct ctl_frontend cfiscsi_frontend =
183 {
184 	.name = "iscsi",
185 	.init = cfiscsi_init,
186 	.ioctl = cfiscsi_ioctl,
187 	.shutdown = cfiscsi_shutdown,
188 };
189 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
190 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
191 
192 static struct icl_pdu *
193 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
194 {
195 
196 	return (icl_pdu_new(request->ip_conn, flags));
197 }
198 
199 static bool
200 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
201 {
202 	const struct iscsi_bhs_scsi_command *bhssc;
203 	struct cfiscsi_session *cs;
204 	uint32_t cmdsn, expstatsn;
205 
206 	cs = PDU_SESSION(request);
207 
208 	/*
209 	 * Every incoming PDU - not just NOP-Out - resets the ping timer.
210 	 * The purpose of the timeout is to reset the connection when it stalls;
211 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
212 	 * in some queue.
213 	 *
214 	 * XXX: Locking?
215 	 */
216 	cs->cs_timeout = 0;
217 
218 	/*
219 	 * Data-Out PDUs don't contain CmdSN.
220 	 */
221 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
222 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
223 		return (false);
224 
225 	/*
226 	 * We're only using fields common for all the request
227 	 * (initiator -> target) PDUs.
228 	 */
229 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
230 	cmdsn = ntohl(bhssc->bhssc_cmdsn);
231 	expstatsn = ntohl(bhssc->bhssc_expstatsn);
232 
233 	CFISCSI_SESSION_LOCK(cs);
234 #if 0
235 	if (expstatsn != cs->cs_statsn) {
236 		CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
237 		    "while current StatSN is %d", expstatsn,
238 		    cs->cs_statsn);
239 	}
240 #endif
241 
242 	if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
243 		/*
244 		 * The target MUST silently ignore any non-immediate command
245 		 * outside of this range.
246 		 */
247 		if (ISCSI_SNLT(cmdsn, cs->cs_cmdsn) ||
248 		    ISCSI_SNGT(cmdsn, cs->cs_cmdsn - 1 + maxtags)) {
249 			CFISCSI_SESSION_UNLOCK(cs);
250 			CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
251 			    "while expected %u", cmdsn, cs->cs_cmdsn);
252 			return (true);
253 		}
254 
255 		/*
256 		 * We don't support multiple connections now, so any
257 		 * discontinuity in CmdSN means lost PDUs.  Since we don't
258 		 * support PDU retransmission -- terminate the connection.
259 		 */
260 		if (cmdsn != cs->cs_cmdsn) {
261 			CFISCSI_SESSION_UNLOCK(cs);
262 			CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
263 			    "while expected %u; dropping connection",
264 			    cmdsn, cs->cs_cmdsn);
265 			cfiscsi_session_terminate(cs);
266 			return (true);
267 		}
268 		cs->cs_cmdsn++;
269 	}
270 
271 	CFISCSI_SESSION_UNLOCK(cs);
272 
273 	return (false);
274 }
275 
276 static void
277 cfiscsi_pdu_handle(struct icl_pdu *request)
278 {
279 	struct cfiscsi_session *cs;
280 	bool ignore;
281 
282 	cs = PDU_SESSION(request);
283 
284 	ignore = cfiscsi_pdu_update_cmdsn(request);
285 	if (ignore) {
286 		icl_pdu_free(request);
287 		return;
288 	}
289 
290 	/*
291 	 * Handle the PDU; this includes e.g. receiving the remaining
292 	 * part of PDU and submitting the SCSI command to CTL
293 	 * or queueing a reply.  The handling routine is responsible
294 	 * for freeing the PDU when it's no longer needed.
295 	 */
296 	switch (request->ip_bhs->bhs_opcode &
297 	    ~ISCSI_BHS_OPCODE_IMMEDIATE) {
298 	case ISCSI_BHS_OPCODE_NOP_OUT:
299 		cfiscsi_pdu_handle_nop_out(request);
300 		break;
301 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
302 		cfiscsi_pdu_handle_scsi_command(request);
303 		break;
304 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
305 		cfiscsi_pdu_handle_task_request(request);
306 		break;
307 	case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
308 		cfiscsi_pdu_handle_data_out(request);
309 		break;
310 	case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
311 		cfiscsi_pdu_handle_logout_request(request);
312 		break;
313 	default:
314 		CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
315 		    "opcode 0x%x; dropping connection",
316 		    request->ip_bhs->bhs_opcode);
317 		icl_pdu_free(request);
318 		cfiscsi_session_terminate(cs);
319 	}
320 
321 }
322 
323 static void
324 cfiscsi_receive_callback(struct icl_pdu *request)
325 {
326 #ifdef ICL_KERNEL_PROXY
327 	struct cfiscsi_session *cs;
328 
329 	cs = PDU_SESSION(request);
330 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
331 		if (cs->cs_login_pdu == NULL)
332 			cs->cs_login_pdu = request;
333 		else
334 			icl_pdu_free(request);
335 		cv_signal(&cs->cs_login_cv);
336 		return;
337 	}
338 #endif
339 
340 	cfiscsi_pdu_handle(request);
341 }
342 
343 static void
344 cfiscsi_error_callback(struct icl_conn *ic)
345 {
346 	struct cfiscsi_session *cs;
347 
348 	cs = CONN_SESSION(ic);
349 
350 	CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
351 	cfiscsi_session_terminate(cs);
352 }
353 
354 static int
355 cfiscsi_pdu_prepare(struct icl_pdu *response)
356 {
357 	struct cfiscsi_session *cs;
358 	struct iscsi_bhs_scsi_response *bhssr;
359 	bool advance_statsn = true;
360 
361 	cs = PDU_SESSION(response);
362 
363 	CFISCSI_SESSION_LOCK_ASSERT(cs);
364 
365 	/*
366 	 * We're only using fields common for all the response
367 	 * (target -> initiator) PDUs.
368 	 */
369 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
370 
371 	/*
372 	 * 10.8.3: "The StatSN for this connection is not advanced
373 	 * after this PDU is sent."
374 	 */
375 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
376 		advance_statsn = false;
377 
378 	/*
379 	 * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
380 	 * StatSN for the connection is not advanced after this PDU is sent."
381 	 */
382 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
383 	    bhssr->bhssr_initiator_task_tag == 0xffffffff)
384 		advance_statsn = false;
385 
386 	/*
387 	 * See the comment below - StatSN is not meaningful and must
388 	 * not be advanced.
389 	 */
390 	if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
391 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
392 		advance_statsn = false;
393 
394 	/*
395 	 * 10.7.3: "The fields StatSN, Status, and Residual Count
396 	 * only have meaningful content if the S bit is set to 1."
397 	 */
398 	if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
399 	    (bhssr->bhssr_flags & BHSDI_FLAGS_S))
400 		bhssr->bhssr_statsn = htonl(cs->cs_statsn);
401 	bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
402 	bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn - 1 +
403 	    imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
404 
405 	if (advance_statsn)
406 		cs->cs_statsn++;
407 
408 	return (0);
409 }
410 
411 static void
412 cfiscsi_pdu_queue(struct icl_pdu *response)
413 {
414 	struct cfiscsi_session *cs;
415 
416 	cs = PDU_SESSION(response);
417 
418 	CFISCSI_SESSION_LOCK(cs);
419 	cfiscsi_pdu_prepare(response);
420 	icl_pdu_queue(response);
421 	CFISCSI_SESSION_UNLOCK(cs);
422 }
423 
424 static void
425 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
426 {
427 	struct cfiscsi_session *cs;
428 	struct iscsi_bhs_nop_out *bhsno;
429 	struct iscsi_bhs_nop_in *bhsni;
430 	struct icl_pdu *response;
431 	void *data = NULL;
432 	size_t datasize;
433 	int error;
434 
435 	cs = PDU_SESSION(request);
436 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
437 
438 	if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
439 		/*
440 		 * Nothing to do, iscsi_pdu_update_statsn() already
441 		 * zeroed the timeout.
442 		 */
443 		icl_pdu_free(request);
444 		return;
445 	}
446 
447 	datasize = icl_pdu_data_segment_length(request);
448 	if (datasize > 0) {
449 		data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
450 		if (data == NULL) {
451 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
452 			    "dropping connection");
453 			icl_pdu_free(request);
454 			cfiscsi_session_terminate(cs);
455 			return;
456 		}
457 		icl_pdu_get_data(request, 0, data, datasize);
458 	}
459 
460 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
461 	if (response == NULL) {
462 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
463 		    "droppping connection");
464 		free(data, M_CFISCSI);
465 		icl_pdu_free(request);
466 		cfiscsi_session_terminate(cs);
467 		return;
468 	}
469 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
470 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
471 	bhsni->bhsni_flags = 0x80;
472 	bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
473 	bhsni->bhsni_target_transfer_tag = 0xffffffff;
474 	if (datasize > 0) {
475 		error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
476 		if (error != 0) {
477 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
478 			    "dropping connection");
479 			free(data, M_CFISCSI);
480 			icl_pdu_free(request);
481 			icl_pdu_free(response);
482 			cfiscsi_session_terminate(cs);
483 			return;
484 		}
485 		free(data, M_CFISCSI);
486 	}
487 
488 	icl_pdu_free(request);
489 	cfiscsi_pdu_queue(response);
490 }
491 
492 static void
493 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
494 {
495 	struct iscsi_bhs_scsi_command *bhssc;
496 	struct cfiscsi_session *cs;
497 	union ctl_io *io;
498 	int error;
499 
500 	cs = PDU_SESSION(request);
501 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
502 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
503 	//    bhssc->bhssc_initiator_task_tag);
504 
505 	if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
506 		CFISCSI_SESSION_WARN(cs, "unsolicited data with "
507 		    "ImmediateData=No; dropping connection");
508 		icl_pdu_free(request);
509 		cfiscsi_session_terminate(cs);
510 		return;
511 	}
512 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
513 	ctl_zero_io(io);
514 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
515 	io->io_hdr.io_type = CTL_IO_SCSI;
516 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
517 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
518 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
519 	io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
520 	switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
521 	case BHSSC_FLAGS_ATTR_UNTAGGED:
522 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
523 		break;
524 	case BHSSC_FLAGS_ATTR_SIMPLE:
525 		io->scsiio.tag_type = CTL_TAG_SIMPLE;
526 		break;
527 	case BHSSC_FLAGS_ATTR_ORDERED:
528         	io->scsiio.tag_type = CTL_TAG_ORDERED;
529 		break;
530 	case BHSSC_FLAGS_ATTR_HOQ:
531         	io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
532 		break;
533 	case BHSSC_FLAGS_ATTR_ACA:
534 		io->scsiio.tag_type = CTL_TAG_ACA;
535 		break;
536 	default:
537 		io->scsiio.tag_type = CTL_TAG_UNTAGGED;
538 		CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
539 		    bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
540 		break;
541 	}
542 	io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
543 	memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
544 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
545 	error = ctl_queue(io);
546 	if (error != CTL_RETVAL_COMPLETE) {
547 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
548 		    "dropping connection", error);
549 		ctl_free_io(io);
550 		refcount_release(&cs->cs_outstanding_ctl_pdus);
551 		icl_pdu_free(request);
552 		cfiscsi_session_terminate(cs);
553 	}
554 }
555 
556 static void
557 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
558 {
559 	struct iscsi_bhs_task_management_request *bhstmr;
560 	struct iscsi_bhs_task_management_response *bhstmr2;
561 	struct icl_pdu *response;
562 	struct cfiscsi_session *cs;
563 	union ctl_io *io;
564 	int error;
565 
566 	cs = PDU_SESSION(request);
567 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
568 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
569 	ctl_zero_io(io);
570 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
571 	io->io_hdr.io_type = CTL_IO_TASK;
572 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
573 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
574 	io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
575 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
576 
577 	switch (bhstmr->bhstmr_function & ~0x80) {
578 	case BHSTMR_FUNCTION_ABORT_TASK:
579 #if 0
580 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
581 #endif
582 		io->taskio.task_action = CTL_TASK_ABORT_TASK;
583 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
584 		break;
585 	case BHSTMR_FUNCTION_ABORT_TASK_SET:
586 #if 0
587 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
588 #endif
589 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
590 		break;
591 	case BHSTMR_FUNCTION_CLEAR_TASK_SET:
592 #if 0
593 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
594 #endif
595 		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
596 		break;
597 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
598 #if 0
599 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
600 #endif
601 		io->taskio.task_action = CTL_TASK_LUN_RESET;
602 		break;
603 	case BHSTMR_FUNCTION_TARGET_WARM_RESET:
604 #if 0
605 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
606 #endif
607 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
608 		break;
609 	case BHSTMR_FUNCTION_TARGET_COLD_RESET:
610 #if 0
611 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
612 #endif
613 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
614 		break;
615 	case BHSTMR_FUNCTION_QUERY_TASK:
616 #if 0
617 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
618 #endif
619 		io->taskio.task_action = CTL_TASK_QUERY_TASK;
620 		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
621 		break;
622 	case BHSTMR_FUNCTION_QUERY_TASK_SET:
623 #if 0
624 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
625 #endif
626 		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
627 		break;
628 	case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
629 #if 0
630 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
631 #endif
632 		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
633 		break;
634 	case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
635 #if 0
636 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
637 #endif
638 		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
639 		break;
640 	default:
641 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
642 		    bhstmr->bhstmr_function & ~0x80);
643 		ctl_free_io(io);
644 
645 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
646 		if (response == NULL) {
647 			CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
648 			    "dropping connection");
649 			icl_pdu_free(request);
650 			cfiscsi_session_terminate(cs);
651 			return;
652 		}
653 		bhstmr2 = (struct iscsi_bhs_task_management_response *)
654 		    response->ip_bhs;
655 		bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
656 		bhstmr2->bhstmr_flags = 0x80;
657 		bhstmr2->bhstmr_response =
658 		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
659 		bhstmr2->bhstmr_initiator_task_tag =
660 		    bhstmr->bhstmr_initiator_task_tag;
661 		icl_pdu_free(request);
662 		cfiscsi_pdu_queue(response);
663 		return;
664 	}
665 
666 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
667 	error = ctl_queue(io);
668 	if (error != CTL_RETVAL_COMPLETE) {
669 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
670 		    "dropping connection", error);
671 		ctl_free_io(io);
672 		refcount_release(&cs->cs_outstanding_ctl_pdus);
673 		icl_pdu_free(request);
674 		cfiscsi_session_terminate(cs);
675 	}
676 }
677 
678 static bool
679 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
680 {
681 	struct iscsi_bhs_data_out *bhsdo;
682 	struct cfiscsi_session *cs;
683 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
684 	size_t copy_len, len, off, buffer_offset;
685 	int ctl_sg_count;
686 	union ctl_io *io;
687 
688 	cs = PDU_SESSION(request);
689 
690 	KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
691 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
692 	    (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
693 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
694 	    ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
695 
696 	/*
697 	 * We're only using fields common for Data-Out and SCSI Command PDUs.
698 	 */
699 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
700 
701 	io = cdw->cdw_ctl_io;
702 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
703 	    ("CTL_FLAG_DATA_IN"));
704 
705 #if 0
706 	CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
707 	    request->ip_data_len, io->scsiio.kern_total_len);
708 #endif
709 
710 	if (io->scsiio.kern_sg_entries > 0) {
711 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
712 		ctl_sg_count = io->scsiio.kern_sg_entries;
713 	} else {
714 		ctl_sglist = &ctl_sg_entry;
715 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
716 		ctl_sglist->len = io->scsiio.kern_data_len;
717 		ctl_sg_count = 1;
718 	}
719 
720 	if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
721 	    ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
722 		buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
723 	else
724 		buffer_offset = 0;
725 	len = icl_pdu_data_segment_length(request);
726 
727 	/*
728 	 * Make sure the offset, as sent by the initiator, matches the offset
729 	 * we're supposed to be at in the scatter-gather list.
730 	 */
731 	if (buffer_offset >
732 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
733 	    buffer_offset + len <=
734 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
735 		CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
736 		    "expected %zd; dropping connection", buffer_offset,
737 		    (size_t)io->scsiio.kern_rel_offset +
738 		    (size_t)io->scsiio.ext_data_filled);
739 		ctl_set_data_phase_error(&io->scsiio);
740 		cfiscsi_session_terminate(cs);
741 		return (true);
742 	}
743 
744 	/*
745 	 * This is the offset within the PDU data segment, as opposed
746 	 * to buffer_offset, which is the offset within the task (SCSI
747 	 * command).
748 	 */
749 	off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
750 	    buffer_offset;
751 
752 	/*
753 	 * Iterate over the scatter/gather segments, filling them with data
754 	 * from the PDU data segment.  Note that this can get called multiple
755 	 * times for one SCSI command; the cdw structure holds state for the
756 	 * scatter/gather list.
757 	 */
758 	for (;;) {
759 		KASSERT(cdw->cdw_sg_index < ctl_sg_count,
760 		    ("cdw->cdw_sg_index >= ctl_sg_count"));
761 		if (cdw->cdw_sg_len == 0) {
762 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
763 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
764 		}
765 		KASSERT(off <= len, ("len > off"));
766 		copy_len = len - off;
767 		if (copy_len > cdw->cdw_sg_len)
768 			copy_len = cdw->cdw_sg_len;
769 
770 		icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
771 		cdw->cdw_sg_addr += copy_len;
772 		cdw->cdw_sg_len -= copy_len;
773 		off += copy_len;
774 		io->scsiio.ext_data_filled += copy_len;
775 		io->scsiio.kern_data_resid -= copy_len;
776 
777 		if (cdw->cdw_sg_len == 0) {
778 			/*
779 			 * End of current segment.
780 			 */
781 			if (cdw->cdw_sg_index == ctl_sg_count - 1) {
782 				/*
783 				 * Last segment in scatter/gather list.
784 				 */
785 				break;
786 			}
787 			cdw->cdw_sg_index++;
788 		}
789 
790 		if (off == len) {
791 			/*
792 			 * End of PDU payload.
793 			 */
794 			break;
795 		}
796 	}
797 
798 	if (len > off) {
799 		/*
800 		 * In case of unsolicited data, it's possible that the buffer
801 		 * provided by CTL is smaller than negotiated FirstBurstLength.
802 		 * Just ignore the superfluous data; will ask for them with R2T
803 		 * on next call to cfiscsi_datamove().
804 		 *
805 		 * This obviously can only happen with SCSI Command PDU.
806 		 */
807 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
808 		    ISCSI_BHS_OPCODE_SCSI_COMMAND)
809 			return (true);
810 
811 		CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
812 		    "expected %zd; dropping connection",
813 		    icl_pdu_data_segment_length(request), off);
814 		ctl_set_data_phase_error(&io->scsiio);
815 		cfiscsi_session_terminate(cs);
816 		return (true);
817 	}
818 
819 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
820 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
821 		CFISCSI_SESSION_WARN(cs, "got the final packet without "
822 		    "the F flag; flags = 0x%x; dropping connection",
823 		    bhsdo->bhsdo_flags);
824 		ctl_set_data_phase_error(&io->scsiio);
825 		cfiscsi_session_terminate(cs);
826 		return (true);
827 	}
828 
829 	if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
830 	    (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
831 		if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
832 		    ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
833 			CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
834 			    "transmitted size was %zd bytes instead of %d; "
835 			    "dropping connection",
836 			    (size_t)io->scsiio.ext_data_filled,
837 			    cdw->cdw_r2t_end);
838 			ctl_set_data_phase_error(&io->scsiio);
839 			cfiscsi_session_terminate(cs);
840 			return (true);
841 		} else {
842 			/*
843 			 * For SCSI Command PDU, this just means we need to
844 			 * solicit more data by sending R2T.
845 			 */
846 			return (false);
847 		}
848 	}
849 
850 	if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
851 #if 0
852 		CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
853 		    "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
854 #endif
855 
856 		return (true);
857 	}
858 
859 	return (false);
860 }
861 
862 static void
863 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
864 {
865 	struct iscsi_bhs_data_out *bhsdo;
866 	struct cfiscsi_session *cs;
867 	struct cfiscsi_data_wait *cdw = NULL;
868 	union ctl_io *io;
869 	bool done;
870 
871 	cs = PDU_SESSION(request);
872 	bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
873 
874 	CFISCSI_SESSION_LOCK(cs);
875 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
876 #if 0
877 		CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
878 		    "ttt 0x%x, itt 0x%x",
879 		    bhsdo->bhsdo_target_transfer_tag,
880 		    bhsdo->bhsdo_initiator_task_tag,
881 		    cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
882 #endif
883 		if (bhsdo->bhsdo_target_transfer_tag ==
884 		    cdw->cdw_target_transfer_tag)
885 			break;
886 	}
887 	CFISCSI_SESSION_UNLOCK(cs);
888 	if (cdw == NULL) {
889 		CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
890 		    "0x%x, not found; dropping connection",
891 		    bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
892 		icl_pdu_free(request);
893 		cfiscsi_session_terminate(cs);
894 		return;
895 	}
896 
897 	if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
898 		CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
899 		    "DataSN %u, while expected %u; dropping connection",
900 		    ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
901 		icl_pdu_free(request);
902 		cfiscsi_session_terminate(cs);
903 		return;
904 	}
905 	cdw->cdw_datasn++;
906 
907 	io = cdw->cdw_ctl_io;
908 	KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
909 	    ("CTL_FLAG_DATA_IN"));
910 
911 	done = cfiscsi_handle_data_segment(request, cdw);
912 	if (done) {
913 		CFISCSI_SESSION_LOCK(cs);
914 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
915 		CFISCSI_SESSION_UNLOCK(cs);
916 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
917 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
918 		cfiscsi_data_wait_free(cs, cdw);
919 		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
920 		if (done)
921 			io->scsiio.be_move_done(io);
922 		else
923 			cfiscsi_datamove_out(io);
924 	}
925 
926 	icl_pdu_free(request);
927 }
928 
929 static void
930 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
931 {
932 	struct iscsi_bhs_logout_request *bhslr;
933 	struct iscsi_bhs_logout_response *bhslr2;
934 	struct icl_pdu *response;
935 	struct cfiscsi_session *cs;
936 
937 	cs = PDU_SESSION(request);
938 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
939 	switch (bhslr->bhslr_reason & 0x7f) {
940 	case BHSLR_REASON_CLOSE_SESSION:
941 	case BHSLR_REASON_CLOSE_CONNECTION:
942 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
943 		if (response == NULL) {
944 			CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
945 			icl_pdu_free(request);
946 			cfiscsi_session_terminate(cs);
947 			return;
948 		}
949 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
950 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
951 		bhslr2->bhslr_flags = 0x80;
952 		bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
953 		bhslr2->bhslr_initiator_task_tag =
954 		    bhslr->bhslr_initiator_task_tag;
955 		icl_pdu_free(request);
956 		cfiscsi_pdu_queue(response);
957 		cfiscsi_session_terminate(cs);
958 		break;
959 	case BHSLR_REASON_REMOVE_FOR_RECOVERY:
960 		response = cfiscsi_pdu_new_response(request, M_NOWAIT);
961 		if (response == NULL) {
962 			CFISCSI_SESSION_WARN(cs,
963 			    "failed to allocate memory; dropping connection");
964 			icl_pdu_free(request);
965 			cfiscsi_session_terminate(cs);
966 			return;
967 		}
968 		bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
969 		bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
970 		bhslr2->bhslr_flags = 0x80;
971 		bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
972 		bhslr2->bhslr_initiator_task_tag =
973 		    bhslr->bhslr_initiator_task_tag;
974 		icl_pdu_free(request);
975 		cfiscsi_pdu_queue(response);
976 		break;
977 	default:
978 		CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
979 		    bhslr->bhslr_reason);
980 		icl_pdu_free(request);
981 		cfiscsi_session_terminate(cs);
982 		break;
983 	}
984 }
985 
986 static void
987 cfiscsi_callout(void *context)
988 {
989 	struct icl_pdu *cp;
990 	struct iscsi_bhs_nop_in *bhsni;
991 	struct cfiscsi_session *cs;
992 
993 	cs = context;
994 
995 	if (cs->cs_terminating)
996 		return;
997 
998 	callout_schedule(&cs->cs_callout, 1 * hz);
999 
1000 	atomic_add_int(&cs->cs_timeout, 1);
1001 
1002 #ifdef ICL_KERNEL_PROXY
1003 	if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1004 		if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1005 			CFISCSI_SESSION_WARN(cs, "login timed out after "
1006 			    "%d seconds; dropping connection", cs->cs_timeout);
1007 			cfiscsi_session_terminate(cs);
1008 		}
1009 		return;
1010 	}
1011 #endif
1012 
1013 	if (ping_timeout <= 0) {
1014 		/*
1015 		 * Pings are disabled.  Don't send NOP-In in this case;
1016 		 * user might have disabled pings to work around problems
1017 		 * with certain initiators that can't properly handle
1018 		 * NOP-In, such as iPXE.  Reset the timeout, to avoid
1019 		 * triggering reconnection, should the user decide to
1020 		 * reenable them.
1021 		 */
1022 		cs->cs_timeout = 0;
1023 		return;
1024 	}
1025 
1026 	if (cs->cs_timeout >= ping_timeout) {
1027 		CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1028 		    "dropping connection",  ping_timeout);
1029 		cfiscsi_session_terminate(cs);
1030 		return;
1031 	}
1032 
1033 	/*
1034 	 * If the ping was reset less than one second ago - which means
1035 	 * that we've received some PDU during the last second - assume
1036 	 * the traffic flows correctly and don't bother sending a NOP-Out.
1037 	 *
1038 	 * (It's 2 - one for one second, and one for incrementing is_timeout
1039 	 * earlier in this routine.)
1040 	 */
1041 	if (cs->cs_timeout < 2)
1042 		return;
1043 
1044 	cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1045 	if (cp == NULL) {
1046 		CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1047 		return;
1048 	}
1049 	bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1050 	bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1051 	bhsni->bhsni_flags = 0x80;
1052 	bhsni->bhsni_initiator_task_tag = 0xffffffff;
1053 
1054 	cfiscsi_pdu_queue(cp);
1055 }
1056 
1057 static struct cfiscsi_data_wait *
1058 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1059     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1060 {
1061 	struct cfiscsi_data_wait *cdw;
1062 	int error;
1063 
1064 	cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1065 	if (cdw == NULL) {
1066 		CFISCSI_SESSION_WARN(cs,
1067 		    "failed to allocate %zd bytes", sizeof(*cdw));
1068 		return (NULL);
1069 	}
1070 
1071 	error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1072 	    &cdw->cdw_icl_prv);
1073 	if (error != 0) {
1074 		CFISCSI_SESSION_WARN(cs,
1075 		    "icl_conn_transfer_setup() failed with error %d", error);
1076 		uma_zfree(cfiscsi_data_wait_zone, cdw);
1077 		return (NULL);
1078 	}
1079 
1080 	cdw->cdw_ctl_io = io;
1081 	cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1082 	cdw->cdw_initiator_task_tag = initiator_task_tag;
1083 
1084 	return (cdw);
1085 }
1086 
1087 static void
1088 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1089     struct cfiscsi_data_wait *cdw)
1090 {
1091 
1092 	icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1093 	uma_zfree(cfiscsi_data_wait_zone, cdw);
1094 }
1095 
1096 static void
1097 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1098 {
1099 	struct cfiscsi_data_wait *cdw;
1100 	union ctl_io *io;
1101 	int error, last, wait;
1102 
1103 	if (cs->cs_target == NULL)
1104 		return;		/* No target yet, so nothing to do. */
1105 	io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1106 	ctl_zero_io(io);
1107 	io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs;
1108 	io->io_hdr.io_type = CTL_IO_TASK;
1109 	io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1110 	io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1111 	io->io_hdr.nexus.targ_lun = 0;
1112 	io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1113 	io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1114 	wait = cs->cs_outstanding_ctl_pdus;
1115 	refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1116 	error = ctl_queue(io);
1117 	if (error != CTL_RETVAL_COMPLETE) {
1118 		CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1119 		refcount_release(&cs->cs_outstanding_ctl_pdus);
1120 		ctl_free_io(io);
1121 	}
1122 
1123 	CFISCSI_SESSION_LOCK(cs);
1124 	while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1125 		TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1126 		CFISCSI_SESSION_UNLOCK(cs);
1127 		/*
1128 		 * Set nonzero port status; this prevents backends from
1129 		 * assuming that the data transfer actually succeeded
1130 		 * and writing uninitialized data to disk.
1131 		 */
1132 		cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1133 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1134 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1135 		cfiscsi_data_wait_free(cs, cdw);
1136 		CFISCSI_SESSION_LOCK(cs);
1137 	}
1138 	CFISCSI_SESSION_UNLOCK(cs);
1139 
1140 	/*
1141 	 * Wait for CTL to terminate all the tasks.
1142 	 */
1143 	if (wait > 0)
1144 		CFISCSI_SESSION_WARN(cs,
1145 		    "waiting for CTL to terminate %d tasks", wait);
1146 	for (;;) {
1147 		refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1148 		last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1149 		if (last != 0)
1150 			break;
1151 		tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1152 		    0, "cfiscsi_terminate", hz / 100);
1153 	}
1154 	if (wait > 0)
1155 		CFISCSI_SESSION_WARN(cs, "tasks terminated");
1156 }
1157 
1158 static void
1159 cfiscsi_maintenance_thread(void *arg)
1160 {
1161 	struct cfiscsi_session *cs;
1162 
1163 	cs = arg;
1164 
1165 	for (;;) {
1166 		CFISCSI_SESSION_LOCK(cs);
1167 		if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1168 			cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1169 		CFISCSI_SESSION_UNLOCK(cs);
1170 
1171 		if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1172 
1173 			/*
1174 			 * We used to wait up to 30 seconds to deliver queued
1175 			 * PDUs to the initiator.  We also tried hard to deliver
1176 			 * SCSI Responses for the aborted PDUs.  We don't do
1177 			 * that anymore.  We might need to revisit that.
1178 			 */
1179 			callout_drain(&cs->cs_callout);
1180 			icl_conn_close(cs->cs_conn);
1181 
1182 			/*
1183 			 * At this point ICL receive thread is no longer
1184 			 * running; no new tasks can be queued.
1185 			 */
1186 			cfiscsi_session_terminate_tasks(cs);
1187 			cfiscsi_session_delete(cs);
1188 			kthread_exit();
1189 			return;
1190 		}
1191 		CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1192 	}
1193 }
1194 
1195 static void
1196 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1197 {
1198 
1199 	cs->cs_terminating = true;
1200 	cv_signal(&cs->cs_maintenance_cv);
1201 #ifdef ICL_KERNEL_PROXY
1202 	cv_signal(&cs->cs_login_cv);
1203 #endif
1204 }
1205 
1206 static int
1207 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1208 {
1209 	struct cfiscsi_target *ct;
1210 	char *name;
1211 	int i;
1212 
1213 	KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1214 
1215 	ct = cs->cs_target;
1216 	name = strdup(cs->cs_initiator_id, M_CTL);
1217 	i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1218 	if (i < 0) {
1219 		CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1220 		    i);
1221 		cs->cs_ctl_initid = -1;
1222 		return (1);
1223 	}
1224 	cs->cs_ctl_initid = i;
1225 #if 0
1226 	CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1227 #endif
1228 
1229 	return (0);
1230 }
1231 
1232 static void
1233 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1234 {
1235 	int error;
1236 
1237 	if (cs->cs_ctl_initid == -1)
1238 		return;
1239 
1240 	error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1241 	if (error != 0) {
1242 		CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1243 		    error);
1244 	}
1245 	cs->cs_ctl_initid = -1;
1246 }
1247 
1248 static struct cfiscsi_session *
1249 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1250 {
1251 	struct cfiscsi_session *cs;
1252 	int error;
1253 
1254 	cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1255 	if (cs == NULL) {
1256 		CFISCSI_WARN("malloc failed");
1257 		return (NULL);
1258 	}
1259 	cs->cs_ctl_initid = -1;
1260 
1261 	refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1262 	TAILQ_INIT(&cs->cs_waiting_for_data_out);
1263 	mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1264 	cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1265 #ifdef ICL_KERNEL_PROXY
1266 	cv_init(&cs->cs_login_cv, "cfiscsi_login");
1267 #endif
1268 
1269 	/*
1270 	 * The purpose of this is to avoid racing with session shutdown.
1271 	 * Otherwise we could have the maintenance thread call icl_conn_close()
1272 	 * before we call icl_conn_handoff().
1273 	 */
1274 	cs->cs_handoff_in_progress = true;
1275 
1276 	cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1277 	if (cs->cs_conn == NULL) {
1278 		free(cs, M_CFISCSI);
1279 		return (NULL);
1280 	}
1281 	cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1282 	cs->cs_conn->ic_error = cfiscsi_error_callback;
1283 	cs->cs_conn->ic_prv0 = cs;
1284 
1285 	error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1286 	if (error != 0) {
1287 		CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1288 		free(cs, M_CFISCSI);
1289 		return (NULL);
1290 	}
1291 
1292 	mtx_lock(&softc->lock);
1293 	cs->cs_id = ++softc->last_session_id;
1294 	TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1295 	mtx_unlock(&softc->lock);
1296 
1297 	/*
1298 	 * Start pinging the initiator.
1299 	 */
1300 	callout_init(&cs->cs_callout, 1);
1301 	callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1302 
1303 	return (cs);
1304 }
1305 
1306 static void
1307 cfiscsi_session_delete(struct cfiscsi_session *cs)
1308 {
1309 	struct cfiscsi_softc *softc;
1310 
1311 	softc = &cfiscsi_softc;
1312 
1313 	KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1314 	    ("destroying session with outstanding CTL pdus"));
1315 	KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1316 	    ("destroying session with non-empty queue"));
1317 
1318 	mtx_lock(&softc->lock);
1319 	TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1320 	mtx_unlock(&softc->lock);
1321 
1322 	cfiscsi_session_unregister_initiator(cs);
1323 	if (cs->cs_target != NULL)
1324 		cfiscsi_target_release(cs->cs_target);
1325 	icl_conn_close(cs->cs_conn);
1326 	icl_conn_free(cs->cs_conn);
1327 	free(cs, M_CFISCSI);
1328 	cv_signal(&softc->sessions_cv);
1329 }
1330 
1331 static int
1332 cfiscsi_init(void)
1333 {
1334 	struct cfiscsi_softc *softc;
1335 
1336 	softc = &cfiscsi_softc;
1337 	bzero(softc, sizeof(*softc));
1338 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1339 
1340 	cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1341 #ifdef ICL_KERNEL_PROXY
1342 	cv_init(&softc->accept_cv, "cfiscsi_accept");
1343 #endif
1344 	TAILQ_INIT(&softc->sessions);
1345 	TAILQ_INIT(&softc->targets);
1346 
1347 	cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1348 	    sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1349 	    UMA_ALIGN_PTR, 0);
1350 
1351 	return (0);
1352 }
1353 
1354 static int
1355 cfiscsi_shutdown(void)
1356 {
1357 	struct cfiscsi_softc *softc = &cfiscsi_softc;
1358 
1359 	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1360 		return (EBUSY);
1361 
1362 	uma_zdestroy(cfiscsi_data_wait_zone);
1363 #ifdef ICL_KERNEL_PROXY
1364 	cv_destroy(&softc->accept_cv);
1365 #endif
1366 	cv_destroy(&softc->sessions_cv);
1367 	mtx_destroy(&softc->lock);
1368 	return (0);
1369 }
1370 
1371 #ifdef ICL_KERNEL_PROXY
1372 static void
1373 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1374 {
1375 	struct cfiscsi_session *cs;
1376 
1377 	cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1378 	if (cs == NULL) {
1379 		CFISCSI_WARN("failed to create session");
1380 		return;
1381 	}
1382 
1383 	icl_conn_handoff_sock(cs->cs_conn, so);
1384 	cs->cs_initiator_sa = sa;
1385 	cs->cs_portal_id = portal_id;
1386 	cs->cs_handoff_in_progress = false;
1387 	cs->cs_waiting_for_ctld = true;
1388 	cv_signal(&cfiscsi_softc.accept_cv);
1389 
1390 	CFISCSI_SESSION_LOCK(cs);
1391 	/*
1392 	 * Wake up the maintenance thread if we got scheduled for termination
1393 	 * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1394 	 */
1395 	if (cs->cs_terminating)
1396 		cfiscsi_session_terminate(cs);
1397 	CFISCSI_SESSION_UNLOCK(cs);
1398 }
1399 #endif
1400 
1401 static void
1402 cfiscsi_online(void *arg)
1403 {
1404 	struct cfiscsi_softc *softc;
1405 	struct cfiscsi_target *ct;
1406 	int online;
1407 
1408 	ct = (struct cfiscsi_target *)arg;
1409 	softc = ct->ct_softc;
1410 
1411 	mtx_lock(&softc->lock);
1412 	if (ct->ct_online) {
1413 		mtx_unlock(&softc->lock);
1414 		return;
1415 	}
1416 	ct->ct_online = 1;
1417 	online = softc->online++;
1418 	mtx_unlock(&softc->lock);
1419 	if (online > 0)
1420 		return;
1421 
1422 #ifdef ICL_KERNEL_PROXY
1423 	if (softc->listener != NULL)
1424 		icl_listen_free(softc->listener);
1425 	softc->listener = icl_listen_new(cfiscsi_accept);
1426 #endif
1427 }
1428 
1429 static void
1430 cfiscsi_offline(void *arg)
1431 {
1432 	struct cfiscsi_softc *softc;
1433 	struct cfiscsi_target *ct;
1434 	struct cfiscsi_session *cs;
1435 	int online;
1436 
1437 	ct = (struct cfiscsi_target *)arg;
1438 	softc = ct->ct_softc;
1439 
1440 	mtx_lock(&softc->lock);
1441 	if (!ct->ct_online) {
1442 		mtx_unlock(&softc->lock);
1443 		return;
1444 	}
1445 	ct->ct_online = 0;
1446 	online = --softc->online;
1447 
1448 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1449 		if (cs->cs_target == ct)
1450 			cfiscsi_session_terminate(cs);
1451 	}
1452 	do {
1453 		TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1454 			if (cs->cs_target == ct)
1455 				break;
1456 		}
1457 		if (cs != NULL)
1458 			cv_wait(&softc->sessions_cv, &softc->lock);
1459 	} while (cs != NULL && ct->ct_online == 0);
1460 	mtx_unlock(&softc->lock);
1461 	if (online > 0)
1462 		return;
1463 
1464 #ifdef ICL_KERNEL_PROXY
1465 	icl_listen_free(softc->listener);
1466 	softc->listener = NULL;
1467 #endif
1468 }
1469 
1470 static int
1471 cfiscsi_info(void *arg, struct sbuf *sb)
1472 {
1473 	struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1474 	int retval;
1475 
1476 	retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1477 	    ct->ct_state);
1478 	return (retval);
1479 }
1480 
1481 static void
1482 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1483 {
1484 	struct cfiscsi_softc *softc;
1485 	struct cfiscsi_session *cs, *cs2;
1486 	struct cfiscsi_target *ct;
1487 	struct ctl_iscsi_handoff_params *cihp;
1488 	int error;
1489 
1490 	cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1491 	softc = &cfiscsi_softc;
1492 
1493 	CFISCSI_DEBUG("new connection from %s (%s) to %s",
1494 	    cihp->initiator_name, cihp->initiator_addr,
1495 	    cihp->target_name);
1496 
1497 	ct = cfiscsi_target_find(softc, cihp->target_name,
1498 	    cihp->portal_group_tag);
1499 	if (ct == NULL) {
1500 		ci->status = CTL_ISCSI_ERROR;
1501 		snprintf(ci->error_str, sizeof(ci->error_str),
1502 		    "%s: target not found", __func__);
1503 		return;
1504 	}
1505 
1506 #ifdef ICL_KERNEL_PROXY
1507 	if (cihp->socket > 0 && cihp->connection_id > 0) {
1508 		snprintf(ci->error_str, sizeof(ci->error_str),
1509 		    "both socket and connection_id set");
1510 		ci->status = CTL_ISCSI_ERROR;
1511 		cfiscsi_target_release(ct);
1512 		return;
1513 	}
1514 	if (cihp->socket == 0) {
1515 		mtx_lock(&cfiscsi_softc.lock);
1516 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1517 			if (cs->cs_id == cihp->connection_id)
1518 				break;
1519 		}
1520 		if (cs == NULL) {
1521 			mtx_unlock(&cfiscsi_softc.lock);
1522 			snprintf(ci->error_str, sizeof(ci->error_str),
1523 			    "connection not found");
1524 			ci->status = CTL_ISCSI_ERROR;
1525 			cfiscsi_target_release(ct);
1526 			return;
1527 		}
1528 		mtx_unlock(&cfiscsi_softc.lock);
1529 	} else {
1530 #endif
1531 		cs = cfiscsi_session_new(softc, cihp->offload);
1532 		if (cs == NULL) {
1533 			ci->status = CTL_ISCSI_ERROR;
1534 			snprintf(ci->error_str, sizeof(ci->error_str),
1535 			    "%s: cfiscsi_session_new failed", __func__);
1536 			cfiscsi_target_release(ct);
1537 			return;
1538 		}
1539 #ifdef ICL_KERNEL_PROXY
1540 	}
1541 #endif
1542 
1543 	/*
1544 	 * First PDU of Full Feature phase has the same CmdSN as the last
1545 	 * PDU from the Login Phase received from the initiator.  Thus,
1546 	 * the -1 below.
1547 	 */
1548 	cs->cs_cmdsn = cihp->cmdsn;
1549 	cs->cs_statsn = cihp->statsn;
1550 	cs->cs_max_recv_data_segment_length = cihp->max_recv_data_segment_length;
1551 	cs->cs_max_send_data_segment_length = cihp->max_send_data_segment_length;
1552 	cs->cs_max_burst_length = cihp->max_burst_length;
1553 	cs->cs_first_burst_length = cihp->first_burst_length;
1554 	cs->cs_immediate_data = !!cihp->immediate_data;
1555 	if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1556 		cs->cs_conn->ic_header_crc32c = true;
1557 	if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1558 		cs->cs_conn->ic_data_crc32c = true;
1559 
1560 	strlcpy(cs->cs_initiator_name,
1561 	    cihp->initiator_name, sizeof(cs->cs_initiator_name));
1562 	strlcpy(cs->cs_initiator_addr,
1563 	    cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1564 	strlcpy(cs->cs_initiator_alias,
1565 	    cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1566 	memcpy(cs->cs_initiator_isid,
1567 	    cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1568 	snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1569 	    "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1570 	    cihp->initiator_isid[0], cihp->initiator_isid[1],
1571 	    cihp->initiator_isid[2], cihp->initiator_isid[3],
1572 	    cihp->initiator_isid[4], cihp->initiator_isid[5]);
1573 
1574 	mtx_lock(&softc->lock);
1575 	if (ct->ct_online == 0) {
1576 		mtx_unlock(&softc->lock);
1577 		cs->cs_handoff_in_progress = false;
1578 		cfiscsi_session_terminate(cs);
1579 		cfiscsi_target_release(ct);
1580 		ci->status = CTL_ISCSI_ERROR;
1581 		snprintf(ci->error_str, sizeof(ci->error_str),
1582 		    "%s: port offline", __func__);
1583 		return;
1584 	}
1585 	cs->cs_target = ct;
1586 	mtx_unlock(&softc->lock);
1587 
1588 restart:
1589 	if (!cs->cs_terminating) {
1590 		mtx_lock(&softc->lock);
1591 		TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1592 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1593 			    cs->cs_target == cs2->cs_target &&
1594 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1595 				if (strcmp(cs->cs_initiator_addr,
1596 				    cs2->cs_initiator_addr) != 0) {
1597 					CFISCSI_SESSION_WARN(cs2,
1598 					    "session reinstatement from "
1599 					    "different address %s",
1600 					    cs->cs_initiator_addr);
1601 				} else {
1602 					CFISCSI_SESSION_DEBUG(cs2,
1603 					    "session reinstatement");
1604 				}
1605 				cfiscsi_session_terminate(cs2);
1606 				mtx_unlock(&softc->lock);
1607 				pause("cfiscsi_reinstate", 1);
1608 				goto restart;
1609 			}
1610 		}
1611 		mtx_unlock(&softc->lock);
1612 	}
1613 
1614 	/*
1615 	 * Register initiator with CTL.
1616 	 */
1617 	cfiscsi_session_register_initiator(cs);
1618 
1619 #ifdef ICL_KERNEL_PROXY
1620 	if (cihp->socket > 0) {
1621 #endif
1622 		error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1623 		if (error != 0) {
1624 			cs->cs_handoff_in_progress = false;
1625 			cfiscsi_session_terminate(cs);
1626 			ci->status = CTL_ISCSI_ERROR;
1627 			snprintf(ci->error_str, sizeof(ci->error_str),
1628 			    "%s: icl_conn_handoff failed with error %d",
1629 			    __func__, error);
1630 			return;
1631 		}
1632 #ifdef ICL_KERNEL_PROXY
1633 	}
1634 #endif
1635 
1636 #ifdef ICL_KERNEL_PROXY
1637 	cs->cs_login_phase = false;
1638 
1639 	/*
1640 	 * First PDU of the Full Feature phase has likely already arrived.
1641 	 * We have to pick it up and execute properly.
1642 	 */
1643 	if (cs->cs_login_pdu != NULL) {
1644 		CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1645 		cfiscsi_pdu_handle(cs->cs_login_pdu);
1646 		cs->cs_login_pdu = NULL;
1647 	}
1648 #endif
1649 
1650 	CFISCSI_SESSION_LOCK(cs);
1651 	cs->cs_handoff_in_progress = false;
1652 
1653 	/*
1654 	 * Wake up the maintenance thread if we got scheduled for termination.
1655 	 */
1656 	if (cs->cs_terminating)
1657 		cfiscsi_session_terminate(cs);
1658 	CFISCSI_SESSION_UNLOCK(cs);
1659 
1660 	ci->status = CTL_ISCSI_OK;
1661 }
1662 
1663 static void
1664 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1665 {
1666 	struct ctl_iscsi_list_params *cilp;
1667 	struct cfiscsi_session *cs;
1668 	struct cfiscsi_softc *softc;
1669 	struct sbuf *sb;
1670 	int error;
1671 
1672 	cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1673 	softc = &cfiscsi_softc;
1674 
1675 	sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1676 	if (sb == NULL) {
1677 		ci->status = CTL_ISCSI_ERROR;
1678 		snprintf(ci->error_str, sizeof(ci->error_str),
1679 		    "Unable to allocate %d bytes for iSCSI session list",
1680 		    cilp->alloc_len);
1681 		return;
1682 	}
1683 
1684 	sbuf_printf(sb, "<ctlislist>\n");
1685 	mtx_lock(&softc->lock);
1686 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1687 #ifdef ICL_KERNEL_PROXY
1688 		if (cs->cs_target == NULL)
1689 			continue;
1690 #endif
1691 		error = sbuf_printf(sb, "<connection id=\"%d\">"
1692 		    "<initiator>%s</initiator>"
1693 		    "<initiator_addr>%s</initiator_addr>"
1694 		    "<initiator_alias>%s</initiator_alias>"
1695 		    "<target>%s</target>"
1696 		    "<target_alias>%s</target_alias>"
1697 		    "<target_portal_group_tag>%u</target_portal_group_tag>"
1698 		    "<header_digest>%s</header_digest>"
1699 		    "<data_digest>%s</data_digest>"
1700 		    "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1701 		    "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1702 		    "<max_burst_length>%d</max_burst_length>"
1703 		    "<first_burst_length>%d</first_burst_length>"
1704 		    "<immediate_data>%d</immediate_data>"
1705 		    "<iser>%d</iser>"
1706 		    "<offload>%s</offload>"
1707 		    "</connection>\n",
1708 		    cs->cs_id,
1709 		    cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1710 		    cs->cs_target->ct_name, cs->cs_target->ct_alias,
1711 		    cs->cs_target->ct_tag,
1712 		    cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1713 		    cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1714 		    cs->cs_max_recv_data_segment_length,
1715 		    cs->cs_max_send_data_segment_length,
1716 		    cs->cs_max_burst_length,
1717 		    cs->cs_first_burst_length,
1718 		    cs->cs_immediate_data,
1719 		    cs->cs_conn->ic_iser,
1720 		    cs->cs_conn->ic_offload);
1721 		if (error != 0)
1722 			break;
1723 	}
1724 	mtx_unlock(&softc->lock);
1725 	error = sbuf_printf(sb, "</ctlislist>\n");
1726 	if (error != 0) {
1727 		sbuf_delete(sb);
1728 		ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1729 		snprintf(ci->error_str, sizeof(ci->error_str),
1730 		    "Out of space, %d bytes is too small", cilp->alloc_len);
1731 		return;
1732 	}
1733 	sbuf_finish(sb);
1734 
1735 	error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1736 	if (error != 0) {
1737 		sbuf_delete(sb);
1738 		snprintf(ci->error_str, sizeof(ci->error_str),
1739 		    "copyout failed with error %d", error);
1740 		ci->status = CTL_ISCSI_ERROR;
1741 		return;
1742 	}
1743 	cilp->fill_len = sbuf_len(sb) + 1;
1744 	ci->status = CTL_ISCSI_OK;
1745 	sbuf_delete(sb);
1746 }
1747 
1748 static void
1749 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1750 {
1751 	struct icl_pdu *response;
1752 	struct iscsi_bhs_asynchronous_message *bhsam;
1753 	struct ctl_iscsi_logout_params *cilp;
1754 	struct cfiscsi_session *cs;
1755 	struct cfiscsi_softc *softc;
1756 	int found = 0;
1757 
1758 	cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1759 	softc = &cfiscsi_softc;
1760 
1761 	mtx_lock(&softc->lock);
1762 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1763 		if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1764 		    strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1765 		    strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1766 			continue;
1767 
1768 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1769 		if (response == NULL) {
1770 			ci->status = CTL_ISCSI_ERROR;
1771 			snprintf(ci->error_str, sizeof(ci->error_str),
1772 			    "Unable to allocate memory");
1773 			mtx_unlock(&softc->lock);
1774 			return;
1775 		}
1776 		bhsam =
1777 		    (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1778 		bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1779 		bhsam->bhsam_flags = 0x80;
1780 		bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1781 		bhsam->bhsam_parameter3 = htons(10);
1782 		cfiscsi_pdu_queue(response);
1783 		found++;
1784 	}
1785 	mtx_unlock(&softc->lock);
1786 
1787 	if (found == 0) {
1788 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1789 		snprintf(ci->error_str, sizeof(ci->error_str),
1790 		    "No matching connections found");
1791 		return;
1792 	}
1793 
1794 	ci->status = CTL_ISCSI_OK;
1795 }
1796 
1797 static void
1798 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1799 {
1800 	struct icl_pdu *response;
1801 	struct iscsi_bhs_asynchronous_message *bhsam;
1802 	struct ctl_iscsi_terminate_params *citp;
1803 	struct cfiscsi_session *cs;
1804 	struct cfiscsi_softc *softc;
1805 	int found = 0;
1806 
1807 	citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1808 	softc = &cfiscsi_softc;
1809 
1810 	mtx_lock(&softc->lock);
1811 	TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1812 		if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1813 		    strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1814 		    strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1815 			continue;
1816 
1817 		response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1818 		if (response == NULL) {
1819 			/*
1820 			 * Oh well.  Just terminate the connection.
1821 			 */
1822 		} else {
1823 			bhsam = (struct iscsi_bhs_asynchronous_message *)
1824 			    response->ip_bhs;
1825 			bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1826 			bhsam->bhsam_flags = 0x80;
1827 			bhsam->bhsam_0xffffffff = 0xffffffff;
1828 			bhsam->bhsam_async_event =
1829 			    BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1830 			cfiscsi_pdu_queue(response);
1831 		}
1832 		cfiscsi_session_terminate(cs);
1833 		found++;
1834 	}
1835 	mtx_unlock(&softc->lock);
1836 
1837 	if (found == 0) {
1838 		ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1839 		snprintf(ci->error_str, sizeof(ci->error_str),
1840 		    "No matching connections found");
1841 		return;
1842 	}
1843 
1844 	ci->status = CTL_ISCSI_OK;
1845 }
1846 
1847 static void
1848 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1849 {
1850 	struct ctl_iscsi_limits_params *cilp;
1851 	struct icl_drv_limits idl;
1852 	int error;
1853 
1854 	cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1855 
1856 	error = icl_limits(cilp->offload, false, &idl);
1857 	if (error != 0) {
1858 		ci->status = CTL_ISCSI_ERROR;
1859 		snprintf(ci->error_str, sizeof(ci->error_str),
1860 			"%s: icl_limits failed with error %d",
1861 			__func__, error);
1862 		return;
1863 	}
1864 
1865 	cilp->max_recv_data_segment_length =
1866 	    idl.idl_max_recv_data_segment_length;
1867 	cilp->max_send_data_segment_length =
1868 	    idl.idl_max_send_data_segment_length;
1869 	cilp->max_burst_length = idl.idl_max_burst_length;
1870 	cilp->first_burst_length = idl.idl_first_burst_length;
1871 
1872 	ci->status = CTL_ISCSI_OK;
1873 }
1874 
1875 #ifdef ICL_KERNEL_PROXY
1876 static void
1877 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1878 {
1879 	struct ctl_iscsi_listen_params *cilp;
1880 	struct sockaddr *sa;
1881 	int error;
1882 
1883 	cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1884 
1885 	if (cfiscsi_softc.listener == NULL) {
1886 		CFISCSI_DEBUG("no listener");
1887 		snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1888 		ci->status = CTL_ISCSI_ERROR;
1889 		return;
1890 	}
1891 
1892 	error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1893 	if (error != 0) {
1894 		CFISCSI_DEBUG("getsockaddr, error %d", error);
1895 		snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1896 		ci->status = CTL_ISCSI_ERROR;
1897 		return;
1898 	}
1899 
1900 	error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1901 	    cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1902 	if (error != 0) {
1903 		free(sa, M_SONAME);
1904 		CFISCSI_DEBUG("icl_listen_add, error %d", error);
1905 		snprintf(ci->error_str, sizeof(ci->error_str),
1906 		    "icl_listen_add failed, error %d", error);
1907 		ci->status = CTL_ISCSI_ERROR;
1908 		return;
1909 	}
1910 
1911 	ci->status = CTL_ISCSI_OK;
1912 }
1913 
1914 static void
1915 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1916 {
1917 	struct ctl_iscsi_accept_params *ciap;
1918 	struct cfiscsi_session *cs;
1919 	int error;
1920 
1921 	ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1922 
1923 	mtx_lock(&cfiscsi_softc.lock);
1924 	for (;;) {
1925 		TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1926 			if (cs->cs_waiting_for_ctld)
1927 				break;
1928 		}
1929 		if (cs != NULL)
1930 			break;
1931 		error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1932 		if (error != 0) {
1933 			mtx_unlock(&cfiscsi_softc.lock);
1934 			snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1935 			ci->status = CTL_ISCSI_ERROR;
1936 			return;
1937 		}
1938 	}
1939 	mtx_unlock(&cfiscsi_softc.lock);
1940 
1941 	cs->cs_waiting_for_ctld = false;
1942 	cs->cs_login_phase = true;
1943 
1944 	ciap->connection_id = cs->cs_id;
1945 	ciap->portal_id = cs->cs_portal_id;
1946 	ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1947 	error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1948 	    cs->cs_initiator_sa->sa_len);
1949 	if (error != 0) {
1950 		snprintf(ci->error_str, sizeof(ci->error_str),
1951 		    "copyout failed with error %d", error);
1952 		ci->status = CTL_ISCSI_ERROR;
1953 		return;
1954 	}
1955 
1956 	ci->status = CTL_ISCSI_OK;
1957 }
1958 
1959 static void
1960 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1961 {
1962 	struct ctl_iscsi_send_params *cisp;
1963 	struct cfiscsi_session *cs;
1964 	struct icl_pdu *ip;
1965 	size_t datalen;
1966 	void *data;
1967 	int error;
1968 
1969 	cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1970 
1971 	mtx_lock(&cfiscsi_softc.lock);
1972 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1973 		if (cs->cs_id == cisp->connection_id)
1974 			break;
1975 	}
1976 	if (cs == NULL) {
1977 		mtx_unlock(&cfiscsi_softc.lock);
1978 		snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1979 		ci->status = CTL_ISCSI_ERROR;
1980 		return;
1981 	}
1982 	mtx_unlock(&cfiscsi_softc.lock);
1983 
1984 #if 0
1985 	if (cs->cs_login_phase == false)
1986 		return (EBUSY);
1987 #endif
1988 
1989 	if (cs->cs_terminating) {
1990 		snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1991 		ci->status = CTL_ISCSI_ERROR;
1992 		return;
1993 	}
1994 
1995 	datalen = cisp->data_segment_len;
1996 	/*
1997 	 * XXX
1998 	 */
1999 	//if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2000 	if (datalen > 65535) {
2001 		snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2002 		ci->status = CTL_ISCSI_ERROR;
2003 		return;
2004 	}
2005 	if (datalen > 0) {
2006 		data = malloc(datalen, M_CFISCSI, M_WAITOK);
2007 		error = copyin(cisp->data_segment, data, datalen);
2008 		if (error != 0) {
2009 			free(data, M_CFISCSI);
2010 			snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2011 			ci->status = CTL_ISCSI_ERROR;
2012 			return;
2013 		}
2014 	}
2015 
2016 	ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2017 	memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2018 	if (datalen > 0) {
2019 		icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2020 		free(data, M_CFISCSI);
2021 	}
2022 	CFISCSI_SESSION_LOCK(cs);
2023 	icl_pdu_queue(ip);
2024 	CFISCSI_SESSION_UNLOCK(cs);
2025 	ci->status = CTL_ISCSI_OK;
2026 }
2027 
2028 static void
2029 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2030 {
2031 	struct ctl_iscsi_receive_params *cirp;
2032 	struct cfiscsi_session *cs;
2033 	struct icl_pdu *ip;
2034 	void *data;
2035 	int error;
2036 
2037 	cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2038 
2039 	mtx_lock(&cfiscsi_softc.lock);
2040 	TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2041 		if (cs->cs_id == cirp->connection_id)
2042 			break;
2043 	}
2044 	if (cs == NULL) {
2045 		mtx_unlock(&cfiscsi_softc.lock);
2046 		snprintf(ci->error_str, sizeof(ci->error_str),
2047 		    "connection not found");
2048 		ci->status = CTL_ISCSI_ERROR;
2049 		return;
2050 	}
2051 	mtx_unlock(&cfiscsi_softc.lock);
2052 
2053 #if 0
2054 	if (is->is_login_phase == false)
2055 		return (EBUSY);
2056 #endif
2057 
2058 	CFISCSI_SESSION_LOCK(cs);
2059 	while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2060 		error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2061 		if (error != 0) {
2062 			CFISCSI_SESSION_UNLOCK(cs);
2063 			snprintf(ci->error_str, sizeof(ci->error_str),
2064 			    "interrupted by signal");
2065 			ci->status = CTL_ISCSI_ERROR;
2066 			return;
2067 		}
2068 	}
2069 
2070 	if (cs->cs_terminating) {
2071 		CFISCSI_SESSION_UNLOCK(cs);
2072 		snprintf(ci->error_str, sizeof(ci->error_str),
2073 		    "connection terminating");
2074 		ci->status = CTL_ISCSI_ERROR;
2075 		return;
2076 	}
2077 	ip = cs->cs_login_pdu;
2078 	cs->cs_login_pdu = NULL;
2079 	CFISCSI_SESSION_UNLOCK(cs);
2080 
2081 	if (ip->ip_data_len > cirp->data_segment_len) {
2082 		icl_pdu_free(ip);
2083 		snprintf(ci->error_str, sizeof(ci->error_str),
2084 		    "data segment too big");
2085 		ci->status = CTL_ISCSI_ERROR;
2086 		return;
2087 	}
2088 
2089 	copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2090 	if (ip->ip_data_len > 0) {
2091 		data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2092 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2093 		copyout(data, cirp->data_segment, ip->ip_data_len);
2094 		free(data, M_CFISCSI);
2095 	}
2096 
2097 	icl_pdu_free(ip);
2098 	ci->status = CTL_ISCSI_OK;
2099 }
2100 
2101 #endif /* !ICL_KERNEL_PROXY */
2102 
2103 static void
2104 cfiscsi_ioctl_port_create(struct ctl_req *req)
2105 {
2106 	struct cfiscsi_target *ct;
2107 	struct ctl_port *port;
2108 	const char *target, *alias, *tags;
2109 	struct scsi_vpd_id_descriptor *desc;
2110 	ctl_options_t opts;
2111 	int retval, len, idlen;
2112 	uint16_t tag;
2113 
2114 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2115 	target = ctl_get_opt(&opts, "cfiscsi_target");
2116 	alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
2117 	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
2118 	if (target == NULL || tags == NULL) {
2119 		req->status = CTL_LUN_ERROR;
2120 		snprintf(req->error_str, sizeof(req->error_str),
2121 		    "Missing required argument");
2122 		ctl_free_opts(&opts);
2123 		return;
2124 	}
2125 	tag = strtol(tags, (char **)NULL, 10);
2126 	ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2127 	if (ct == NULL) {
2128 		req->status = CTL_LUN_ERROR;
2129 		snprintf(req->error_str, sizeof(req->error_str),
2130 		    "failed to create target \"%s\"", target);
2131 		ctl_free_opts(&opts);
2132 		return;
2133 	}
2134 	if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2135 		req->status = CTL_LUN_ERROR;
2136 		snprintf(req->error_str, sizeof(req->error_str),
2137 		    "target \"%s\" for portal group tag %u already exists",
2138 		    target, tag);
2139 		cfiscsi_target_release(ct);
2140 		ctl_free_opts(&opts);
2141 		return;
2142 	}
2143 	port = &ct->ct_port;
2144 	// WAT
2145 	if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2146 		goto done;
2147 
2148 	port->frontend = &cfiscsi_frontend;
2149 	port->port_type = CTL_PORT_ISCSI;
2150 	/* XXX KDM what should the real number be here? */
2151 	port->num_requested_ctl_io = 4096;
2152 	port->port_name = "iscsi";
2153 	port->physical_port = tag;
2154 	port->virtual_port = ct->ct_target_id;
2155 	port->port_online = cfiscsi_online;
2156 	port->port_offline = cfiscsi_offline;
2157 	port->port_info = cfiscsi_info;
2158 	port->onoff_arg = ct;
2159 	port->fe_datamove = cfiscsi_datamove;
2160 	port->fe_done = cfiscsi_done;
2161 	port->targ_port = -1;
2162 
2163 	port->options = opts;
2164 	STAILQ_INIT(&opts);
2165 
2166 	/* Generate Port ID. */
2167 	idlen = strlen(target) + strlen(",t,0x0001") + 1;
2168 	idlen = roundup2(idlen, 4);
2169 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2170 	port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2171 	    M_CTL, M_WAITOK | M_ZERO);
2172 	port->port_devid->len = len;
2173 	desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2174 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2175 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2176 	    SVPD_ID_TYPE_SCSI_NAME;
2177 	desc->length = idlen;
2178 	snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2179 
2180 	/* Generate Target ID. */
2181 	idlen = strlen(target) + 1;
2182 	idlen = roundup2(idlen, 4);
2183 	len = sizeof(struct scsi_vpd_device_id) + idlen;
2184 	port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2185 	    M_CTL, M_WAITOK | M_ZERO);
2186 	port->target_devid->len = len;
2187 	desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2188 	desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2189 	desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2190 	    SVPD_ID_TYPE_SCSI_NAME;
2191 	desc->length = idlen;
2192 	strlcpy(desc->identifier, target, idlen);
2193 
2194 	retval = ctl_port_register(port);
2195 	if (retval != 0) {
2196 		ctl_free_opts(&port->options);
2197 		free(port->port_devid, M_CFISCSI);
2198 		free(port->target_devid, M_CFISCSI);
2199 		cfiscsi_target_release(ct);
2200 		req->status = CTL_LUN_ERROR;
2201 		snprintf(req->error_str, sizeof(req->error_str),
2202 		    "ctl_port_register() failed with error %d", retval);
2203 		return;
2204 	}
2205 done:
2206 	ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2207 	req->status = CTL_LUN_OK;
2208 	memcpy(req->kern_args[0].kvalue, &port->targ_port,
2209 	    sizeof(port->targ_port)); //XXX
2210 }
2211 
2212 static void
2213 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2214 {
2215 	struct cfiscsi_target *ct;
2216 	const char *target, *tags;
2217 	ctl_options_t opts;
2218 	uint16_t tag;
2219 
2220 	ctl_init_opts(&opts, req->num_args, req->kern_args);
2221 	target = ctl_get_opt(&opts, "cfiscsi_target");
2222 	tags = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
2223 	if (target == NULL || tags == NULL) {
2224 		ctl_free_opts(&opts);
2225 		req->status = CTL_LUN_ERROR;
2226 		snprintf(req->error_str, sizeof(req->error_str),
2227 		    "Missing required argument");
2228 		return;
2229 	}
2230 	tag = strtol(tags, (char **)NULL, 10);
2231 	ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2232 	if (ct == NULL) {
2233 		ctl_free_opts(&opts);
2234 		req->status = CTL_LUN_ERROR;
2235 		snprintf(req->error_str, sizeof(req->error_str),
2236 		    "can't find target \"%s\"", target);
2237 		return;
2238 	}
2239 	if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
2240 		ctl_free_opts(&opts);
2241 		req->status = CTL_LUN_ERROR;
2242 		snprintf(req->error_str, sizeof(req->error_str),
2243 		    "target \"%s\" is already dying", target);
2244 		return;
2245 	}
2246 	ctl_free_opts(&opts);
2247 
2248 	ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2249 	ctl_port_offline(&ct->ct_port);
2250 	cfiscsi_target_release(ct);
2251 	cfiscsi_target_release(ct);
2252 	req->status = CTL_LUN_OK;
2253 }
2254 
2255 static int
2256 cfiscsi_ioctl(struct cdev *dev,
2257     u_long cmd, caddr_t addr, int flag, struct thread *td)
2258 {
2259 	struct ctl_iscsi *ci;
2260 	struct ctl_req *req;
2261 
2262 	if (cmd == CTL_PORT_REQ) {
2263 		req = (struct ctl_req *)addr;
2264 		switch (req->reqtype) {
2265 		case CTL_REQ_CREATE:
2266 			cfiscsi_ioctl_port_create(req);
2267 			break;
2268 		case CTL_REQ_REMOVE:
2269 			cfiscsi_ioctl_port_remove(req);
2270 			break;
2271 		default:
2272 			req->status = CTL_LUN_ERROR;
2273 			snprintf(req->error_str, sizeof(req->error_str),
2274 			    "Unsupported request type %d", req->reqtype);
2275 		}
2276 		return (0);
2277 	}
2278 
2279 	if (cmd != CTL_ISCSI)
2280 		return (ENOTTY);
2281 
2282 	ci = (struct ctl_iscsi *)addr;
2283 	switch (ci->type) {
2284 	case CTL_ISCSI_HANDOFF:
2285 		cfiscsi_ioctl_handoff(ci);
2286 		break;
2287 	case CTL_ISCSI_LIST:
2288 		cfiscsi_ioctl_list(ci);
2289 		break;
2290 	case CTL_ISCSI_LOGOUT:
2291 		cfiscsi_ioctl_logout(ci);
2292 		break;
2293 	case CTL_ISCSI_TERMINATE:
2294 		cfiscsi_ioctl_terminate(ci);
2295 		break;
2296 	case CTL_ISCSI_LIMITS:
2297 		cfiscsi_ioctl_limits(ci);
2298 		break;
2299 #ifdef ICL_KERNEL_PROXY
2300 	case CTL_ISCSI_LISTEN:
2301 		cfiscsi_ioctl_listen(ci);
2302 		break;
2303 	case CTL_ISCSI_ACCEPT:
2304 		cfiscsi_ioctl_accept(ci);
2305 		break;
2306 	case CTL_ISCSI_SEND:
2307 		cfiscsi_ioctl_send(ci);
2308 		break;
2309 	case CTL_ISCSI_RECEIVE:
2310 		cfiscsi_ioctl_receive(ci);
2311 		break;
2312 #else
2313 	case CTL_ISCSI_LISTEN:
2314 	case CTL_ISCSI_ACCEPT:
2315 	case CTL_ISCSI_SEND:
2316 	case CTL_ISCSI_RECEIVE:
2317 		ci->status = CTL_ISCSI_ERROR;
2318 		snprintf(ci->error_str, sizeof(ci->error_str),
2319 		    "%s: CTL compiled without ICL_KERNEL_PROXY",
2320 		    __func__);
2321 		break;
2322 #endif /* !ICL_KERNEL_PROXY */
2323 	default:
2324 		ci->status = CTL_ISCSI_ERROR;
2325 		snprintf(ci->error_str, sizeof(ci->error_str),
2326 		    "%s: invalid iSCSI request type %d", __func__, ci->type);
2327 		break;
2328 	}
2329 
2330 	return (0);
2331 }
2332 
2333 static void
2334 cfiscsi_target_hold(struct cfiscsi_target *ct)
2335 {
2336 
2337 	refcount_acquire(&ct->ct_refcount);
2338 }
2339 
2340 static void
2341 cfiscsi_target_release(struct cfiscsi_target *ct)
2342 {
2343 	struct cfiscsi_softc *softc;
2344 
2345 	softc = ct->ct_softc;
2346 	mtx_lock(&softc->lock);
2347 	if (refcount_release(&ct->ct_refcount)) {
2348 		TAILQ_REMOVE(&softc->targets, ct, ct_next);
2349 		mtx_unlock(&softc->lock);
2350 		if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2351 			ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2352 			if (ctl_port_deregister(&ct->ct_port) != 0)
2353 				printf("%s: ctl_port_deregister() failed\n",
2354 				    __func__);
2355 		}
2356 		free(ct, M_CFISCSI);
2357 
2358 		return;
2359 	}
2360 	mtx_unlock(&softc->lock);
2361 }
2362 
2363 static struct cfiscsi_target *
2364 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2365 {
2366 	struct cfiscsi_target *ct;
2367 
2368 	mtx_lock(&softc->lock);
2369 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2370 		if (ct->ct_tag != tag ||
2371 		    strcmp(name, ct->ct_name) != 0 ||
2372 		    ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2373 			continue;
2374 		cfiscsi_target_hold(ct);
2375 		mtx_unlock(&softc->lock);
2376 		return (ct);
2377 	}
2378 	mtx_unlock(&softc->lock);
2379 
2380 	return (NULL);
2381 }
2382 
2383 static struct cfiscsi_target *
2384 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2385     const char *alias, uint16_t tag)
2386 {
2387 	struct cfiscsi_target *ct, *newct;
2388 
2389 	if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2390 		return (NULL);
2391 
2392 	newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2393 
2394 	mtx_lock(&softc->lock);
2395 	TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2396 		if (ct->ct_tag != tag ||
2397 		    strcmp(name, ct->ct_name) != 0 ||
2398 		    ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2399 			continue;
2400 		cfiscsi_target_hold(ct);
2401 		mtx_unlock(&softc->lock);
2402 		free(newct, M_CFISCSI);
2403 		return (ct);
2404 	}
2405 
2406 	strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2407 	if (alias != NULL)
2408 		strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2409 	newct->ct_tag = tag;
2410 	refcount_init(&newct->ct_refcount, 1);
2411 	newct->ct_softc = softc;
2412 	if (TAILQ_EMPTY(&softc->targets))
2413 		softc->last_target_id = 0;
2414 	newct->ct_target_id = ++softc->last_target_id;
2415 	TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2416 	mtx_unlock(&softc->lock);
2417 
2418 	return (newct);
2419 }
2420 
2421 static void
2422 cfiscsi_datamove_in(union ctl_io *io)
2423 {
2424 	struct cfiscsi_session *cs;
2425 	struct icl_pdu *request, *response;
2426 	const struct iscsi_bhs_scsi_command *bhssc;
2427 	struct iscsi_bhs_data_in *bhsdi;
2428 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2429 	size_t len, expected_len, sg_len, buffer_offset;
2430 	const char *sg_addr;
2431 	int ctl_sg_count, error, i;
2432 
2433 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2434 	cs = PDU_SESSION(request);
2435 
2436 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2437 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2438 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2439 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2440 
2441 	if (io->scsiio.kern_sg_entries > 0) {
2442 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2443 		ctl_sg_count = io->scsiio.kern_sg_entries;
2444 	} else {
2445 		ctl_sglist = &ctl_sg_entry;
2446 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2447 		ctl_sglist->len = io->scsiio.kern_data_len;
2448 		ctl_sg_count = 1;
2449 	}
2450 
2451 	/*
2452 	 * This is the total amount of data to be transferred within the current
2453 	 * SCSI command.  We need to record it so that we can properly report
2454 	 * underflow/underflow.
2455 	 */
2456 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2457 
2458 	/*
2459 	 * This is the offset within the current SCSI command; for the first
2460 	 * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2461 	 * it will be the sum of lengths of previous ones.
2462 	 */
2463 	buffer_offset = io->scsiio.kern_rel_offset;
2464 
2465 	/*
2466 	 * This is the transfer length expected by the initiator.  In theory,
2467 	 * it could be different from the correct amount of data from the SCSI
2468 	 * point of view, even if that doesn't make any sense.
2469 	 */
2470 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2471 #if 0
2472 	if (expected_len != io->scsiio.kern_total_len) {
2473 		CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2474 		    "actual length %zd", expected_len,
2475 		    (size_t)io->scsiio.kern_total_len);
2476 	}
2477 #endif
2478 
2479 	if (buffer_offset >= expected_len) {
2480 #if 0
2481 		CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2482 		    "already sent the expected len", buffer_offset);
2483 #endif
2484 		io->scsiio.be_move_done(io);
2485 		return;
2486 	}
2487 
2488 	i = 0;
2489 	sg_addr = NULL;
2490 	sg_len = 0;
2491 	response = NULL;
2492 	bhsdi = NULL;
2493 	for (;;) {
2494 		if (response == NULL) {
2495 			response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2496 			if (response == NULL) {
2497 				CFISCSI_SESSION_WARN(cs, "failed to "
2498 				    "allocate memory; dropping connection");
2499 				ctl_set_busy(&io->scsiio);
2500 				io->scsiio.be_move_done(io);
2501 				cfiscsi_session_terminate(cs);
2502 				return;
2503 			}
2504 			bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2505 			bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2506 			bhsdi->bhsdi_initiator_task_tag =
2507 			    bhssc->bhssc_initiator_task_tag;
2508 			bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2509 			bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2510 			PDU_EXPDATASN(request)++;
2511 			bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2512 		}
2513 
2514 		KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2515 		if (sg_len == 0) {
2516 			sg_addr = ctl_sglist[i].addr;
2517 			sg_len = ctl_sglist[i].len;
2518 			KASSERT(sg_len > 0, ("sg_len <= 0"));
2519 		}
2520 
2521 		len = sg_len;
2522 
2523 		/*
2524 		 * Truncate to maximum data segment length.
2525 		 */
2526 		KASSERT(response->ip_data_len < cs->cs_max_send_data_segment_length,
2527 		    ("ip_data_len %zd >= max_send_data_segment_length %d",
2528 		    response->ip_data_len, cs->cs_max_send_data_segment_length));
2529 		if (response->ip_data_len + len >
2530 		    cs->cs_max_send_data_segment_length) {
2531 			len = cs->cs_max_send_data_segment_length -
2532 			    response->ip_data_len;
2533 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2534 			    len, sg_len));
2535 		}
2536 
2537 		/*
2538 		 * Truncate to expected data transfer length.
2539 		 */
2540 		KASSERT(buffer_offset + response->ip_data_len < expected_len,
2541 		    ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2542 		    buffer_offset, response->ip_data_len, expected_len));
2543 		if (buffer_offset + response->ip_data_len + len > expected_len) {
2544 			CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2545 			    "to expected data transfer length %zd",
2546 			    buffer_offset + response->ip_data_len + len, expected_len);
2547 			len = expected_len - (buffer_offset + response->ip_data_len);
2548 			KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2549 			    len, sg_len));
2550 		}
2551 
2552 		error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2553 		if (error != 0) {
2554 			CFISCSI_SESSION_WARN(cs, "failed to "
2555 			    "allocate memory; dropping connection");
2556 			icl_pdu_free(response);
2557 			ctl_set_busy(&io->scsiio);
2558 			io->scsiio.be_move_done(io);
2559 			cfiscsi_session_terminate(cs);
2560 			return;
2561 		}
2562 		sg_addr += len;
2563 		sg_len -= len;
2564 		io->scsiio.kern_data_resid -= len;
2565 
2566 		KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2567 		    ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2568 		    buffer_offset, response->ip_data_len, expected_len));
2569 		if (buffer_offset + response->ip_data_len == expected_len) {
2570 			/*
2571 			 * Already have the amount of data the initiator wanted.
2572 			 */
2573 			break;
2574 		}
2575 
2576 		if (sg_len == 0) {
2577 			/*
2578 			 * End of scatter-gather segment;
2579 			 * proceed to the next one...
2580 			 */
2581 			if (i == ctl_sg_count - 1) {
2582 				/*
2583 				 * ... unless this was the last one.
2584 				 */
2585 				break;
2586 			}
2587 			i++;
2588 		}
2589 
2590 		if (response->ip_data_len == cs->cs_max_send_data_segment_length) {
2591 			/*
2592 			 * Can't stuff more data into the current PDU;
2593 			 * queue it.  Note that's not enough to check
2594 			 * for kern_data_resid == 0 instead; there
2595 			 * may be several Data-In PDUs for the final
2596 			 * call to cfiscsi_datamove(), and we want
2597 			 * to set the F flag only on the last of them.
2598 			 */
2599 			buffer_offset += response->ip_data_len;
2600 			if (buffer_offset == io->scsiio.kern_total_len ||
2601 			    buffer_offset == expected_len) {
2602 				buffer_offset -= response->ip_data_len;
2603 				break;
2604 			}
2605 			cfiscsi_pdu_queue(response);
2606 			response = NULL;
2607 			bhsdi = NULL;
2608 		}
2609 	}
2610 	if (response != NULL) {
2611 		buffer_offset += response->ip_data_len;
2612 		if (buffer_offset == io->scsiio.kern_total_len ||
2613 		    buffer_offset == expected_len) {
2614 			bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2615 			if (io->io_hdr.status == CTL_SUCCESS) {
2616 				bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2617 				if (PDU_TOTAL_TRANSFER_LEN(request) <
2618 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2619 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2620 					bhsdi->bhsdi_residual_count =
2621 					    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2622 					    PDU_TOTAL_TRANSFER_LEN(request));
2623 				} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2624 				    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2625 					bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2626 					bhsdi->bhsdi_residual_count =
2627 					    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2628 					    ntohl(bhssc->bhssc_expected_data_transfer_length));
2629 				}
2630 				bhsdi->bhsdi_status = io->scsiio.scsi_status;
2631 				io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2632 			}
2633 		}
2634 		KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2635 		cfiscsi_pdu_queue(response);
2636 	}
2637 
2638 	io->scsiio.be_move_done(io);
2639 }
2640 
2641 static void
2642 cfiscsi_datamove_out(union ctl_io *io)
2643 {
2644 	struct cfiscsi_session *cs;
2645 	struct icl_pdu *request, *response;
2646 	const struct iscsi_bhs_scsi_command *bhssc;
2647 	struct iscsi_bhs_r2t *bhsr2t;
2648 	struct cfiscsi_data_wait *cdw;
2649 	struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2650 	uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2651 	uint32_t target_transfer_tag;
2652 	bool done;
2653 
2654 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2655 	cs = PDU_SESSION(request);
2656 
2657 	bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2658 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2659 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2660 	    ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2661 
2662 	/*
2663 	 * We need to record it so that we can properly report
2664 	 * underflow/underflow.
2665 	 */
2666 	PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2667 
2668 	/*
2669 	 * Complete write underflow.  Not a single byte to read.  Return.
2670 	 */
2671 	expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2672 	if (io->scsiio.kern_rel_offset >= expected_len) {
2673 		io->scsiio.be_move_done(io);
2674 		return;
2675 	}
2676 	datamove_len = MIN(io->scsiio.kern_data_len,
2677 	    expected_len - io->scsiio.kern_rel_offset);
2678 
2679 	target_transfer_tag =
2680 	    atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2681 	cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2682 	    &target_transfer_tag);
2683 	if (cdw == NULL) {
2684 		CFISCSI_SESSION_WARN(cs, "failed to "
2685 		    "allocate memory; dropping connection");
2686 		ctl_set_busy(&io->scsiio);
2687 		io->scsiio.be_move_done(io);
2688 		cfiscsi_session_terminate(cs);
2689 		return;
2690 	}
2691 #if 0
2692 	CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2693 	    "task tag 0x%x, target transfer tag 0x%x",
2694 	    bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2695 #endif
2696 
2697 	cdw->cdw_ctl_io = io;
2698 	cdw->cdw_target_transfer_tag = target_transfer_tag;
2699 	cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2700 	cdw->cdw_r2t_end = datamove_len;
2701 	cdw->cdw_datasn = 0;
2702 
2703 	/* Set initial data pointer for the CDW respecting ext_data_filled. */
2704 	if (io->scsiio.kern_sg_entries > 0) {
2705 		ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2706 	} else {
2707 		ctl_sglist = &ctl_sg_entry;
2708 		ctl_sglist->addr = io->scsiio.kern_data_ptr;
2709 		ctl_sglist->len = datamove_len;
2710 	}
2711 	cdw->cdw_sg_index = 0;
2712 	cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2713 	cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2714 	r2t_off = io->scsiio.ext_data_filled;
2715 	while (r2t_off > 0) {
2716 		if (r2t_off >= cdw->cdw_sg_len) {
2717 			r2t_off -= cdw->cdw_sg_len;
2718 			cdw->cdw_sg_index++;
2719 			cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2720 			cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2721 			continue;
2722 		}
2723 		cdw->cdw_sg_addr += r2t_off;
2724 		cdw->cdw_sg_len -= r2t_off;
2725 		r2t_off = 0;
2726 	}
2727 
2728 	if (cs->cs_immediate_data &&
2729 	    io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2730 	    icl_pdu_data_segment_length(request)) {
2731 		done = cfiscsi_handle_data_segment(request, cdw);
2732 		if (done) {
2733 			cfiscsi_data_wait_free(cs, cdw);
2734 			io->scsiio.be_move_done(io);
2735 			return;
2736 		}
2737 	}
2738 
2739 	r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2740 	r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2741 	    cs->cs_max_burst_length);
2742 	cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2743 
2744 	CFISCSI_SESSION_LOCK(cs);
2745 	TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2746 	CFISCSI_SESSION_UNLOCK(cs);
2747 
2748 	/*
2749 	 * XXX: We should limit the number of outstanding R2T PDUs
2750 	 * 	per task to MaxOutstandingR2T.
2751 	 */
2752 	response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2753 	if (response == NULL) {
2754 		CFISCSI_SESSION_WARN(cs, "failed to "
2755 		    "allocate memory; dropping connection");
2756 		ctl_set_busy(&io->scsiio);
2757 		io->scsiio.be_move_done(io);
2758 		cfiscsi_session_terminate(cs);
2759 		return;
2760 	}
2761 	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2762 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2763 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2764 	bhsr2t->bhsr2t_flags = 0x80;
2765 	bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2766 	bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2767 	bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2768 	/*
2769 	 * XXX: Here we assume that cfiscsi_datamove() won't ever
2770 	 *	be running concurrently on several CPUs for a given
2771 	 *	command.
2772 	 */
2773 	bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2774 	PDU_R2TSN(request)++;
2775 	/*
2776 	 * This is the offset within the current SCSI command;
2777 	 * i.e. for the first call of datamove(), it will be 0,
2778 	 * and for subsequent ones it will be the sum of lengths
2779 	 * of previous ones.
2780 	 *
2781 	 * The ext_data_filled is to account for unsolicited
2782 	 * (immediate) data that might have already arrived.
2783 	 */
2784 	bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2785 	/*
2786 	 * This is the total length (sum of S/G lengths) this call
2787 	 * to cfiscsi_datamove() is supposed to handle, limited by
2788 	 * MaxBurstLength.
2789 	 */
2790 	bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2791 	cfiscsi_pdu_queue(response);
2792 }
2793 
2794 static void
2795 cfiscsi_datamove(union ctl_io *io)
2796 {
2797 
2798 	if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2799 		cfiscsi_datamove_in(io);
2800 	else {
2801 		/* We hadn't received anything during this datamove yet. */
2802 		io->scsiio.ext_data_filled = 0;
2803 		cfiscsi_datamove_out(io);
2804 	}
2805 }
2806 
2807 static void
2808 cfiscsi_scsi_command_done(union ctl_io *io)
2809 {
2810 	struct icl_pdu *request, *response;
2811 	struct iscsi_bhs_scsi_command *bhssc;
2812 	struct iscsi_bhs_scsi_response *bhssr;
2813 #ifdef DIAGNOSTIC
2814 	struct cfiscsi_data_wait *cdw;
2815 #endif
2816 	struct cfiscsi_session *cs;
2817 	uint16_t sense_length;
2818 
2819 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2820 	cs = PDU_SESSION(request);
2821 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2822 	KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2823 	    ISCSI_BHS_OPCODE_SCSI_COMMAND,
2824 	    ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2825 
2826 	//CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2827 	//    bhssc->bhssc_initiator_task_tag);
2828 
2829 #ifdef DIAGNOSTIC
2830 	CFISCSI_SESSION_LOCK(cs);
2831 	TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2832 		KASSERT(bhssc->bhssc_initiator_task_tag !=
2833 		    cdw->cdw_initiator_task_tag, ("dangling cdw"));
2834 	CFISCSI_SESSION_UNLOCK(cs);
2835 #endif
2836 
2837 	/*
2838 	 * Do not return status for aborted commands.
2839 	 * There are exceptions, but none supported by CTL yet.
2840 	 */
2841 	if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2842 	     (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2843 	    (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2844 		ctl_free_io(io);
2845 		icl_pdu_free(request);
2846 		return;
2847 	}
2848 
2849 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2850 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2851 	bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2852 	bhssr->bhssr_flags = 0x80;
2853 	/*
2854 	 * XXX: We don't deal with bidirectional under/overflows;
2855 	 *	does anything actually support those?
2856 	 */
2857 	if (PDU_TOTAL_TRANSFER_LEN(request) <
2858 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2859 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2860 		bhssr->bhssr_residual_count =
2861 		    htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2862 		    PDU_TOTAL_TRANSFER_LEN(request));
2863 		//CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2864 		//    ntohl(bhssr->bhssr_residual_count));
2865 	} else if (PDU_TOTAL_TRANSFER_LEN(request) >
2866 	    ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2867 		bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2868 		bhssr->bhssr_residual_count =
2869 		    htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2870 		    ntohl(bhssc->bhssc_expected_data_transfer_length));
2871 		//CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2872 		//    ntohl(bhssr->bhssr_residual_count));
2873 	}
2874 	bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2875 	bhssr->bhssr_status = io->scsiio.scsi_status;
2876 	bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2877 	bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2878 
2879 	if (io->scsiio.sense_len > 0) {
2880 #if 0
2881 		CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2882 		    io->scsiio.sense_len);
2883 #endif
2884 		sense_length = htons(io->scsiio.sense_len);
2885 		icl_pdu_append_data(response,
2886 		    &sense_length, sizeof(sense_length), M_WAITOK);
2887 		icl_pdu_append_data(response,
2888 		    &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2889 	}
2890 
2891 	ctl_free_io(io);
2892 	icl_pdu_free(request);
2893 	cfiscsi_pdu_queue(response);
2894 }
2895 
2896 static void
2897 cfiscsi_task_management_done(union ctl_io *io)
2898 {
2899 	struct icl_pdu *request, *response;
2900 	struct iscsi_bhs_task_management_request *bhstmr;
2901 	struct iscsi_bhs_task_management_response *bhstmr2;
2902 	struct cfiscsi_data_wait *cdw, *tmpcdw;
2903 	struct cfiscsi_session *cs, *tcs;
2904 	struct cfiscsi_softc *softc;
2905 	int cold_reset = 0;
2906 
2907 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2908 	cs = PDU_SESSION(request);
2909 	bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2910 	KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2911 	    ISCSI_BHS_OPCODE_TASK_REQUEST,
2912 	    ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2913 
2914 #if 0
2915 	CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2916 	    bhstmr->bhstmr_initiator_task_tag,
2917 	    bhstmr->bhstmr_referenced_task_tag);
2918 #endif
2919 
2920 	if ((bhstmr->bhstmr_function & ~0x80) ==
2921 	    BHSTMR_FUNCTION_ABORT_TASK) {
2922 		/*
2923 		 * Make sure we no longer wait for Data-Out for this command.
2924 		 */
2925 		CFISCSI_SESSION_LOCK(cs);
2926 		TAILQ_FOREACH_SAFE(cdw,
2927 		    &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2928 			if (bhstmr->bhstmr_referenced_task_tag !=
2929 			    cdw->cdw_initiator_task_tag)
2930 				continue;
2931 
2932 #if 0
2933 			CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2934 			    "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2935 #endif
2936 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2937 			    cdw, cdw_next);
2938 			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
2939 			cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
2940 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2941 			cfiscsi_data_wait_free(cs, cdw);
2942 		}
2943 		CFISCSI_SESSION_UNLOCK(cs);
2944 	}
2945 	if ((bhstmr->bhstmr_function & ~0x80) ==
2946 	    BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2947 	    io->io_hdr.status == CTL_SUCCESS)
2948 		cold_reset = 1;
2949 
2950 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
2951 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
2952 	    response->ip_bhs;
2953 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2954 	bhstmr2->bhstmr_flags = 0x80;
2955 	switch (io->taskio.task_status) {
2956 	case CTL_TASK_FUNCTION_COMPLETE:
2957 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2958 		break;
2959 	case CTL_TASK_FUNCTION_SUCCEEDED:
2960 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
2961 		break;
2962 	case CTL_TASK_LUN_DOES_NOT_EXIST:
2963 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
2964 		break;
2965 	case CTL_TASK_FUNCTION_NOT_SUPPORTED:
2966 	default:
2967 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2968 		break;
2969 	}
2970 	memcpy(bhstmr2->bhstmr_additional_reponse_information,
2971 	    io->taskio.task_resp, sizeof(io->taskio.task_resp));
2972 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2973 
2974 	ctl_free_io(io);
2975 	icl_pdu_free(request);
2976 	cfiscsi_pdu_queue(response);
2977 
2978 	if (cold_reset) {
2979 		softc = cs->cs_target->ct_softc;
2980 		mtx_lock(&softc->lock);
2981 		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
2982 			if (tcs->cs_target == cs->cs_target)
2983 				cfiscsi_session_terminate(tcs);
2984 		}
2985 		mtx_unlock(&softc->lock);
2986 	}
2987 }
2988 
2989 static void
2990 cfiscsi_done(union ctl_io *io)
2991 {
2992 	struct icl_pdu *request;
2993 	struct cfiscsi_session *cs;
2994 
2995 	KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2996 		("invalid CTL status %#x", io->io_hdr.status));
2997 
2998 	if (io->io_hdr.io_type == CTL_IO_TASK &&
2999 	    io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
3000 		/*
3001 		 * Implicit task termination has just completed; nothing to do.
3002 		 */
3003 		cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
3004 		cs->cs_tasks_aborted = true;
3005 		refcount_release(&cs->cs_outstanding_ctl_pdus);
3006 		wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3007 		ctl_free_io(io);
3008 		return;
3009 	}
3010 
3011 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
3012 	cs = PDU_SESSION(request);
3013 
3014 	switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3015 	case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3016 		cfiscsi_scsi_command_done(io);
3017 		break;
3018 	case ISCSI_BHS_OPCODE_TASK_REQUEST:
3019 		cfiscsi_task_management_done(io);
3020 		break;
3021 	default:
3022 		panic("cfiscsi_done called with wrong opcode 0x%x",
3023 		    request->ip_bhs->bhs_opcode);
3024 	}
3025 
3026 	refcount_release(&cs->cs_outstanding_ctl_pdus);
3027 }
3028