xref: /freebsd/sys/dev/iscsi/iscsi.c (revision a223d3ed90bfe313ce5987d468a25a915d7d1254)
1 /*-
2  * Copyright (c) 2012 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Edward Tomasz Napierala under sponsorship
6  * from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #include <sys/param.h>
33 #include <sys/condvar.h>
34 #include <sys/conf.h>
35 #include <sys/endian.h>
36 #include <sys/eventhandler.h>
37 #include <sys/file.h>
38 #include <sys/kernel.h>
39 #include <sys/kthread.h>
40 #include <sys/lock.h>
41 #include <sys/malloc.h>
42 #include <sys/mutex.h>
43 #include <sys/module.h>
44 #include <sys/sysctl.h>
45 #include <sys/systm.h>
46 #include <sys/sx.h>
47 #include <vm/uma.h>
48 
49 #include <cam/cam.h>
50 #include <cam/cam_ccb.h>
51 #include <cam/cam_xpt.h>
52 #include <cam/cam_debug.h>
53 #include <cam/cam_sim.h>
54 #include <cam/cam_xpt_sim.h>
55 #include <cam/cam_xpt_periph.h>
56 #include <cam/cam_periph.h>
57 #include <cam/scsi/scsi_all.h>
58 #include <cam/scsi/scsi_message.h>
59 
60 #include "iscsi_ioctl.h"
61 #include "iscsi.h"
62 #include "icl.h"
63 #include "iscsi_proto.h"
64 
65 #ifdef ICL_KERNEL_PROXY
66 #include <sys/socketvar.h>
67 #endif
68 
69 #ifdef ICL_KERNEL_PROXY
70 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
71 #endif
72 
73 /*
74  * XXX: This is global so the iscsi_unload() can access it.
75  * 	Think about how to do this properly.
76  */
77 static struct iscsi_softc	*sc;
78 
79 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
80 static int debug = 1;
81 TUNABLE_INT("kern.iscsi.debug", &debug);
82 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
83     &debug, 0, "Enable debug messages");
84 static int ping_timeout = 5;
85 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
86 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
87     0, "Timeout for ping (NOP-Out) requests, in seconds");
88 static int iscsid_timeout = 60;
89 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
90 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
91     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
92 static int login_timeout = 60;
93 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
94 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
95     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
96 static int maxtags = 255;
97 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
98 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
99     0, "Max number of IO requests queued");
100 static int fail_on_disconnection = 0;
101 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
102 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
103     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
104 
105 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
106 static uma_zone_t iscsi_outstanding_zone;
107 
108 #define	CONN_SESSION(X)	((struct iscsi_session *)X->ic_prv0)
109 #define	PDU_SESSION(X)	(CONN_SESSION(X->ip_conn))
110 
111 #define	ISCSI_DEBUG(X, ...)						\
112 	do {								\
113 		if (debug > 1) 						\
114 			printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
115 	} while (0)
116 
117 #define	ISCSI_WARN(X, ...)						\
118 	do {								\
119 		if (debug > 0) {					\
120 			printf("WARNING: %s: " X "\n",			\
121 			    __func__, ## __VA_ARGS__);			\
122 		}							\
123 	} while (0)
124 
125 #define	ISCSI_SESSION_DEBUG(S, X, ...)					\
126 	do {								\
127 		if (debug > 1) {					\
128 			printf("%s: %s (%s): " X "\n",			\
129 			    __func__, S->is_conf.isc_target_addr,	\
130 			    S->is_conf.isc_target, ## __VA_ARGS__);	\
131 		}							\
132 	} while (0)
133 
134 #define	ISCSI_SESSION_WARN(S, X, ...)					\
135 	do {								\
136 		if (debug > 0) {					\
137 			printf("WARNING: %s (%s): " X "\n",		\
138 			    S->is_conf.isc_target_addr,			\
139 			    S->is_conf.isc_target, ## __VA_ARGS__);	\
140 		}							\
141 	} while (0)
142 
143 #define ISCSI_SESSION_LOCK(X)		mtx_lock(&X->is_lock)
144 #define ISCSI_SESSION_UNLOCK(X)		mtx_unlock(&X->is_lock)
145 #define ISCSI_SESSION_LOCK_ASSERT(X)	mtx_assert(&X->is_lock, MA_OWNED)
146 
147 static int	iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
148 		    int mode, struct thread *td);
149 
150 static struct cdevsw iscsi_cdevsw = {
151      .d_version = D_VERSION,
152      .d_ioctl   = iscsi_ioctl,
153      .d_name    = "iscsi",
154 };
155 
156 static void	iscsi_pdu_queue_locked(struct icl_pdu *request);
157 static void	iscsi_pdu_queue(struct icl_pdu *request);
158 static void	iscsi_pdu_update_statsn(const struct icl_pdu *response);
159 static void	iscsi_pdu_handle_nop_in(struct icl_pdu *response);
160 static void	iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
161 static void	iscsi_pdu_handle_data_in(struct icl_pdu *response);
162 static void	iscsi_pdu_handle_logout_response(struct icl_pdu *response);
163 static void	iscsi_pdu_handle_r2t(struct icl_pdu *response);
164 static void	iscsi_pdu_handle_async_message(struct icl_pdu *response);
165 static void	iscsi_pdu_handle_reject(struct icl_pdu *response);
166 static void	iscsi_session_reconnect(struct iscsi_session *is);
167 static void	iscsi_session_terminate(struct iscsi_session *is);
168 static void	iscsi_action(struct cam_sim *sim, union ccb *ccb);
169 static void	iscsi_poll(struct cam_sim *sim);
170 static struct iscsi_outstanding	*iscsi_outstanding_find(struct iscsi_session *is,
171 		    uint32_t initiator_task_tag);
172 static int	iscsi_outstanding_add(struct iscsi_session *is,
173 		    uint32_t initiator_task_tag, union ccb *ccb);
174 static void	iscsi_outstanding_remove(struct iscsi_session *is,
175 		    struct iscsi_outstanding *io);
176 
177 static bool
178 iscsi_pdu_prepare(struct icl_pdu *request)
179 {
180 	struct iscsi_session *is;
181 	struct iscsi_bhs_scsi_command *bhssc;
182 
183 	is = PDU_SESSION(request);
184 
185 	ISCSI_SESSION_LOCK_ASSERT(is);
186 
187 	/*
188 	 * We're only using fields common for all the request
189 	 * (initiator -> target) PDUs.
190 	 */
191 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
192 
193 	/*
194 	 * Data-Out PDU does not contain CmdSN.
195 	 */
196 	if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
197 		if (is->is_cmdsn > is->is_maxcmdsn &&
198 		    (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
199 			/*
200 			 * Current MaxCmdSN prevents us from sending any more
201 			 * SCSI Command PDUs to the target; postpone the PDU.
202 			 * It will get resent by either iscsi_pdu_queue(),
203 			 * or by maintenance thread.
204 			 */
205 #if 0
206 			ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %d, ExpCmdSN %d, MaxCmdSN %d, opcode 0x%x",
207 			    is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode);
208 #endif
209 			return (true);
210 		}
211 		bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
212 		if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
213 			is->is_cmdsn++;
214 	}
215 	bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
216 
217 	return (false);
218 }
219 
220 static void
221 iscsi_session_send_postponed(struct iscsi_session *is)
222 {
223 	struct icl_pdu *request;
224 	bool postpone;
225 
226 	ISCSI_SESSION_LOCK_ASSERT(is);
227 
228 	while (!STAILQ_EMPTY(&is->is_postponed)) {
229 		request = STAILQ_FIRST(&is->is_postponed);
230 		postpone = iscsi_pdu_prepare(request);
231 		if (postpone)
232 			break;
233 		STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
234 		icl_pdu_queue(request);
235 	}
236 }
237 
238 static void
239 iscsi_pdu_queue_locked(struct icl_pdu *request)
240 {
241 	struct iscsi_session *is;
242 	bool postpone;
243 
244 	is = PDU_SESSION(request);
245 	ISCSI_SESSION_LOCK_ASSERT(is);
246 	iscsi_session_send_postponed(is);
247 	postpone = iscsi_pdu_prepare(request);
248 	if (postpone) {
249 		STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
250 		return;
251 	}
252 	icl_pdu_queue(request);
253 }
254 
255 static void
256 iscsi_pdu_queue(struct icl_pdu *request)
257 {
258 	struct iscsi_session *is;
259 
260 	is = PDU_SESSION(request);
261 	ISCSI_SESSION_LOCK(is);
262 	iscsi_pdu_queue_locked(request);
263 	ISCSI_SESSION_UNLOCK(is);
264 }
265 
266 static void
267 iscsi_session_logout(struct iscsi_session *is)
268 {
269 	struct icl_pdu *request;
270 	struct iscsi_bhs_logout_request *bhslr;
271 
272 	request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
273 	if (request == NULL)
274 		return;
275 
276 	bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
277 	bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
278 	bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
279 	iscsi_pdu_queue_locked(request);
280 }
281 
282 static void
283 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
284 {
285 	struct iscsi_outstanding *io, *tmp;
286 
287 	ISCSI_SESSION_LOCK_ASSERT(is);
288 
289 	TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
290 		if (requeue) {
291 			io->io_ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
292 			io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
293 		} else {
294 			io->io_ccb->ccb_h.status = CAM_REQ_ABORTED;
295 		}
296 
297 		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
298 			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
299 			ISCSI_SESSION_DEBUG(is, "freezing devq");
300 		}
301 		io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
302 		xpt_done(io->io_ccb);
303 		iscsi_outstanding_remove(is, io);
304 	}
305 }
306 
307 static void
308 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
309 {
310 	struct icl_pdu *pdu;
311 
312 	ISCSI_SESSION_LOCK_ASSERT(is);
313 
314 	/*
315 	 * Don't queue any new PDUs.
316 	 */
317 	if (is->is_sim != NULL && is->is_simq_frozen == false) {
318 		ISCSI_SESSION_DEBUG(is, "freezing");
319 		xpt_freeze_simq(is->is_sim, 1);
320 		is->is_simq_frozen = true;
321 	}
322 
323 	/*
324 	 * Remove postponed PDUs.
325 	 */
326 	while (!STAILQ_EMPTY(&is->is_postponed)) {
327 		pdu = STAILQ_FIRST(&is->is_postponed);
328 		STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
329 		icl_pdu_free(pdu);
330 	}
331 
332 	if (destroy_sim == false) {
333 		/*
334 		 * Terminate SCSI tasks, asking CAM to requeue them.
335 		 */
336 		iscsi_session_terminate_tasks(is, true);
337 		return;
338 	}
339 
340 	iscsi_session_terminate_tasks(is, false);
341 
342 	if (is->is_sim == NULL)
343 		return;
344 
345 	ISCSI_SESSION_DEBUG(is, "deregistering SIM");
346 	xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
347 
348 	if (is->is_simq_frozen) {
349 		xpt_release_simq(is->is_sim, 1);
350 		is->is_simq_frozen = false;
351 	}
352 
353 	xpt_free_path(is->is_path);
354 	is->is_path = NULL;
355 	xpt_bus_deregister(cam_sim_path(is->is_sim));
356 	cam_sim_free(is->is_sim, TRUE /*free_devq*/);
357 	is->is_sim = NULL;
358 	is->is_devq = NULL;
359 }
360 
361 static void
362 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
363 {
364 
365 	icl_conn_shutdown(is->is_conn);
366 	icl_conn_close(is->is_conn);
367 
368 	ISCSI_SESSION_LOCK(is);
369 
370 	is->is_connected = false;
371 	is->is_reconnecting = false;
372 	is->is_login_phase = false;
373 
374 #ifdef ICL_KERNEL_PROXY
375 	if (is->is_login_pdu != NULL) {
376 		icl_pdu_free(is->is_login_pdu);
377 		is->is_login_pdu = NULL;
378 	}
379 	cv_signal(&is->is_login_cv);
380 #endif
381 
382 	if (fail_on_disconnection) {
383 		ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
384 		iscsi_session_cleanup(is, true);
385 	} else {
386 		iscsi_session_cleanup(is, false);
387 	}
388 
389 	KASSERT(TAILQ_EMPTY(&is->is_outstanding),
390 	    ("destroying session with active tasks"));
391 	KASSERT(STAILQ_EMPTY(&is->is_postponed),
392 	    ("destroying session with postponed PDUs"));
393 
394 	/*
395 	 * Request immediate reconnection from iscsid(8).
396 	 */
397 	//ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
398 	is->is_waiting_for_iscsid = true;
399 	strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
400 	is->is_timeout = 0;
401 	ISCSI_SESSION_UNLOCK(is);
402 	cv_signal(&is->is_softc->sc_cv);
403 }
404 
405 static void
406 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
407 {
408 	struct iscsi_softc *sc;
409 
410 	sc = is->is_softc;
411 	sx_xlock(&sc->sc_lock);
412 	TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
413 	sx_xunlock(&sc->sc_lock);
414 
415 	icl_conn_close(is->is_conn);
416 
417 	ISCSI_SESSION_LOCK(is);
418 
419 	KASSERT(is->is_terminating, ("is_terminating == false"));
420 
421 #ifdef ICL_KERNEL_PROXY
422 	if (is->is_login_pdu != NULL) {
423 		icl_pdu_free(is->is_login_pdu);
424 		is->is_login_pdu = NULL;
425 	}
426 	cv_signal(&is->is_login_cv);
427 #endif
428 
429 	callout_drain(&is->is_callout);
430 
431 	iscsi_session_cleanup(is, true);
432 
433 	KASSERT(TAILQ_EMPTY(&is->is_outstanding),
434 	    ("destroying session with active tasks"));
435 	KASSERT(STAILQ_EMPTY(&is->is_postponed),
436 	    ("destroying session with postponed PDUs"));
437 
438 	ISCSI_SESSION_UNLOCK(is);
439 
440 	icl_conn_free(is->is_conn);
441 	mtx_destroy(&is->is_lock);
442 	cv_destroy(&is->is_maintenance_cv);
443 #ifdef ICL_KERNEL_PROXY
444 	cv_destroy(&is->is_login_cv);
445 #endif
446 	ISCSI_SESSION_DEBUG(is, "terminated");
447 	free(is, M_ISCSI);
448 
449 	/*
450 	 * The iscsi_unload() routine might be waiting.
451 	 */
452 	cv_signal(&sc->sc_cv);
453 }
454 
455 static void
456 iscsi_maintenance_thread(void *arg)
457 {
458 	struct iscsi_session *is;
459 
460 	is = arg;
461 
462 	for (;;) {
463 		ISCSI_SESSION_LOCK(is);
464 		if (is->is_reconnecting == false &&
465 		    is->is_terminating == false &&
466 		    STAILQ_EMPTY(&is->is_postponed))
467 			cv_wait(&is->is_maintenance_cv, &is->is_lock);
468 
469 		if (is->is_reconnecting) {
470 			ISCSI_SESSION_UNLOCK(is);
471 			iscsi_maintenance_thread_reconnect(is);
472 			continue;
473 		}
474 
475 		if (is->is_terminating) {
476 			ISCSI_SESSION_UNLOCK(is);
477 			iscsi_maintenance_thread_terminate(is);
478 			kthread_exit();
479 			return;
480 		}
481 
482 		iscsi_session_send_postponed(is);
483 		ISCSI_SESSION_UNLOCK(is);
484 	}
485 }
486 
487 static void
488 iscsi_session_reconnect(struct iscsi_session *is)
489 {
490 
491 	/*
492 	 * XXX: We can't use locking here, because
493 	 * 	it's being called from various contexts.
494 	 * 	Hope it doesn't break anything.
495 	 */
496 	if (is->is_reconnecting)
497 		return;
498 
499 	is->is_reconnecting = true;
500 	cv_signal(&is->is_maintenance_cv);
501 }
502 
503 static void
504 iscsi_session_terminate(struct iscsi_session *is)
505 {
506 	if (is->is_terminating)
507 		return;
508 
509 	is->is_terminating = true;
510 
511 #if 0
512 	iscsi_session_logout(is);
513 #endif
514 	cv_signal(&is->is_maintenance_cv);
515 }
516 
517 static void
518 iscsi_callout(void *context)
519 {
520 	struct icl_pdu *request;
521 	struct iscsi_bhs_nop_out *bhsno;
522 	struct iscsi_session *is;
523 	bool reconnect_needed = false;
524 
525 	is = context;
526 
527 	if (is->is_terminating)
528 		return;
529 
530 	callout_schedule(&is->is_callout, 1 * hz);
531 
532 	ISCSI_SESSION_LOCK(is);
533 	is->is_timeout++;
534 
535 	if (is->is_waiting_for_iscsid) {
536 		if (is->is_timeout > iscsid_timeout) {
537 			ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
538 			    "for %d seconds; reconnecting",
539 			    is->is_timeout);
540 			reconnect_needed = true;
541 		}
542 		goto out;
543 	}
544 
545 	if (is->is_login_phase) {
546 		if (is->is_timeout > login_timeout) {
547 			ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
548 			    "reconnecting", is->is_timeout);
549 			reconnect_needed = true;
550 		}
551 		goto out;
552 	}
553 
554 	if (is->is_timeout >= ping_timeout) {
555 		ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
556 		    "reconnecting", ping_timeout);
557 		reconnect_needed = true;
558 		goto out;
559 	}
560 
561 	ISCSI_SESSION_UNLOCK(is);
562 
563 	/*
564 	 * If the ping was reset less than one second ago - which means
565 	 * that we've received some PDU during the last second - assume
566 	 * the traffic flows correctly and don't bother sending a NOP-Out.
567 	 *
568 	 * (It's 2 - one for one second, and one for incrementing is_timeout
569 	 * earlier in this routine.)
570 	 */
571 	if (is->is_timeout < 2)
572 		return;
573 
574 	request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
575 	if (request == NULL) {
576 		ISCSI_SESSION_WARN(is, "failed to allocate PDU");
577 		return;
578 	}
579 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
580 	bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
581 	    ISCSI_BHS_OPCODE_IMMEDIATE;
582 	bhsno->bhsno_flags = 0x80;
583 	bhsno->bhsno_target_transfer_tag = 0xffffffff;
584 	iscsi_pdu_queue(request);
585 	return;
586 
587 out:
588 	ISCSI_SESSION_UNLOCK(is);
589 
590 	if (reconnect_needed)
591 		iscsi_session_reconnect(is);
592 }
593 
594 static void
595 iscsi_pdu_update_statsn(const struct icl_pdu *response)
596 {
597 	const struct iscsi_bhs_data_in *bhsdi;
598 	struct iscsi_session *is;
599 	uint32_t expcmdsn, maxcmdsn;
600 
601 	is = PDU_SESSION(response);
602 
603 	ISCSI_SESSION_LOCK_ASSERT(is);
604 
605 	/*
606 	 * We're only using fields common for all the response
607 	 * (target -> initiator) PDUs.
608 	 */
609 	bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
610 	/*
611 	 * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
612 	 * and Residual Count only have meaningful content if the S bit
613 	 * is set to 1", so we also need to check the bit specific for
614 	 * Data-In PDU.
615 	 */
616 	if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
617 	    (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
618 		if (ntohl(bhsdi->bhsdi_statsn) < is->is_statsn) {
619 			ISCSI_SESSION_WARN(is,
620 			    "PDU StatSN %d >= session StatSN %d, opcode 0x%x",
621 			    is->is_statsn, ntohl(bhsdi->bhsdi_statsn),
622 			    bhsdi->bhsdi_opcode);
623 		}
624 		is->is_statsn = ntohl(bhsdi->bhsdi_statsn);
625 	}
626 
627 	expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
628 	maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
629 
630 	/*
631 	 * XXX: Compare using Serial Arithmetic Sense.
632 	 */
633 	if (maxcmdsn + 1 < expcmdsn) {
634 		ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d + 1 < PDU ExpCmdSN %d; ignoring",
635 		    maxcmdsn, expcmdsn);
636 	} else {
637 		if (maxcmdsn > is->is_maxcmdsn) {
638 			is->is_maxcmdsn = maxcmdsn;
639 
640 			/*
641 			 * Command window increased; kick the maintanance thread
642 			 * to send out postponed commands.
643 			 */
644 			if (!STAILQ_EMPTY(&is->is_postponed))
645 				cv_signal(&is->is_maintenance_cv);
646 		} else if (maxcmdsn < is->is_maxcmdsn) {
647 			ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d < session MaxCmdSN %d; ignoring",
648 			    maxcmdsn, is->is_maxcmdsn);
649 		}
650 
651 		if (expcmdsn > is->is_expcmdsn) {
652 			is->is_expcmdsn = expcmdsn;
653 		} else if (expcmdsn < is->is_expcmdsn) {
654 			ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %d < session ExpCmdSN %d; ignoring",
655 			    expcmdsn, is->is_expcmdsn);
656 		}
657 	}
658 
659 	/*
660 	 * Every incoming PDU - not just NOP-In - resets the ping timer.
661 	 * The purpose of the timeout is to reset the connection when it stalls;
662 	 * we don't want this to happen when NOP-In or NOP-Out ends up delayed
663 	 * in some queue.
664 	 */
665 	is->is_timeout = 0;
666 }
667 
668 static void
669 iscsi_receive_callback(struct icl_pdu *response)
670 {
671 	struct iscsi_session *is;
672 
673 	is = PDU_SESSION(response);
674 
675 	ISCSI_SESSION_LOCK(is);
676 
677 #ifdef ICL_KERNEL_PROXY
678 	if (is->is_login_phase) {
679 		if (is->is_login_pdu == NULL)
680 			is->is_login_pdu = response;
681 		else
682 			icl_pdu_free(response);
683 		ISCSI_SESSION_UNLOCK(is);
684 		cv_signal(&is->is_login_cv);
685 		return;
686 	}
687 #endif
688 
689 	iscsi_pdu_update_statsn(response);
690 
691 	/*
692 	 * The handling routine is responsible for freeing the PDU
693 	 * when it's no longer needed.
694 	 */
695 	switch (response->ip_bhs->bhs_opcode) {
696 	case ISCSI_BHS_OPCODE_NOP_IN:
697 		iscsi_pdu_handle_nop_in(response);
698 		break;
699 	case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
700 		iscsi_pdu_handle_scsi_response(response);
701 		break;
702 	case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
703 		iscsi_pdu_handle_data_in(response);
704 		break;
705 	case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
706 		iscsi_pdu_handle_logout_response(response);
707 		break;
708 	case ISCSI_BHS_OPCODE_R2T:
709 		iscsi_pdu_handle_r2t(response);
710 		break;
711 	case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
712 		iscsi_pdu_handle_async_message(response);
713 		break;
714 	case ISCSI_BHS_OPCODE_REJECT:
715 		iscsi_pdu_handle_reject(response);
716 		break;
717 	default:
718 		ISCSI_SESSION_WARN(is, "received PDU with unsupported "
719 		    "opcode 0x%x; reconnecting",
720 		    response->ip_bhs->bhs_opcode);
721 		iscsi_session_reconnect(is);
722 		icl_pdu_free(response);
723 	}
724 
725 	ISCSI_SESSION_UNLOCK(is);
726 }
727 
728 static void
729 iscsi_error_callback(struct icl_conn *ic)
730 {
731 	struct iscsi_session *is;
732 
733 	is = CONN_SESSION(ic);
734 
735 	ISCSI_SESSION_WARN(is, "connection error; reconnecting");
736 	iscsi_session_reconnect(is);
737 }
738 
739 static void
740 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
741 {
742 	struct iscsi_session *is;
743 	struct iscsi_bhs_nop_out *bhsno;
744 	struct iscsi_bhs_nop_in *bhsni;
745 	struct icl_pdu *request;
746 	void *data = NULL;
747 	size_t datasize;
748 	int error;
749 
750 	is = PDU_SESSION(response);
751 	bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
752 
753 	if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
754 		/*
755 		 * Nothing to do; iscsi_pdu_update_statsn() already
756 		 * zeroed the timeout.
757 		 */
758 		icl_pdu_free(response);
759 		return;
760 	}
761 
762 	datasize = icl_pdu_data_segment_length(response);
763 	if (datasize > 0) {
764 		data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
765 		if (data == NULL) {
766 			ISCSI_SESSION_WARN(is, "failed to allocate memory; "
767 			    "reconnecting");
768 			icl_pdu_free(response);
769 			iscsi_session_reconnect(is);
770 			return;
771 		}
772 		icl_pdu_get_data(response, 0, data, datasize);
773 	}
774 
775 	request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
776 	if (request == NULL) {
777 		ISCSI_SESSION_WARN(is, "failed to allocate memory; "
778 		    "reconnecting");
779 		free(data, M_ISCSI);
780 		icl_pdu_free(response);
781 		iscsi_session_reconnect(is);
782 		return;
783 	}
784 	bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
785 	bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
786 	    ISCSI_BHS_OPCODE_IMMEDIATE;
787 	bhsno->bhsno_flags = 0x80;
788 	bhsno->bhsno_initiator_task_tag = 0xffffffff;
789 	bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
790 	if (datasize > 0) {
791 		error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
792 		if (error != 0) {
793 			ISCSI_SESSION_WARN(is, "failed to allocate memory; "
794 			    "reconnecting");
795 			free(data, M_ISCSI);
796 			icl_pdu_free(request);
797 			icl_pdu_free(response);
798 			iscsi_session_reconnect(is);
799 			return;
800 		}
801 		free(data, M_ISCSI);
802 	}
803 
804 	icl_pdu_free(response);
805 	iscsi_pdu_queue_locked(request);
806 }
807 
808 static void
809 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
810 {
811 	struct iscsi_bhs_scsi_response *bhssr;
812 	struct iscsi_outstanding *io;
813 	struct iscsi_session *is;
814 	struct ccb_scsiio *csio;
815 	size_t data_segment_len;
816 	uint16_t sense_len;
817 
818 	is = PDU_SESSION(response);
819 
820 	bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
821 	io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
822 	if (io == NULL) {
823 		ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
824 		icl_pdu_free(response);
825 		iscsi_session_reconnect(is);
826 		return;
827 	}
828 
829 	if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
830 		ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
831  		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
832  			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
833 			ISCSI_SESSION_DEBUG(is, "freezing devq");
834 		}
835  		io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
836 	} else if (bhssr->bhssr_status == 0) {
837 		io->io_ccb->ccb_h.status = CAM_REQ_CMP;
838 	} else {
839  		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
840  			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
841 			ISCSI_SESSION_DEBUG(is, "freezing devq");
842 		}
843  		io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
844 		io->io_ccb->csio.scsi_status = bhssr->bhssr_status;
845 	}
846 
847 	if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
848 		ISCSI_SESSION_WARN(is, "target indicated residual overflow");
849 		icl_pdu_free(response);
850 		iscsi_session_reconnect(is);
851 		return;
852 	}
853 
854 	csio = &io->io_ccb->csio;
855 
856 	data_segment_len = icl_pdu_data_segment_length(response);
857 	if (data_segment_len > 0) {
858 		if (data_segment_len < sizeof(sense_len)) {
859 			ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
860 			    data_segment_len);
861 			if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
862 				xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
863 				ISCSI_SESSION_DEBUG(is, "freezing devq");
864 			}
865 			io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
866 			goto out;
867 		}
868 		icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
869 		sense_len = ntohs(sense_len);
870 #if 0
871 		ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
872 		    sense_len, data_segment_len);
873 #endif
874 		if (sizeof(sense_len) + sense_len > data_segment_len) {
875 			ISCSI_SESSION_WARN(is, "truncated data segment "
876 			    "(%zd bytes, should be %zd)",
877 			    data_segment_len, sizeof(sense_len) + sense_len);
878 			if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
879 				xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
880 				ISCSI_SESSION_DEBUG(is, "freezing devq");
881 			}
882 			io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
883 			goto out;
884 		} else if (sizeof(sense_len) + sense_len < data_segment_len)
885 			ISCSI_SESSION_WARN(is, "oversize data segment "
886 			    "(%zd bytes, should be %zd)",
887 			    data_segment_len, sizeof(sense_len) + sense_len);
888 		if (sense_len > csio->sense_len) {
889 			ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
890 			    sense_len, csio->sense_len);
891 			sense_len = csio->sense_len;
892 		}
893 		icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
894 		csio->sense_resid = csio->sense_len - sense_len;
895 		io->io_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
896 	}
897 
898 out:
899 	if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
900 		csio->resid = ntohl(bhssr->bhssr_residual_count);
901 
902 	if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
903 		KASSERT(io->io_received <= csio->dxfer_len,
904 		    ("io->io_received > csio->dxfer_len"));
905 		if (io->io_received < csio->dxfer_len) {
906 			if (csio->resid != csio->dxfer_len - io->io_received) {
907 				ISCSI_SESSION_WARN(is, "underflow mismatch: "
908 				    "target indicates %d, we calculated %zd",
909 				    csio->resid,
910 				    csio->dxfer_len - io->io_received);
911 			}
912 			csio->resid = csio->dxfer_len - io->io_received;
913 		}
914 	}
915 
916 	xpt_done(io->io_ccb);
917 	iscsi_outstanding_remove(is, io);
918 	icl_pdu_free(response);
919 }
920 
921 static void
922 iscsi_pdu_handle_data_in(struct icl_pdu *response)
923 {
924 	struct iscsi_bhs_data_in *bhsdi;
925 	struct iscsi_outstanding *io;
926 	struct iscsi_session *is;
927 	struct ccb_scsiio *csio;
928 	size_t data_segment_len;
929 
930 	is = PDU_SESSION(response);
931 	bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
932 	io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
933 	if (io == NULL) {
934 		ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
935 		icl_pdu_free(response);
936 		iscsi_session_reconnect(is);
937 		return;
938 	}
939 
940 	data_segment_len = icl_pdu_data_segment_length(response);
941 	if (data_segment_len == 0) {
942 		/*
943 		 * "The sending of 0 length data segments should be avoided,
944 		 * but initiators and targets MUST be able to properly receive
945 		 * 0 length data segments."
946 		 */
947 		icl_pdu_free(response);
948 		return;
949 	}
950 
951 	/*
952 	 * We need to track this for security reasons - without it, malicious target
953 	 * could respond to SCSI READ without sending Data-In PDUs, which would result
954 	 * in read operation on the initiator side returning random kernel data.
955 	 */
956 	if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
957 		ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
958 		    io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
959 		icl_pdu_free(response);
960 		iscsi_session_reconnect(is);
961 		return;
962 	}
963 
964 	csio = &io->io_ccb->csio;
965 
966 	if (io->io_received + data_segment_len > csio->dxfer_len) {
967 		ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
968 		    "at offset %zd, buffer is %d)",
969 		    data_segment_len, io->io_received, csio->dxfer_len);
970 		icl_pdu_free(response);
971 		iscsi_session_reconnect(is);
972 		return;
973 	}
974 
975 	icl_pdu_get_data(response, 0, csio->data_ptr + io->io_received, data_segment_len);
976 	io->io_received += data_segment_len;
977 
978 	/*
979 	 * XXX: Check DataSN.
980 	 * XXX: Check F.
981 	 */
982 	if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
983 		/*
984 		 * Nothing more to do.
985 		 */
986 		icl_pdu_free(response);
987 		return;
988 	}
989 
990 	//ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
991 	if (bhsdi->bhsdi_status == 0) {
992 		io->io_ccb->ccb_h.status = CAM_REQ_CMP;
993 	} else {
994 		if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
995 			xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
996 			ISCSI_SESSION_DEBUG(is, "freezing devq");
997 		}
998 		io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
999 		csio->scsi_status = bhsdi->bhsdi_status;
1000 	}
1001 
1002 	if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1003 		KASSERT(io->io_received <= csio->dxfer_len,
1004 		    ("io->io_received > csio->dxfer_len"));
1005 		if (io->io_received < csio->dxfer_len) {
1006 			csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1007 			if (csio->resid != csio->dxfer_len - io->io_received) {
1008 				ISCSI_SESSION_WARN(is, "underflow mismatch: "
1009 				    "target indicates %d, we calculated %zd",
1010 				    csio->resid,
1011 				    csio->dxfer_len - io->io_received);
1012 			}
1013 			csio->resid = csio->dxfer_len - io->io_received;
1014 		}
1015 	}
1016 
1017 	xpt_done(io->io_ccb);
1018 	iscsi_outstanding_remove(is, io);
1019 	icl_pdu_free(response);
1020 }
1021 
1022 static void
1023 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1024 {
1025 
1026 	ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1027 	icl_pdu_free(response);
1028 }
1029 
1030 static void
1031 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1032 {
1033 	struct icl_pdu *request;
1034 	struct iscsi_session *is;
1035 	struct iscsi_bhs_r2t *bhsr2t;
1036 	struct iscsi_bhs_data_out *bhsdo;
1037 	struct iscsi_outstanding *io;
1038 	struct ccb_scsiio *csio;
1039 	size_t off, len, total_len;
1040 	int error;
1041 
1042 	is = PDU_SESSION(response);
1043 
1044 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1045 	io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1046 	if (io == NULL) {
1047 		ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1048 		    bhsr2t->bhsr2t_initiator_task_tag);
1049 		icl_pdu_free(response);
1050 		iscsi_session_reconnect(is);
1051 		return;
1052 	}
1053 
1054 	csio = &io->io_ccb->csio;
1055 
1056 	if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1057 		ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1058 		icl_pdu_free(response);
1059 		iscsi_session_reconnect(is);
1060 		return;
1061 	}
1062 
1063 	/*
1064 	 * XXX: Verify R2TSN.
1065 	 */
1066 
1067 	io->io_datasn = 0;
1068 
1069 	off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1070 	if (off > csio->dxfer_len) {
1071 		ISCSI_SESSION_WARN(is, "target requested invalid offset "
1072 		    "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1073 		icl_pdu_free(response);
1074 		iscsi_session_reconnect(is);
1075 		return;
1076 	}
1077 
1078 	total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1079 	if (total_len == 0 || total_len > csio->dxfer_len) {
1080 		ISCSI_SESSION_WARN(is, "target requested invalid length "
1081 		    "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1082 		icl_pdu_free(response);
1083 		iscsi_session_reconnect(is);
1084 		return;
1085 	}
1086 
1087 	//ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1088 
1089 	for (;;) {
1090 		len = total_len;
1091 
1092 		if (len > is->is_max_data_segment_length)
1093 			len = is->is_max_data_segment_length;
1094 
1095 		if (off + len > csio->dxfer_len) {
1096 			ISCSI_SESSION_WARN(is, "target requested invalid "
1097 			    "length/offset %zd, buffer is %d; reconnecting",
1098 			    off + len, csio->dxfer_len);
1099 			icl_pdu_free(response);
1100 			iscsi_session_reconnect(is);
1101 			return;
1102 		}
1103 
1104 		request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
1105 		if (request == NULL) {
1106 			icl_pdu_free(response);
1107 			iscsi_session_reconnect(is);
1108 			return;
1109 		}
1110 
1111 		bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1112 		bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1113 		bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1114 		bhsdo->bhsdo_initiator_task_tag =
1115 		    bhsr2t->bhsr2t_initiator_task_tag;
1116 		bhsdo->bhsdo_target_transfer_tag =
1117 		    bhsr2t->bhsr2t_target_transfer_tag;
1118 		bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1119 		bhsdo->bhsdo_buffer_offset = htonl(off);
1120 		error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1121 		    M_NOWAIT);
1122 		if (error != 0) {
1123 			ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1124 			    "reconnecting");
1125 			icl_pdu_free(request);
1126 			icl_pdu_free(response);
1127 			iscsi_session_reconnect(is);
1128 			return;
1129 		}
1130 
1131 		off += len;
1132 		total_len -= len;
1133 
1134 		if (total_len == 0) {
1135 			bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1136 			//ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1137 		} else {
1138 			//ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1139 		}
1140 
1141 		iscsi_pdu_queue_locked(request);
1142 
1143 		if (total_len == 0)
1144 			break;
1145 	}
1146 
1147 	icl_pdu_free(response);
1148 }
1149 
1150 static void
1151 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1152 {
1153 	struct iscsi_bhs_asynchronous_message *bhsam;
1154 	struct iscsi_session *is;
1155 
1156 	is = PDU_SESSION(response);
1157 	bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1158 	switch (bhsam->bhsam_async_event) {
1159 	case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1160 		ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1161 		iscsi_session_logout(is);
1162 		iscsi_session_terminate(is);
1163 		break;
1164 	case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1165 		ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1166 		break;
1167 	case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1168 		ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1169 		break;
1170 	default:
1171 		/*
1172 		 * XXX: Technically, we're obligated to also handle
1173 		 * 	parameter renegotiation.
1174 		 */
1175 		ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1176 		break;
1177 	}
1178 
1179 	icl_pdu_free(response);
1180 }
1181 
1182 static void
1183 iscsi_pdu_handle_reject(struct icl_pdu *response)
1184 {
1185 	struct iscsi_bhs_reject *bhsr;
1186 	struct iscsi_session *is;
1187 
1188 	is = PDU_SESSION(response);
1189 	bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1190 	ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1191 	    bhsr->bhsr_reason);
1192 
1193 	icl_pdu_free(response);
1194 }
1195 
1196 static int
1197 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1198     struct iscsi_daemon_request *request)
1199 {
1200 	struct iscsi_session *is;
1201 	int error;
1202 
1203 	sx_slock(&sc->sc_lock);
1204 	for (;;) {
1205 		TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1206 			ISCSI_SESSION_LOCK(is);
1207 			if (is->is_waiting_for_iscsid)
1208 				break;
1209 			ISCSI_SESSION_UNLOCK(is);
1210 		}
1211 
1212 		if (is == NULL) {
1213 			/*
1214 			 * No session requires attention from iscsid(8); wait.
1215 			 */
1216 			error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1217 			if (error != 0) {
1218 				sx_sunlock(&sc->sc_lock);
1219 				return (error);
1220 			}
1221 			continue;
1222 		}
1223 
1224 		is->is_waiting_for_iscsid = false;
1225 		is->is_login_phase = true;
1226 		is->is_reason[0] = '\0';
1227 		ISCSI_SESSION_UNLOCK(is);
1228 
1229 		request->idr_session_id = is->is_id;
1230 		memcpy(&request->idr_conf, &is->is_conf,
1231 		    sizeof(request->idr_conf));
1232 
1233 		sx_sunlock(&sc->sc_lock);
1234 		return (0);
1235 	}
1236 }
1237 
1238 static int
1239 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1240     struct iscsi_daemon_handoff *handoff)
1241 {
1242 	struct iscsi_session *is;
1243 	int error;
1244 
1245 	sx_slock(&sc->sc_lock);
1246 
1247 	/*
1248 	 * Find the session to hand off socket to.
1249 	 */
1250 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1251 		if (is->is_id == handoff->idh_session_id)
1252 			break;
1253 	}
1254 	if (is == NULL) {
1255 		sx_sunlock(&sc->sc_lock);
1256 		return (ESRCH);
1257 	}
1258 	ISCSI_SESSION_LOCK(is);
1259 	if (is->is_conf.isc_discovery || is->is_terminating) {
1260 		ISCSI_SESSION_UNLOCK(is);
1261 		sx_sunlock(&sc->sc_lock);
1262 		return (EINVAL);
1263 	}
1264 	if (is->is_connected) {
1265 		/*
1266 		 * This might have happened because another iscsid(8)
1267 		 * instance handed off the connection in the meantime.
1268 		 * Just return.
1269 		 */
1270 		ISCSI_SESSION_WARN(is, "handoff on already connected "
1271 		    "session");
1272 		ISCSI_SESSION_UNLOCK(is);
1273 		sx_sunlock(&sc->sc_lock);
1274 		return (EBUSY);
1275 	}
1276 
1277 	strlcpy(is->is_target_alias, handoff->idh_target_alias,
1278 	    sizeof(is->is_target_alias));
1279 	memcpy(is->is_isid, handoff->idh_isid, sizeof(is->is_isid));
1280 	is->is_statsn = handoff->idh_statsn;
1281 	is->is_initial_r2t = handoff->idh_initial_r2t;
1282 	is->is_immediate_data = handoff->idh_immediate_data;
1283 	is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1284 	is->is_max_burst_length = handoff->idh_max_burst_length;
1285 	is->is_first_burst_length = handoff->idh_first_burst_length;
1286 
1287 	if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1288 		is->is_conn->ic_header_crc32c = true;
1289 	else
1290 		is->is_conn->ic_header_crc32c = false;
1291 	if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1292 		is->is_conn->ic_data_crc32c = true;
1293 	else
1294 		is->is_conn->ic_data_crc32c = false;
1295 
1296 	is->is_cmdsn = 0;
1297 	is->is_expcmdsn = 0;
1298 	is->is_maxcmdsn = 0;
1299 	is->is_waiting_for_iscsid = false;
1300 	is->is_login_phase = false;
1301 	is->is_timeout = 0;
1302 	is->is_connected = true;
1303 	is->is_reason[0] = '\0';
1304 
1305 	ISCSI_SESSION_UNLOCK(is);
1306 
1307 #ifdef ICL_KERNEL_PROXY
1308 	if (handoff->idh_socket != 0) {
1309 #endif
1310 		/*
1311 		 * Handoff without using ICL proxy.
1312 		 */
1313 		error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1314 		if (error != 0) {
1315 			sx_sunlock(&sc->sc_lock);
1316 			iscsi_session_terminate(is);
1317 			return (error);
1318 		}
1319 #ifdef ICL_KERNEL_PROXY
1320 	}
1321 #endif
1322 
1323 	sx_sunlock(&sc->sc_lock);
1324 
1325 	if (is->is_sim != NULL) {
1326 		/*
1327 		 * When reconnecting, there already is SIM allocated for the session.
1328 		 */
1329 		KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1330 		ISCSI_SESSION_LOCK(is);
1331 		ISCSI_SESSION_DEBUG(is, "releasing");
1332 		xpt_release_simq(is->is_sim, 1);
1333 		is->is_simq_frozen = false;
1334 		ISCSI_SESSION_UNLOCK(is);
1335 
1336 	} else {
1337 		ISCSI_SESSION_LOCK(is);
1338 		is->is_devq = cam_simq_alloc(maxtags);
1339 		if (is->is_devq == NULL) {
1340 			ISCSI_SESSION_WARN(is, "failed to allocate simq");
1341 			iscsi_session_terminate(is);
1342 			return (ENOMEM);
1343 		}
1344 
1345 		is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1346 		    is, is->is_id /* unit */, &is->is_lock,
1347 		    maxtags, maxtags, is->is_devq);
1348 		if (is->is_sim == NULL) {
1349 			ISCSI_SESSION_UNLOCK(is);
1350 			ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1351 			cam_simq_free(is->is_devq);
1352 			iscsi_session_terminate(is);
1353 			return (ENOMEM);
1354 		}
1355 
1356 		error = xpt_bus_register(is->is_sim, NULL, 0);
1357 		if (error != 0) {
1358 			ISCSI_SESSION_UNLOCK(is);
1359 			ISCSI_SESSION_WARN(is, "failed to register bus");
1360 			iscsi_session_terminate(is);
1361 			return (ENOMEM);
1362 		}
1363 
1364 		error = xpt_create_path(&is->is_path, /*periph*/NULL,
1365 		    cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1366 		    CAM_LUN_WILDCARD);
1367 		if (error != CAM_REQ_CMP) {
1368 			ISCSI_SESSION_UNLOCK(is);
1369 			ISCSI_SESSION_WARN(is, "failed to create path");
1370 			iscsi_session_terminate(is);
1371 			return (ENOMEM);
1372 		}
1373 		ISCSI_SESSION_UNLOCK(is);
1374 	}
1375 
1376 	return (0);
1377 }
1378 
1379 static int
1380 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1381     struct iscsi_daemon_fail *fail)
1382 {
1383 	struct iscsi_session *is;
1384 
1385 	sx_slock(&sc->sc_lock);
1386 
1387 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1388 		if (is->is_id == fail->idf_session_id)
1389 			break;
1390 	}
1391 	if (is == NULL) {
1392 		sx_sunlock(&sc->sc_lock);
1393 		return (ESRCH);
1394 	}
1395 	ISCSI_SESSION_LOCK(is);
1396 	ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1397 	    fail->idf_reason);
1398 	strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1399 	//is->is_waiting_for_iscsid = false;
1400 	//is->is_login_phase = true;
1401 	//iscsi_session_reconnect(is);
1402 	ISCSI_SESSION_UNLOCK(is);
1403 	sx_sunlock(&sc->sc_lock);
1404 
1405 	return (0);
1406 }
1407 
1408 #ifdef ICL_KERNEL_PROXY
1409 static int
1410 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1411     struct iscsi_daemon_connect *idc)
1412 {
1413 	struct iscsi_session *is;
1414 	struct sockaddr *from_sa, *to_sa;
1415 	int error;
1416 
1417 	sx_slock(&sc->sc_lock);
1418 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1419 		if (is->is_id == idc->idc_session_id)
1420 			break;
1421 	}
1422 	if (is == NULL) {
1423 		sx_sunlock(&sc->sc_lock);
1424 		return (ESRCH);
1425 	}
1426 	sx_sunlock(&sc->sc_lock);
1427 
1428 	if (idc->idc_from_addrlen > 0) {
1429 		error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1430 		if (error != 0) {
1431 			ISCSI_SESSION_WARN(is,
1432 			    "getsockaddr failed with error %d", error);
1433 			return (error);
1434 		}
1435 	} else {
1436 		from_sa = NULL;
1437 	}
1438 	error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1439 	if (error != 0) {
1440 		ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1441 		    error);
1442 		free(from_sa, M_SONAME);
1443 		return (error);
1444 	}
1445 
1446 	ISCSI_SESSION_LOCK(is);
1447 	is->is_waiting_for_iscsid = false;
1448 	is->is_login_phase = true;
1449 	is->is_timeout = 0;
1450 	ISCSI_SESSION_UNLOCK(is);
1451 
1452 	error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1453 	    idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1454 	free(from_sa, M_SONAME);
1455 	free(to_sa, M_SONAME);
1456 
1457 	/*
1458 	 * Digests are always disabled during login phase.
1459 	 */
1460 	is->is_conn->ic_header_crc32c = false;
1461 	is->is_conn->ic_data_crc32c = false;
1462 
1463 	return (error);
1464 }
1465 
1466 static int
1467 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1468     struct iscsi_daemon_send *ids)
1469 {
1470 	struct iscsi_session *is;
1471 	struct icl_pdu *ip;
1472 	size_t datalen;
1473 	void *data;
1474 	int error;
1475 
1476 	sx_slock(&sc->sc_lock);
1477 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1478 		if (is->is_id == ids->ids_session_id)
1479 			break;
1480 	}
1481 	if (is == NULL) {
1482 		sx_sunlock(&sc->sc_lock);
1483 		return (ESRCH);
1484 	}
1485 	sx_sunlock(&sc->sc_lock);
1486 
1487 	if (is->is_login_phase == false)
1488 		return (EBUSY);
1489 
1490 	if (is->is_terminating || is->is_reconnecting)
1491 		return (EIO);
1492 
1493 	datalen = ids->ids_data_segment_len;
1494 	if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1495 		return (EINVAL);
1496 	if (datalen > 0) {
1497 		data = malloc(datalen, M_ISCSI, M_WAITOK);
1498 		error = copyin(ids->ids_data_segment, data, datalen);
1499 		if (error != 0) {
1500 			free(data, M_ISCSI);
1501 			return (error);
1502 		}
1503 	}
1504 
1505 	ip = icl_pdu_new_bhs(is->is_conn, M_WAITOK);
1506 	memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1507 	if (datalen > 0) {
1508 		error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1509 		KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1510 		free(data, M_ISCSI);
1511 	}
1512 	icl_pdu_queue(ip);
1513 
1514 	return (0);
1515 }
1516 
1517 static int
1518 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1519     struct iscsi_daemon_receive *idr)
1520 {
1521 	struct iscsi_session *is;
1522 	struct icl_pdu *ip;
1523 	void *data;
1524 
1525 	sx_slock(&sc->sc_lock);
1526 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1527 		if (is->is_id == idr->idr_session_id)
1528 			break;
1529 	}
1530 	if (is == NULL) {
1531 		sx_sunlock(&sc->sc_lock);
1532 		return (ESRCH);
1533 	}
1534 	sx_sunlock(&sc->sc_lock);
1535 
1536 	if (is->is_login_phase == false)
1537 		return (EBUSY);
1538 
1539 	ISCSI_SESSION_LOCK(is);
1540 	while (is->is_login_pdu == NULL &&
1541 	    is->is_terminating == false &&
1542 	    is->is_reconnecting == false)
1543 		cv_wait(&is->is_login_cv, &is->is_lock);
1544 	if (is->is_terminating || is->is_reconnecting) {
1545 		ISCSI_SESSION_UNLOCK(is);
1546 		return (EIO);
1547 	}
1548 	ip = is->is_login_pdu;
1549 	is->is_login_pdu = NULL;
1550 	ISCSI_SESSION_UNLOCK(is);
1551 
1552 	if (ip->ip_data_len > idr->idr_data_segment_len) {
1553 		icl_pdu_free(ip);
1554 		return (EMSGSIZE);
1555 	}
1556 
1557 	copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1558 	if (ip->ip_data_len > 0) {
1559 		data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1560 		icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1561 		copyout(data, idr->idr_data_segment, ip->ip_data_len);
1562 		free(data, M_ISCSI);
1563 	}
1564 
1565 	icl_pdu_free(ip);
1566 
1567 	return (0);
1568 }
1569 #endif /* ICL_KERNEL_PROXY */
1570 
1571 static void
1572 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1573 {
1574 	/*
1575 	 * Just make sure all the fields are null-terminated.
1576 	 *
1577 	 * XXX: This is not particularly secure.  We should
1578 	 * 	create our own conf and then copy in relevant
1579 	 * 	fields.
1580 	 */
1581 	isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1582 	isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1583 	isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1584 	isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1585 	isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1586 	isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1587 	isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1588 	isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1589 	isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1590 }
1591 
1592 static bool
1593 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1594 {
1595 
1596 	if (isc->isc_initiator[0] == '\0') {
1597 		ISCSI_DEBUG("empty isc_initiator");
1598 		return (false);
1599 	}
1600 
1601 	if (isc->isc_target_addr[0] == '\0') {
1602 		ISCSI_DEBUG("empty isc_target_addr");
1603 		return (false);
1604 	}
1605 
1606 	if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1607 		ISCSI_DEBUG("non-empty isc_target for discovery session");
1608 		return (false);
1609 	}
1610 
1611 	if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1612 		ISCSI_DEBUG("empty isc_target for non-discovery session");
1613 		return (false);
1614 	}
1615 
1616 	return (true);
1617 }
1618 
1619 static int
1620 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1621 {
1622 	struct iscsi_session *is;
1623 	const struct iscsi_session *is2;
1624 	int error;
1625 
1626 	iscsi_sanitize_session_conf(&isa->isa_conf);
1627 	if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1628 		return (EINVAL);
1629 
1630 	is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1631 	memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1632 
1633 	sx_xlock(&sc->sc_lock);
1634 
1635 	/*
1636 	 * Prevent duplicates.
1637 	 */
1638 	TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1639 		if (!!is->is_conf.isc_discovery !=
1640 		    !!is2->is_conf.isc_discovery)
1641 			continue;
1642 
1643 		if (strcmp(is->is_conf.isc_target_addr,
1644 		    is2->is_conf.isc_target_addr) != 0)
1645 			continue;
1646 
1647 		if (is->is_conf.isc_discovery == 0 &&
1648 		    strcmp(is->is_conf.isc_target,
1649 		    is2->is_conf.isc_target) != 0)
1650 			continue;
1651 
1652 		sx_xunlock(&sc->sc_lock);
1653 		free(is, M_ISCSI);
1654 		return (EBUSY);
1655 	}
1656 
1657 	is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1658 	is->is_conn->ic_receive = iscsi_receive_callback;
1659 	is->is_conn->ic_error = iscsi_error_callback;
1660 	is->is_conn->ic_prv0 = is;
1661 	TAILQ_INIT(&is->is_outstanding);
1662 	STAILQ_INIT(&is->is_postponed);
1663 	mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1664 	cv_init(&is->is_maintenance_cv, "iscsi_mt");
1665 #ifdef ICL_KERNEL_PROXY
1666 	cv_init(&is->is_login_cv, "iscsi_login");
1667 #endif
1668 
1669 	is->is_softc = sc;
1670 	sc->sc_last_session_id++;
1671 	is->is_id = sc->sc_last_session_id;
1672 	callout_init(&is->is_callout, 1);
1673 	callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1674 	TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1675 
1676 	error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1677 	if (error != 0) {
1678 		ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1679 		return (error);
1680 	}
1681 
1682 	/*
1683 	 * Trigger immediate reconnection.
1684 	 */
1685 	ISCSI_SESSION_LOCK(is);
1686 	is->is_waiting_for_iscsid = true;
1687 	strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1688 	ISCSI_SESSION_UNLOCK(is);
1689 	cv_signal(&sc->sc_cv);
1690 
1691 	sx_xunlock(&sc->sc_lock);
1692 
1693 	return (0);
1694 }
1695 
1696 static bool
1697 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1698     unsigned int id2, const struct iscsi_session_conf *c2)
1699 {
1700 	if (id2 == 0 && c2->isc_target[0] == '\0' &&
1701 	    c2->isc_target_addr[0] == '\0')
1702 		return (true);
1703 	if (id2 != 0 && id2 == id1)
1704 		return (true);
1705 	if (c2->isc_target[0] != '\0' &&
1706 	    strcmp(c1->isc_target, c2->isc_target) == 0)
1707 		return (true);
1708 	if (c2->isc_target_addr[0] != '\0' &&
1709 	    strcmp(c1->isc_target_addr, c2->isc_target_addr) == 0)
1710 		return (true);
1711 	return (false);
1712 }
1713 
1714 static int
1715 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1716     struct iscsi_session_remove *isr)
1717 {
1718 	struct iscsi_session *is, *tmp;
1719 	bool found = false;
1720 
1721 	iscsi_sanitize_session_conf(&isr->isr_conf);
1722 
1723 	sx_xlock(&sc->sc_lock);
1724 	TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1725 		ISCSI_SESSION_LOCK(is);
1726 		if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1727 		    isr->isr_session_id, &isr->isr_conf)) {
1728 			found = true;
1729 			iscsi_session_logout(is);
1730 			iscsi_session_terminate(is);
1731 		}
1732 		ISCSI_SESSION_UNLOCK(is);
1733 	}
1734 	sx_xunlock(&sc->sc_lock);
1735 
1736 	if (!found)
1737 		return (ESRCH);
1738 
1739 	return (0);
1740 }
1741 
1742 static int
1743 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1744 {
1745 	int error;
1746 	unsigned int i = 0;
1747 	struct iscsi_session *is;
1748 	struct iscsi_session_state iss;
1749 
1750 	sx_slock(&sc->sc_lock);
1751 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1752 		if (i >= isl->isl_nentries) {
1753 			sx_sunlock(&sc->sc_lock);
1754 			return (EMSGSIZE);
1755 		}
1756 		memset(&iss, 0, sizeof(iss));
1757 		memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1758 		iss.iss_id = is->is_id;
1759 		strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1760 		strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1761 
1762 		if (is->is_conn->ic_header_crc32c)
1763 			iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1764 		else
1765 			iss.iss_header_digest = ISCSI_DIGEST_NONE;
1766 
1767 		if (is->is_conn->ic_data_crc32c)
1768 			iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1769 		else
1770 			iss.iss_data_digest = ISCSI_DIGEST_NONE;
1771 
1772 		iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1773 		iss.iss_immediate_data = is->is_immediate_data;
1774 		iss.iss_connected = is->is_connected;
1775 
1776 		error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1777 		if (error != 0) {
1778 			sx_sunlock(&sc->sc_lock);
1779 			return (error);
1780 		}
1781 		i++;
1782 	}
1783 	sx_sunlock(&sc->sc_lock);
1784 
1785 	isl->isl_nentries = i;
1786 
1787 	return (0);
1788 }
1789 
1790 static int
1791 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
1792     struct iscsi_session_modify *ism)
1793 {
1794 	struct iscsi_session *is;
1795 
1796 	iscsi_sanitize_session_conf(&ism->ism_conf);
1797 	if (iscsi_valid_session_conf(&ism->ism_conf) == false)
1798 		return (EINVAL);
1799 
1800 	sx_xlock(&sc->sc_lock);
1801 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1802 		ISCSI_SESSION_LOCK(is);
1803 		if (is->is_id == ism->ism_session_id)
1804 			break;
1805 		ISCSI_SESSION_UNLOCK(is);
1806 	}
1807 	if (is == NULL) {
1808 		sx_xunlock(&sc->sc_lock);
1809 		return (ESRCH);
1810 	}
1811 	sx_xunlock(&sc->sc_lock);
1812 
1813 	memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
1814 	ISCSI_SESSION_UNLOCK(is);
1815 
1816 	iscsi_session_reconnect(is);
1817 
1818 	return (0);
1819 }
1820 
1821 static int
1822 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1823     struct thread *td)
1824 {
1825 	struct iscsi_softc *sc;
1826 
1827 	sc = dev->si_drv1;
1828 
1829 	switch (cmd) {
1830 	case ISCSIDWAIT:
1831 		return (iscsi_ioctl_daemon_wait(sc,
1832 		    (struct iscsi_daemon_request *)arg));
1833 	case ISCSIDHANDOFF:
1834 		return (iscsi_ioctl_daemon_handoff(sc,
1835 		    (struct iscsi_daemon_handoff *)arg));
1836 	case ISCSIDFAIL:
1837 		return (iscsi_ioctl_daemon_fail(sc,
1838 		    (struct iscsi_daemon_fail *)arg));
1839 #ifdef ICL_KERNEL_PROXY
1840 	case ISCSIDCONNECT:
1841 		return (iscsi_ioctl_daemon_connect(sc,
1842 		    (struct iscsi_daemon_connect *)arg));
1843 	case ISCSIDSEND:
1844 		return (iscsi_ioctl_daemon_send(sc,
1845 		    (struct iscsi_daemon_send *)arg));
1846 	case ISCSIDRECEIVE:
1847 		return (iscsi_ioctl_daemon_receive(sc,
1848 		    (struct iscsi_daemon_receive *)arg));
1849 #endif /* ICL_KERNEL_PROXY */
1850 	case ISCSISADD:
1851 		return (iscsi_ioctl_session_add(sc,
1852 		    (struct iscsi_session_add *)arg));
1853 	case ISCSISREMOVE:
1854 		return (iscsi_ioctl_session_remove(sc,
1855 		    (struct iscsi_session_remove *)arg));
1856 	case ISCSISLIST:
1857 		return (iscsi_ioctl_session_list(sc,
1858 		    (struct iscsi_session_list *)arg));
1859 	case ISCSISMODIFY:
1860 		return (iscsi_ioctl_session_modify(sc,
1861 		    (struct iscsi_session_modify *)arg));
1862 	default:
1863 		return (EINVAL);
1864 	}
1865 }
1866 
1867 static struct iscsi_outstanding *
1868 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1869 {
1870 	struct iscsi_outstanding *io;
1871 
1872 	ISCSI_SESSION_LOCK_ASSERT(is);
1873 
1874 	TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1875 		if (io->io_initiator_task_tag == initiator_task_tag)
1876 			return (io);
1877 	}
1878 	return (NULL);
1879 }
1880 
1881 static int
1882 iscsi_outstanding_add(struct iscsi_session *is,
1883     uint32_t initiator_task_tag, union ccb *ccb)
1884 {
1885 	struct iscsi_outstanding *io;
1886 
1887 	ISCSI_SESSION_LOCK_ASSERT(is);
1888 
1889 	KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
1890 	    ("initiator_task_tag 0x%x already added", initiator_task_tag));
1891 
1892 	io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
1893 	if (io == NULL) {
1894 		ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
1895 		return (ENOMEM);
1896 	}
1897 	io->io_initiator_task_tag = initiator_task_tag;
1898 	io->io_ccb = ccb;
1899 	TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
1900 	return (0);
1901 }
1902 
1903 static void
1904 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
1905 {
1906 
1907 	ISCSI_SESSION_LOCK_ASSERT(is);
1908 
1909 	TAILQ_REMOVE(&is->is_outstanding, io, io_next);
1910 	uma_zfree(iscsi_outstanding_zone, io);
1911 }
1912 
1913 static void
1914 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
1915 {
1916 	struct icl_pdu *request;
1917 	struct iscsi_bhs_scsi_command *bhssc;
1918 	struct ccb_scsiio *csio;
1919 	size_t len;
1920 	int error;
1921 
1922 	ISCSI_SESSION_LOCK_ASSERT(is);
1923 
1924 #if 0
1925 	KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
1926 #else
1927 	if (is->is_login_phase) {
1928 		ISCSI_SESSION_DEBUG(is, "called during login phase");
1929 		if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1930 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1931 			ISCSI_SESSION_DEBUG(is, "freezing devq");
1932 		}
1933 		ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
1934 		xpt_done(ccb);
1935 		return;
1936 	}
1937 #endif
1938 
1939 	request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
1940 	if (request == NULL) {
1941 		if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1942 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1943 			ISCSI_SESSION_DEBUG(is, "freezing devq");
1944 		}
1945 		ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1946 		xpt_done(ccb);
1947 		return;
1948 	}
1949 
1950 	csio = &ccb->csio;
1951 	bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
1952 	bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
1953 	bhssc->bhssc_flags |= BHSSC_FLAGS_F;
1954 	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1955 	case CAM_DIR_IN:
1956 		bhssc->bhssc_flags |= BHSSC_FLAGS_R;
1957 		break;
1958 	case CAM_DIR_OUT:
1959 		bhssc->bhssc_flags |= BHSSC_FLAGS_W;
1960 		break;
1961 	}
1962 
1963         switch (csio->tag_action) {
1964         case MSG_HEAD_OF_Q_TAG:
1965 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
1966 		break;
1967                 break;
1968         case MSG_ORDERED_Q_TAG:
1969 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
1970                 break;
1971         case MSG_ACA_TASK:
1972 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
1973                 break;
1974         case CAM_TAG_ACTION_NONE:
1975         case MSG_SIMPLE_Q_TAG:
1976         default:
1977 		bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
1978                 break;
1979         }
1980 
1981 	bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
1982 	bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
1983 	is->is_initiator_task_tag++;
1984 	bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
1985 	KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
1986 	    ("unsupported CDB size %zd", (size_t)csio->cdb_len));
1987 
1988 	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1989 		memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
1990 	else
1991 		memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
1992 
1993 	error = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
1994 	if (error != 0) {
1995 		icl_pdu_free(request);
1996 		if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1997 			xpt_freeze_devq(ccb->ccb_h.path, 1);
1998 			ISCSI_SESSION_DEBUG(is, "freezing devq");
1999 		}
2000 		ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2001 		xpt_done(ccb);
2002 		return;
2003 	}
2004 
2005 	if (is->is_immediate_data &&
2006 	    (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2007 		len = csio->dxfer_len;
2008 		//ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2009 		if (len > is->is_first_burst_length) {
2010 			ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2011 			len = is->is_first_burst_length;
2012 		}
2013 
2014 		error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2015 		if (error != 0) {
2016 			icl_pdu_free(request);
2017 			if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2018 				xpt_freeze_devq(ccb->ccb_h.path, 1);
2019 				ISCSI_SESSION_DEBUG(is, "freezing devq");
2020 			}
2021 			ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2022 			xpt_done(ccb);
2023 			return;
2024 		}
2025 	}
2026 	iscsi_pdu_queue_locked(request);
2027 }
2028 
2029 static void
2030 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2031 {
2032 	struct iscsi_session *is;
2033 
2034 	is = cam_sim_softc(sim);
2035 
2036 	ISCSI_SESSION_LOCK_ASSERT(is);
2037 
2038 	if (is->is_terminating ||
2039 	    (is->is_connected == false && fail_on_disconnection)) {
2040 		ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2041 		xpt_done(ccb);
2042 		return;
2043 	}
2044 
2045 	switch (ccb->ccb_h.func_code) {
2046 	case XPT_PATH_INQ:
2047 	{
2048 		struct ccb_pathinq *cpi = &ccb->cpi;
2049 
2050 		cpi->version_num = 1;
2051 		cpi->hba_inquiry = PI_TAG_ABLE;
2052 		cpi->target_sprt = 0;
2053 		cpi->hba_misc = PIM_EXTLUNS;
2054 		cpi->hba_eng_cnt = 0;
2055 		cpi->max_target = 0;
2056 		cpi->max_lun = 0;
2057 		cpi->initiator_id = ~0;
2058 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2059 		strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2060 		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2061 		cpi->unit_number = cam_sim_unit(sim);
2062 		cpi->bus_id = cam_sim_bus(sim);
2063 		cpi->base_transfer_speed = 150000; /* XXX */
2064 		cpi->transport = XPORT_ISCSI;
2065 		cpi->transport_version = 0;
2066 		cpi->protocol = PROTO_SCSI;
2067 		cpi->protocol_version = SCSI_REV_SPC3;
2068 		cpi->maxio = MAXPHYS;
2069 		cpi->ccb_h.status = CAM_REQ_CMP;
2070 		break;
2071 	}
2072 	case XPT_CALC_GEOMETRY:
2073 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
2074 		ccb->ccb_h.status = CAM_REQ_CMP;
2075 		break;
2076 #if 0
2077 	/*
2078 	 * XXX: What's the point?
2079 	 */
2080 	case XPT_RESET_BUS:
2081 	case XPT_ABORT:
2082 	case XPT_TERM_IO:
2083 		ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2084 		ccb->ccb_h.status = CAM_REQ_CMP;
2085 		break;
2086 #endif
2087 	case XPT_SCSI_IO:
2088 		iscsi_action_scsiio(is, ccb);
2089 		return;
2090 	default:
2091 #if 0
2092 		ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2093 #endif
2094 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2095 		break;
2096 	}
2097 	xpt_done(ccb);
2098 }
2099 
2100 static void
2101 iscsi_poll(struct cam_sim *sim)
2102 {
2103 
2104 	KASSERT(0, ("%s: you're not supposed to be here", __func__));
2105 }
2106 
2107 static void
2108 iscsi_shutdown(struct iscsi_softc *sc)
2109 {
2110 	struct iscsi_session *is;
2111 
2112 	ISCSI_DEBUG("removing all sessions due to shutdown");
2113 
2114 	sx_slock(&sc->sc_lock);
2115 	TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2116 		iscsi_session_terminate(is);
2117 	sx_sunlock(&sc->sc_lock);
2118 }
2119 
2120 static int
2121 iscsi_load(void)
2122 {
2123 	int error;
2124 
2125 	sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2126 	sx_init(&sc->sc_lock, "iscsi");
2127 	TAILQ_INIT(&sc->sc_sessions);
2128 	cv_init(&sc->sc_cv, "iscsi_cv");
2129 
2130 	iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2131 	    sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2132 	    UMA_ALIGN_PTR, 0);
2133 
2134 	error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2135 	    NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2136 	if (error != 0) {
2137 		ISCSI_WARN("failed to create device node, error %d", error);
2138 		return (error);
2139 	}
2140 	sc->sc_cdev->si_drv1 = sc;
2141 
2142 	/*
2143 	 * Note that this needs to get run before dashutdown().  Otherwise,
2144 	 * when rebooting with iSCSI session with outstanding requests,
2145 	 * but disconnected, dashutdown() will hang on cam_periph_runccb().
2146 	 */
2147 	sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2148 	    iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2149 
2150 	return (0);
2151 }
2152 
2153 static int
2154 iscsi_unload(void)
2155 {
2156 	struct iscsi_session *is, *tmp;
2157 
2158 	if (sc->sc_cdev != NULL) {
2159 		ISCSI_DEBUG("removing device node");
2160 		destroy_dev(sc->sc_cdev);
2161 		ISCSI_DEBUG("device node removed");
2162 	}
2163 
2164 	if (sc->sc_shutdown_eh != NULL)
2165 		EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2166 
2167 	sx_slock(&sc->sc_lock);
2168 	TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2169 		iscsi_session_terminate(is);
2170 	while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2171 		ISCSI_DEBUG("waiting for sessions to terminate");
2172 		cv_wait(&sc->sc_cv, &sc->sc_lock);
2173 	}
2174 	ISCSI_DEBUG("all sessions terminated");
2175 	sx_sunlock(&sc->sc_lock);
2176 
2177 	uma_zdestroy(iscsi_outstanding_zone);
2178 	sx_destroy(&sc->sc_lock);
2179 	cv_destroy(&sc->sc_cv);
2180 	free(sc, M_ISCSI);
2181 	return (0);
2182 }
2183 
2184 static int
2185 iscsi_quiesce(void)
2186 {
2187 	sx_slock(&sc->sc_lock);
2188 	if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2189 		sx_sunlock(&sc->sc_lock);
2190 		return (EBUSY);
2191 	}
2192 	sx_sunlock(&sc->sc_lock);
2193 	return (0);
2194 }
2195 
2196 static int
2197 iscsi_modevent(module_t mod, int what, void *arg)
2198 {
2199 	int error;
2200 
2201 	switch (what) {
2202 	case MOD_LOAD:
2203 		error = iscsi_load();
2204 		break;
2205 	case MOD_UNLOAD:
2206 		error = iscsi_unload();
2207 		break;
2208 	case MOD_QUIESCE:
2209 		error = iscsi_quiesce();
2210 		break;
2211 	default:
2212 		error = EINVAL;
2213 		break;
2214 	}
2215 	return (error);
2216 }
2217 
2218 moduledata_t iscsi_data = {
2219 	"iscsi",
2220 	iscsi_modevent,
2221 	0
2222 };
2223 
2224 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2225 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2226 MODULE_DEPEND(iscsi, icl, 1, 1, 1);
2227