xref: /freebsd/share/examples/scsi_target/scsi_target.c (revision ddd5b8e9b4d8957fce018c520657cdfa4ecffad3)
1 /*
2  * SCSI Disk Emulator
3  *
4  * Copyright (c) 2002 Nate Lawson.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions, and the following disclaimer,
12  *    without modification, immediately at the beginning of the file.
13  * 2. The name of the author may not be used to endorse or promote products
14  *    derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #include <sys/types.h>
32 #include <ctype.h>
33 #include <errno.h>
34 #include <err.h>
35 #include <fcntl.h>
36 #include <signal.h>
37 #include <stddef.h>
38 #include <stdio.h>
39 #include <stdlib.h>
40 #include <string.h>
41 #include <sysexits.h>
42 #include <unistd.h>
43 #include <aio.h>
44 #include <assert.h>
45 #include <sys/stat.h>
46 #include <sys/queue.h>
47 #include <sys/event.h>
48 #include <sys/param.h>
49 #include <sys/disk.h>
50 #include <cam/cam_queue.h>
51 #include <cam/scsi/scsi_all.h>
52 #include <cam/scsi/scsi_targetio.h>
53 #include <cam/scsi/scsi_message.h>
54 #include "scsi_target.h"
55 
56 /* Maximum amount to transfer per CTIO */
57 #define MAX_XFER	MAXPHYS
58 /* Maximum number of allocated CTIOs */
59 #define MAX_CTIOS	64
60 /* Maximum sector size for emulated volume */
61 #define MAX_SECTOR	32768
62 
63 /* Global variables */
64 int		debug;
65 int		notaio = 0;
66 off_t		volume_size;
67 u_int		sector_size;
68 size_t		buf_size;
69 
70 /* Local variables */
71 static int    targ_fd;
72 static int    kq_fd;
73 static int    file_fd;
74 static int    num_ctios;
75 static struct ccb_queue		pending_queue;
76 static struct ccb_queue		work_queue;
77 static struct ioc_enable_lun	ioc_enlun = {
78 	CAM_BUS_WILDCARD,
79 	CAM_TARGET_WILDCARD,
80 	CAM_LUN_WILDCARD
81 };
82 
83 /* Local functions */
84 static void		cleanup(void);
85 static int		init_ccbs(void);
86 static void		request_loop(void);
87 static void		handle_read(void);
88 /* static int		work_atio(struct ccb_accept_tio *); */
89 static void		queue_io(struct ccb_scsiio *);
90 static int		run_queue(struct ccb_accept_tio *);
91 static int		work_inot(struct ccb_immediate_notify *);
92 static struct ccb_scsiio *
93 			get_ctio(void);
94 /* static void		free_ccb(union ccb *); */
95 static cam_status	get_sim_flags(u_int16_t *);
96 static void		rel_simq(void);
97 static void		abort_all_pending(void);
98 static void		usage(void);
99 
100 int
101 main(int argc, char *argv[])
102 {
103 	int ch;
104 	char *file_name;
105 	u_int16_t req_flags, sim_flags;
106 	off_t user_size;
107 
108 	/* Initialize */
109 	debug = 0;
110 	req_flags = sim_flags = 0;
111 	user_size = 0;
112 	targ_fd = file_fd = kq_fd = -1;
113 	num_ctios = 0;
114 	sector_size = SECTOR_SIZE;
115 	buf_size = DFLTPHYS;
116 
117 	/* Prepare resource pools */
118 	TAILQ_INIT(&pending_queue);
119 	TAILQ_INIT(&work_queue);
120 
121 	while ((ch = getopt(argc, argv, "AdSTYb:c:s:W:")) != -1) {
122 		switch(ch) {
123 		case 'A':
124 			req_flags |= SID_Addr16;
125 			break;
126 		case 'd':
127 			debug = 1;
128 			break;
129 		case 'S':
130 			req_flags |= SID_Sync;
131 			break;
132 		case 'T':
133 			req_flags |= SID_CmdQue;
134 			break;
135 		case 'b':
136 			buf_size = atoi(optarg);
137 			if (buf_size < 256 || buf_size > MAX_XFER)
138 				errx(1, "Unreasonable buf size: %s", optarg);
139 			break;
140 		case 'c':
141 			sector_size = atoi(optarg);
142 			if (sector_size < 512 || sector_size > MAX_SECTOR)
143 				errx(1, "Unreasonable sector size: %s", optarg);
144 			break;
145 		case 's':
146 		{
147 			int last, shift = 0;
148 
149 			last = strlen(optarg) - 1;
150 			if (last > 0) {
151 				switch (tolower(optarg[last])) {
152 				case 'e':
153 					shift += 10;
154 					/* FALLTHROUGH */
155 				case 'p':
156 					shift += 10;
157 					/* FALLTHROUGH */
158 				case 't':
159 					shift += 10;
160 					/* FALLTHROUGH */
161 				case 'g':
162 					shift += 10;
163 					/* FALLTHROUGH */
164 				case 'm':
165 					shift += 10;
166 					/* FALLTHROUGH */
167 				case 'k':
168 					shift += 10;
169 					optarg[last] = 0;
170 					break;
171 				}
172 			}
173 			user_size = strtoll(optarg, (char **)NULL, /*base*/10);
174 			user_size <<= shift;
175 			if (user_size < 0)
176 				errx(1, "Unreasonable volume size: %s", optarg);
177 			break;
178 		}
179 		case 'W':
180 			req_flags &= ~(SID_WBus16 | SID_WBus32);
181 			switch (atoi(optarg)) {
182 			case 8:
183 				/* Leave req_flags zeroed */
184 				break;
185 			case 16:
186 				req_flags |= SID_WBus16;
187 				break;
188 			case 32:
189 				req_flags |= SID_WBus32;
190 				break;
191 			default:
192 				warnx("Width %s not supported", optarg);
193 				usage();
194 				/* NOTREACHED */
195 			}
196 			break;
197 		case 'Y':
198 			notaio = 1;
199 			break;
200 		default:
201 			usage();
202 			/* NOTREACHED */
203 		}
204 	}
205 	argc -= optind;
206 	argv += optind;
207 
208 	if (argc != 2)
209 		usage();
210 
211 	sscanf(argv[0], "%u:%u:%u", &ioc_enlun.path_id, &ioc_enlun.target_id,
212 	       &ioc_enlun.lun_id);
213 	file_name = argv[1];
214 
215 	if (ioc_enlun.path_id == CAM_BUS_WILDCARD ||
216 	    ioc_enlun.target_id == CAM_TARGET_WILDCARD ||
217 	    ioc_enlun.lun_id == CAM_LUN_WILDCARD) {
218 		warnx("Incomplete target path specified");
219 		usage();
220 		/* NOTREACHED */
221 	}
222 	/* We don't support any vendor-specific commands */
223 	ioc_enlun.grp6_len = 0;
224 	ioc_enlun.grp7_len = 0;
225 
226 	/* Open backing store for IO */
227 	file_fd = open(file_name, O_RDWR);
228 	if (file_fd < 0)
229 		errx(EX_NOINPUT, "open backing store file");
230 
231 	/* Check backing store size or use the size user gave us */
232 	if (user_size == 0) {
233 		struct stat st;
234 
235 		if (fstat(file_fd, &st) < 0)
236 			err(1, "fstat file");
237 #if __FreeBSD_version >= 500000
238 		if ((st.st_mode & S_IFCHR) != 0) {
239 			/* raw device */
240 			off_t mediasize;
241 			if (ioctl(file_fd, DIOCGMEDIASIZE, &mediasize) < 0)
242 				err(1, "DIOCGMEDIASIZE");
243 
244 			/* XXX get sector size by ioctl()?? */
245 			volume_size = mediasize / sector_size;
246 		} else
247 #endif
248 			volume_size = st.st_size / sector_size;
249 	} else {
250 		volume_size = user_size / sector_size;
251 	}
252 	if (debug)
253 		warnx("volume_size: %d bytes x " OFF_FMT " sectors",
254 		    sector_size, volume_size);
255 
256 	if (volume_size <= 0)
257 		errx(1, "volume must be larger than %d", sector_size);
258 
259 	if (notaio == 0) {
260 		struct aiocb aio, *aiop;
261 
262 		/* See if we have we have working AIO support */
263 		memset(&aio, 0, sizeof(aio));
264 		aio.aio_buf = malloc(sector_size);
265 		if (aio.aio_buf == NULL)
266 			err(1, "malloc");
267 		aio.aio_fildes = file_fd;
268 		aio.aio_offset = 0;
269 		aio.aio_nbytes = sector_size;
270 		signal(SIGSYS, SIG_IGN);
271 		if (aio_read(&aio) != 0) {
272 			printf("AIO support is not available- switchin to"
273 			       " single-threaded mode.\n");
274 			notaio = 1;
275 		} else {
276 			if (aio_waitcomplete(&aiop, NULL) != sector_size)
277 				err(1, "aio_waitcomplete");
278 			assert(aiop == &aio);
279 			signal(SIGSYS, SIG_DFL);
280 		}
281 		free((void *)aio.aio_buf);
282 		if (debug && notaio == 0)
283 			warnx("aio support tested ok");
284 	}
285 
286 	targ_fd = open("/dev/targ", O_RDWR);
287 	if (targ_fd < 0)
288     	    err(1, "/dev/targ");
289 	else
290 	    warnx("opened /dev/targ");
291 
292 	/* The first three are handled by kevent() later */
293 	signal(SIGHUP, SIG_IGN);
294 	signal(SIGINT, SIG_IGN);
295 	signal(SIGTERM, SIG_IGN);
296 	signal(SIGPROF, SIG_IGN);
297 	signal(SIGALRM, SIG_IGN);
298 	signal(SIGSTOP, SIG_IGN);
299 	signal(SIGTSTP, SIG_IGN);
300 
301 	/* Register a cleanup handler to run when exiting */
302 	atexit(cleanup);
303 
304 	/* Enable listening on the specified LUN */
305 	if (ioctl(targ_fd, TARGIOCENABLE, &ioc_enlun) != 0)
306 		err(1, "TARGIOCENABLE");
307 
308 	/* Enable debugging if requested */
309 	if (debug) {
310 		if (ioctl(targ_fd, TARGIOCDEBUG, &debug) != 0)
311 			warnx("TARGIOCDEBUG");
312 	}
313 
314 	/* Set up inquiry data according to what SIM supports */
315 	if (get_sim_flags(&sim_flags) != CAM_REQ_CMP)
316 		errx(1, "get_sim_flags");
317 
318 	if (tcmd_init(req_flags, sim_flags) != 0)
319 		errx(1, "Initializing tcmd subsystem failed");
320 
321 	/* Queue ATIOs and INOTs on descriptor */
322 	if (init_ccbs() != 0)
323 		errx(1, "init_ccbs failed");
324 
325 	if (debug)
326 		warnx("main loop beginning");
327 
328 	request_loop();
329 
330 	exit(0);
331 }
332 
333 static void
334 cleanup()
335 {
336 	struct ccb_hdr *ccb_h;
337 
338 	if (debug) {
339 		warnx("cleanup called");
340 		debug = 0;
341 		ioctl(targ_fd, TARGIOCDEBUG, &debug);
342 	}
343 	ioctl(targ_fd, TARGIOCDISABLE, NULL);
344 	close(targ_fd);
345 
346 	while ((ccb_h = TAILQ_FIRST(&pending_queue)) != NULL) {
347 		TAILQ_REMOVE(&pending_queue, ccb_h, periph_links.tqe);
348 		free_ccb((union ccb *)ccb_h);
349 	}
350 	while ((ccb_h = TAILQ_FIRST(&work_queue)) != NULL) {
351 		TAILQ_REMOVE(&work_queue, ccb_h, periph_links.tqe);
352 		free_ccb((union ccb *)ccb_h);
353 	}
354 
355 	if (kq_fd != -1)
356 		close(kq_fd);
357 }
358 
359 /* Allocate ATIOs/INOTs and queue on HBA */
360 static int
361 init_ccbs()
362 {
363 	int i;
364 
365 	for (i = 0; i < MAX_INITIATORS; i++) {
366 		struct ccb_accept_tio *atio;
367 		struct atio_descr *a_descr;
368 		struct ccb_immed_notify *inot;
369 
370 		atio = (struct ccb_accept_tio *)malloc(sizeof(*atio));
371 		if (atio == NULL) {
372 			warn("malloc ATIO");
373 			return (-1);
374 		}
375 		a_descr = (struct atio_descr *)malloc(sizeof(*a_descr));
376 		if (a_descr == NULL) {
377 			free(atio);
378 			warn("malloc atio_descr");
379 			return (-1);
380 		}
381 		atio->ccb_h.func_code = XPT_ACCEPT_TARGET_IO;
382 		atio->ccb_h.targ_descr = a_descr;
383 		send_ccb((union ccb *)atio, /*priority*/1);
384 
385 		inot = (struct ccb_immed_notify *)malloc(sizeof(*inot));
386 		if (inot == NULL) {
387 			warn("malloc INOT");
388 			return (-1);
389 		}
390 		inot->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY;
391 		send_ccb((union ccb *)inot, /*priority*/1);
392 	}
393 
394 	return (0);
395 }
396 
397 static void
398 request_loop()
399 {
400 	struct kevent events[MAX_EVENTS];
401 	struct timespec ts, *tptr;
402 	int quit;
403 
404 	/* Register kqueue for event notification */
405 	if ((kq_fd = kqueue()) < 0)
406 		err(1, "init kqueue");
407 
408 	/* Set up some default events */
409 	EV_SET(&events[0], SIGHUP, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0, 0, 0);
410 	EV_SET(&events[1], SIGINT, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0, 0, 0);
411 	EV_SET(&events[2], SIGTERM, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0, 0, 0);
412 	EV_SET(&events[3], targ_fd, EVFILT_READ, EV_ADD|EV_ENABLE, 0, 0, 0);
413 	if (kevent(kq_fd, events, 4, NULL, 0, NULL) < 0)
414 		err(1, "kevent signal registration");
415 
416 	ts.tv_sec = 0;
417 	ts.tv_nsec = 0;
418 	tptr = NULL;
419 	quit = 0;
420 
421 	/* Loop until user signal */
422 	while (quit == 0) {
423 		int retval, i, oo;
424 		struct ccb_hdr *ccb_h;
425 
426 		/* Check for the next signal, read ready, or AIO completion */
427 		retval = kevent(kq_fd, NULL, 0, events, MAX_EVENTS, tptr);
428 		if (retval < 0) {
429 			if (errno == EINTR) {
430 				if (debug)
431 					warnx("EINTR, looping");
432 				continue;
433             		}
434 			else {
435 				err(1, "kevent failed");
436 			}
437 		} else if (retval > MAX_EVENTS) {
438 			errx(1, "kevent returned more events than allocated?");
439 		}
440 
441 		/* Process all received events. */
442 		for (oo = i = 0; i < retval; i++) {
443 			if ((events[i].flags & EV_ERROR) != 0)
444 				errx(1, "kevent registration failed");
445 
446 			switch (events[i].filter) {
447 			case EVFILT_READ:
448 				if (debug)
449 					warnx("read ready");
450 				handle_read();
451 				break;
452 			case EVFILT_AIO:
453 			{
454 				struct ccb_scsiio *ctio;
455 				struct ctio_descr *c_descr;
456 				if (debug)
457 					warnx("aio ready");
458 
459 				ctio = (struct ccb_scsiio *)events[i].udata;
460 				c_descr = (struct ctio_descr *)
461 					  ctio->ccb_h.targ_descr;
462 				c_descr->event = AIO_DONE;
463 				/* Queue on the appropriate ATIO */
464 				queue_io(ctio);
465 				/* Process any queued completions. */
466 				oo += run_queue(c_descr->atio);
467 				break;
468 			}
469 			case EVFILT_SIGNAL:
470 				if (debug)
471 					warnx("signal ready, setting quit");
472 				quit = 1;
473 				break;
474 			default:
475 				warnx("unknown event %d", events[i].filter);
476 				break;
477 			}
478 
479 			if (debug)
480 				warnx("event %d done", events[i].filter);
481 		}
482 
483 		if (oo) {
484 			tptr = &ts;
485 			continue;
486 		}
487 
488 		/* Grab the first CCB and perform one work unit. */
489 		if ((ccb_h = TAILQ_FIRST(&work_queue)) != NULL) {
490 			union ccb *ccb;
491 
492 			ccb = (union ccb *)ccb_h;
493 			switch (ccb_h->func_code) {
494 			case XPT_ACCEPT_TARGET_IO:
495 				/* Start one more transfer. */
496 				retval = work_atio(&ccb->atio);
497 				break;
498 			case XPT_IMMEDIATE_NOTIFY:
499 				retval = work_inot(&ccb->cin1);
500 				break;
501 			default:
502 				warnx("Unhandled ccb type %#x on workq",
503 				      ccb_h->func_code);
504 				abort();
505 				/* NOTREACHED */
506 			}
507 
508 			/* Assume work function handled the exception */
509 			if ((ccb_h->status & CAM_DEV_QFRZN) != 0) {
510 				if (debug) {
511 					warnx("Queue frozen receiving CCB, "
512 					      "releasing");
513 				}
514 				rel_simq();
515 			}
516 
517 			/* No more work needed for this command. */
518 			if (retval == 0) {
519 				TAILQ_REMOVE(&work_queue, ccb_h,
520 					     periph_links.tqe);
521 			}
522 		}
523 
524 		/*
525 		 * Poll for new events (i.e. completions) while we
526 		 * are processing CCBs on the work_queue. Once it's
527 		 * empty, use an infinite wait.
528 		 */
529 		if (!TAILQ_EMPTY(&work_queue))
530 			tptr = &ts;
531 		else
532 			tptr = NULL;
533 	}
534 }
535 
536 /* CCBs are ready from the kernel */
537 static void
538 handle_read()
539 {
540 	union ccb *ccb_array[MAX_INITIATORS], *ccb;
541 	int ccb_count, i, oo;
542 
543 	ccb_count = read(targ_fd, ccb_array, sizeof(ccb_array));
544 	if (ccb_count <= 0) {
545 		warn("read ccb ptrs");
546 		return;
547 	}
548 	ccb_count /= sizeof(union ccb *);
549 	if (ccb_count < 1) {
550 		warnx("truncated read ccb ptr?");
551 		return;
552 	}
553 
554 	for (i = 0; i < ccb_count; i++) {
555 		ccb = ccb_array[i];
556 		TAILQ_REMOVE(&pending_queue, &ccb->ccb_h, periph_links.tqe);
557 
558 		switch (ccb->ccb_h.func_code) {
559 		case XPT_ACCEPT_TARGET_IO:
560 		{
561 			struct ccb_accept_tio *atio;
562 			struct atio_descr *a_descr;
563 
564 			/* Initialize ATIO descr for this transaction */
565 			atio = &ccb->atio;
566 			a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
567 			bzero(a_descr, sizeof(*a_descr));
568 			TAILQ_INIT(&a_descr->cmplt_io);
569 			a_descr->flags = atio->ccb_h.flags &
570 				(CAM_DIS_DISCONNECT | CAM_TAG_ACTION_VALID);
571 			/* XXX add a_descr->priority */
572 			if ((atio->ccb_h.flags & CAM_CDB_POINTER) == 0)
573 				a_descr->cdb = atio->cdb_io.cdb_bytes;
574 			else
575 				a_descr->cdb = atio->cdb_io.cdb_ptr;
576 
577 			/* ATIOs are processed in FIFO order */
578 			TAILQ_INSERT_TAIL(&work_queue, &ccb->ccb_h,
579 					  periph_links.tqe);
580 			break;
581 		}
582 		case XPT_CONT_TARGET_IO:
583 		{
584 			struct ccb_scsiio *ctio;
585 			struct ctio_descr *c_descr;
586 
587 			ctio = &ccb->ctio;
588 			c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
589 			c_descr->event = CTIO_DONE;
590 			/* Queue on the appropriate ATIO */
591 			queue_io(ctio);
592 			/* Process any queued completions. */
593 			oo += run_queue(c_descr->atio);
594 			break;
595 		}
596 		case XPT_IMMED_NOTIFY:
597 			/* INOTs are handled with priority */
598 			TAILQ_INSERT_HEAD(&work_queue, &ccb->ccb_h,
599 					  periph_links.tqe);
600 			break;
601 		default:
602 			warnx("Unhandled ccb type %#x in handle_read",
603 			      ccb->ccb_h.func_code);
604 			break;
605 		}
606 	}
607 }
608 
609 /* Process an ATIO CCB from the kernel */
610 int
611 work_atio(struct ccb_accept_tio *atio)
612 {
613 	struct ccb_scsiio *ctio;
614 	struct atio_descr *a_descr;
615 	struct ctio_descr *c_descr;
616 	cam_status status;
617 	int ret;
618 
619 	if (debug)
620 		warnx("Working on ATIO %p", atio);
621 
622 	a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
623 
624 	/* Get a CTIO and initialize it according to our known parameters */
625 	ctio = get_ctio();
626 	if (ctio == NULL) {
627 		return (1);
628 	}
629 	ret = 0;
630 	ctio->ccb_h.flags = a_descr->flags;
631 	ctio->tag_id = atio->tag_id;
632 	ctio->init_id = atio->init_id;
633 	/* XXX priority needs to be added to a_descr */
634 	c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
635 	c_descr->atio = atio;
636 	if ((a_descr->flags & CAM_DIR_IN) != 0)
637 		c_descr->offset = a_descr->base_off + a_descr->targ_req;
638 	else if ((a_descr->flags & CAM_DIR_MASK) == CAM_DIR_OUT)
639 		c_descr->offset = a_descr->base_off + a_descr->init_req;
640 	else
641 		c_descr->offset = a_descr->base_off;
642 
643 	/*
644 	 * Return a check condition if there was an error while
645 	 * receiving this ATIO.
646 	 */
647 	if (atio->sense_len != 0) {
648 		struct scsi_sense_data_fixed *sense;
649 
650 		if (debug) {
651 			warnx("ATIO with %u bytes sense received",
652 			      atio->sense_len);
653 		}
654 		sense = (struct scsi_sense_data_fixed *)&atio->sense_data;
655 		tcmd_sense(ctio->init_id, ctio, sense->flags,
656 			   sense->add_sense_code, sense->add_sense_code_qual);
657 		send_ccb((union ccb *)ctio, /*priority*/1);
658 		return (0);
659 	}
660 
661 	status = atio->ccb_h.status & CAM_STATUS_MASK;
662 	switch (status) {
663 	case CAM_CDB_RECVD:
664 		ret = tcmd_handle(atio, ctio, ATIO_WORK);
665 		break;
666 	case CAM_REQ_ABORTED:
667 		warn("ATIO %p aborted", a_descr);
668 		/* Requeue on HBA */
669 		TAILQ_REMOVE(&work_queue, &atio->ccb_h, periph_links.tqe);
670 		send_ccb((union ccb *)atio, /*priority*/1);
671 		ret = 1;
672 		break;
673 	default:
674 		warnx("ATIO completed with unhandled status %#x", status);
675 		abort();
676 		/* NOTREACHED */
677 		break;
678 	}
679 
680 	return (ret);
681 }
682 
683 static void
684 queue_io(struct ccb_scsiio *ctio)
685 {
686 	struct ccb_hdr *ccb_h;
687 	struct io_queue *ioq;
688 	struct ctio_descr *c_descr;
689 
690 	c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
691 	if (c_descr->atio == NULL) {
692 		errx(1, "CTIO %p has NULL ATIO", ctio);
693 	}
694 	ioq = &((struct atio_descr *)c_descr->atio->ccb_h.targ_descr)->cmplt_io;
695 
696 	if (TAILQ_EMPTY(ioq)) {
697 		TAILQ_INSERT_HEAD(ioq, &ctio->ccb_h, periph_links.tqe);
698 		return;
699 	}
700 
701 	TAILQ_FOREACH_REVERSE(ccb_h, ioq, io_queue, periph_links.tqe) {
702 		struct ctio_descr *curr_descr =
703 		    (struct ctio_descr *)ccb_h->targ_descr;
704 		if (curr_descr->offset <= c_descr->offset) {
705 			break;
706 		}
707 	}
708 
709 	if (ccb_h) {
710 		TAILQ_INSERT_AFTER(ioq, ccb_h, &ctio->ccb_h, periph_links.tqe);
711 	} else {
712 		TAILQ_INSERT_HEAD(ioq, &ctio->ccb_h, periph_links.tqe);
713 	}
714 }
715 
716 /*
717  * Go through all completed AIO/CTIOs for a given ATIO and advance data
718  * counts, start continuation IO, etc.
719  */
720 static int
721 run_queue(struct ccb_accept_tio *atio)
722 {
723 	struct atio_descr *a_descr;
724 	struct ccb_hdr *ccb_h;
725 	int sent_status, event;
726 
727 	if (atio == NULL)
728 		return (0);
729 
730 	a_descr = (struct atio_descr *)atio->ccb_h.targ_descr;
731 
732 	while ((ccb_h = TAILQ_FIRST(&a_descr->cmplt_io)) != NULL) {
733 		struct ccb_scsiio *ctio;
734 		struct ctio_descr *c_descr;
735 
736 		ctio = (struct ccb_scsiio *)ccb_h;
737 		c_descr = (struct ctio_descr *)ctio->ccb_h.targ_descr;
738 
739 		if (ctio->ccb_h.status == CAM_REQ_ABORTED) {
740 			TAILQ_REMOVE(&a_descr->cmplt_io, ccb_h,
741 				     periph_links.tqe);
742 			free_ccb((union ccb *)ctio);
743 			send_ccb((union ccb *)atio, /*priority*/1);
744 			continue;
745 		}
746 
747 		/* If completed item is in range, call handler */
748 		if ((c_descr->event == AIO_DONE &&
749 		    c_descr->offset == a_descr->base_off + a_descr->targ_ack)
750 		 || (c_descr->event == CTIO_DONE &&
751 		    c_descr->offset == a_descr->base_off + a_descr->init_ack)) {
752 			sent_status = (ccb_h->flags & CAM_SEND_STATUS) != 0;
753 			event = c_descr->event;
754 
755 			TAILQ_REMOVE(&a_descr->cmplt_io, ccb_h,
756 				     periph_links.tqe);
757 			tcmd_handle(atio, ctio, c_descr->event);
758 
759 			/* If entire transfer complete, send back ATIO */
760 			if (sent_status != 0 && event == CTIO_DONE)
761 				send_ccb((union ccb *)atio, /*priority*/1);
762 		} else {
763 			/* Gap in offsets so wait until later callback */
764 			if (/* debug */ 1)
765 				warnx("IO %p:%p out of order %s",  ccb_h,
766 				    a_descr, c_descr->event == AIO_DONE?
767 				    "aio" : "ctio");
768 			return (1);
769 		}
770 	}
771 	return (0);
772 }
773 
774 static int
775 work_inot(struct ccb_immediate_notify *inot)
776 {
777 	cam_status status;
778 
779 	if (debug)
780 		warnx("Working on INOT %p", inot);
781 
782 	status = inot->ccb_h.status;
783 	status &= CAM_STATUS_MASK;
784 
785 	switch (status) {
786 	case CAM_SCSI_BUS_RESET:
787 		tcmd_ua(CAM_TARGET_WILDCARD, UA_BUS_RESET);
788 		abort_all_pending();
789 		break;
790 	case CAM_BDR_SENT:
791 		tcmd_ua(CAM_TARGET_WILDCARD, UA_BDR);
792 		abort_all_pending();
793 		break;
794 	case CAM_MESSAGE_RECV:
795 		switch (inot->arg) {
796 		case MSG_TASK_COMPLETE:
797 		case MSG_INITIATOR_DET_ERR:
798 		case MSG_ABORT_TASK_SET:
799 		case MSG_MESSAGE_REJECT:
800 		case MSG_NOOP:
801 		case MSG_PARITY_ERROR:
802 		case MSG_TARGET_RESET:
803 		case MSG_ABORT_TASK:
804 		case MSG_CLEAR_TASK_SET:
805 		default:
806 			warnx("INOT message %#x", inot->arg);
807 			break;
808 		}
809 		break;
810 	case CAM_REQ_ABORTED:
811 		warnx("INOT %p aborted", inot);
812 		break;
813 	default:
814 		warnx("Unhandled INOT status %#x", status);
815 		break;
816 	}
817 
818 	/* Requeue on SIM */
819 	TAILQ_REMOVE(&work_queue, &inot->ccb_h, periph_links.tqe);
820 	send_ccb((union ccb *)inot, /*priority*/1);
821 
822 	return (1);
823 }
824 
825 void
826 send_ccb(union ccb *ccb, int priority)
827 {
828 	if (debug)
829 		warnx("sending ccb (%#x)", ccb->ccb_h.func_code);
830 	ccb->ccb_h.pinfo.priority = priority;
831 	if (XPT_FC_IS_QUEUED(ccb)) {
832 		TAILQ_INSERT_TAIL(&pending_queue, &ccb->ccb_h,
833 				  periph_links.tqe);
834 	}
835 	if (write(targ_fd, &ccb, sizeof(ccb)) != sizeof(ccb)) {
836 		warn("write ccb");
837 		ccb->ccb_h.status = CAM_PROVIDE_FAIL;
838 	}
839 }
840 
841 /* Return a CTIO/descr/buf combo from the freelist or malloc one */
842 static struct ccb_scsiio *
843 get_ctio()
844 {
845 	struct ccb_scsiio *ctio;
846 	struct ctio_descr *c_descr;
847 	struct sigevent *se;
848 
849 	if (num_ctios == MAX_CTIOS) {
850 		warnx("at CTIO max");
851 		return (NULL);
852 	}
853 
854 	ctio = (struct ccb_scsiio *)malloc(sizeof(*ctio));
855 	if (ctio == NULL) {
856 		warn("malloc CTIO");
857 		return (NULL);
858 	}
859 	c_descr = (struct ctio_descr *)malloc(sizeof(*c_descr));
860 	if (c_descr == NULL) {
861 		free(ctio);
862 		warn("malloc ctio_descr");
863 		return (NULL);
864 	}
865 	c_descr->buf = malloc(buf_size);
866 	if (c_descr->buf == NULL) {
867 		free(c_descr);
868 		free(ctio);
869 		warn("malloc backing store");
870 		return (NULL);
871 	}
872 	num_ctios++;
873 
874 	/* Initialize CTIO, CTIO descr, and AIO */
875 	ctio->ccb_h.func_code = XPT_CONT_TARGET_IO;
876 	ctio->ccb_h.retry_count = 2;
877 	ctio->ccb_h.timeout = CAM_TIME_INFINITY;
878 	ctio->data_ptr = c_descr->buf;
879 	ctio->ccb_h.targ_descr = c_descr;
880 	c_descr->aiocb.aio_buf = c_descr->buf;
881 	c_descr->aiocb.aio_fildes = file_fd;
882 	se = &c_descr->aiocb.aio_sigevent;
883 	se->sigev_notify = SIGEV_KEVENT;
884 	se->sigev_notify_kqueue = kq_fd;
885 	se->sigev_value.sival_ptr = ctio;
886 
887 	return (ctio);
888 }
889 
890 void
891 free_ccb(union ccb *ccb)
892 {
893 	switch (ccb->ccb_h.func_code) {
894 	case XPT_CONT_TARGET_IO:
895 	{
896 		struct ctio_descr *c_descr;
897 
898 		c_descr = (struct ctio_descr *)ccb->ccb_h.targ_descr;
899 		free(c_descr->buf);
900 		num_ctios--;
901 		/* FALLTHROUGH */
902 	}
903 	case XPT_ACCEPT_TARGET_IO:
904 		free(ccb->ccb_h.targ_descr);
905 		/* FALLTHROUGH */
906 	case XPT_IMMED_NOTIFY:
907 	default:
908 		free(ccb);
909 		break;
910 	}
911 }
912 
913 static cam_status
914 get_sim_flags(u_int16_t *flags)
915 {
916 	struct ccb_pathinq cpi;
917 	cam_status status;
918 
919 	/* Find SIM capabilities */
920 	bzero(&cpi, sizeof(cpi));
921 	cpi.ccb_h.func_code = XPT_PATH_INQ;
922 	send_ccb((union ccb *)&cpi, /*priority*/1);
923 	status = cpi.ccb_h.status & CAM_STATUS_MASK;
924 	if (status != CAM_REQ_CMP) {
925 		fprintf(stderr, "CPI failed, status %#x\n", status);
926 		return (status);
927 	}
928 
929 	/* Can only enable on controllers that support target mode */
930 	if ((cpi.target_sprt & PIT_PROCESSOR) == 0) {
931 		fprintf(stderr, "HBA does not support target mode\n");
932 		status = CAM_PATH_INVALID;
933 		return (status);
934 	}
935 
936 	*flags = cpi.hba_inquiry;
937 	return (status);
938 }
939 
940 static void
941 rel_simq()
942 {
943 	struct ccb_relsim crs;
944 
945 	bzero(&crs, sizeof(crs));
946 	crs.ccb_h.func_code = XPT_REL_SIMQ;
947 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
948 	crs.openings = 0;
949 	crs.release_timeout = 0;
950 	crs.qfrozen_cnt = 0;
951 	send_ccb((union ccb *)&crs, /*priority*/0);
952 }
953 
954 /* Cancel all pending CCBs. */
955 static void
956 abort_all_pending()
957 {
958 	struct ccb_abort	 cab;
959 	struct ccb_hdr		*ccb_h;
960 
961 	if (debug)
962 		  warnx("abort_all_pending");
963 
964 	bzero(&cab, sizeof(cab));
965 	cab.ccb_h.func_code = XPT_ABORT;
966 	TAILQ_FOREACH(ccb_h, &pending_queue, periph_links.tqe) {
967 		if (debug)
968 			  warnx("Aborting pending CCB %p\n", ccb_h);
969 		cab.abort_ccb = (union ccb *)ccb_h;
970 		send_ccb((union ccb *)&cab, /*priority*/1);
971 		if (cab.ccb_h.status != CAM_REQ_CMP) {
972 			warnx("Unable to abort CCB, status %#x\n",
973 			       cab.ccb_h.status);
974 		}
975 	}
976 }
977 
978 static void
979 usage()
980 {
981 	fprintf(stderr,
982 		"Usage: scsi_target [-AdSTY] [-b bufsize] [-c sectorsize]\n"
983 		"\t\t[-r numbufs] [-s volsize] [-W 8,16,32]\n"
984 		"\t\tbus:target:lun filename\n");
985 	exit(1);
986 }
987