172 				      struct pass_io_req *io_req);
174 				       struct pass_io_req *io_req,
177 				     struct pass_io_req *io_req);
179 				    struct pass_io_req *io_req);
235 	struct pass_io_req *io_req, *io_req2;  in passrejectios()  local
244 	TAILQ_FOREACH_SAFE(io_req, &softc->done_queue, links, io_req2) {  in passrejectios()
245 		TAILQ_REMOVE(&softc->done_queue, io_req, links);  in passrejectios()
246 		passiocleanup(softc, io_req);  in passrejectios()
247 		uma_zfree(softc->pass_zone, io_req);  in passrejectios()
255 	TAILQ_FOREACH_SAFE(io_req, &softc->incoming_queue, links, io_req2) {  in passrejectios()
256 		TAILQ_REMOVE(&softc->incoming_queue, io_req, links);  in passrejectios()
257 		passiocleanup(softc, io_req);  in passrejectios()
258 		uma_zfree(softc->pass_zone, io_req);  in passrejectios()
270 	TAILQ_FOREACH_SAFE(io_req, &softc->active_queue, links, io_req2) {  in passrejectios()
271 		TAILQ_REMOVE(&softc->active_queue, io_req, links);  in passrejectios()
272 		io_req->flags |= PASS_IO_ABANDONED;  in passrejectios()
273 		TAILQ_INSERT_TAIL(&softc->abandoned_queue, io_req, links);  in passrejectios()
787 		struct pass_io_req *io_req, *io_req2;  in passclose()  local
789 		TAILQ_FOREACH_SAFE(io_req, &softc->done_queue, links, io_req2) {  in passclose()
790 			TAILQ_REMOVE(&softc->done_queue, io_req, links);  in passclose()
791 			passiocleanup(softc, io_req);  in passclose()
792 			uma_zfree(softc->pass_zone, io_req);  in passclose()
795 		TAILQ_FOREACH_SAFE(io_req, &softc->incoming_queue, links,  in passclose()
797 			TAILQ_REMOVE(&softc->incoming_queue, io_req, links);  in passclose()
798 			passiocleanup(softc, io_req);  in passclose()
799 			uma_zfree(softc->pass_zone, io_req);  in passclose()
808 		io_req = TAILQ_FIRST(&softc->active_queue);  in passclose()
809 		if ((io_req != NULL)  in passclose()
823 		TAILQ_FOREACH_SAFE(io_req, &softc->active_queue, links,  in passclose()
825 			TAILQ_REMOVE(&softc->active_queue, io_req, links);  in passclose()
826 			io_req->flags |= PASS_IO_ABANDONED;  in passclose()
827 			TAILQ_INSERT_TAIL(&softc->abandoned_queue, io_req,  in passclose()
860 		struct pass_io_req *io_req;  in passstart()  local
866 		io_req = TAILQ_FIRST(&softc->incoming_queue);  in passstart()
867 		if (io_req == NULL) {  in passstart()
871 		TAILQ_REMOVE(&softc->incoming_queue, io_req, links);  in passstart()
872 		TAILQ_INSERT_TAIL(&softc->active_queue, io_req, links);  in passstart()
876 		xpt_merge_ccb(start_ccb, &io_req->ccb);  in passstart()
878 		start_ccb->ccb_h.ccb_ioreq = io_req;  in passstart()
880 		io_req->alloced_ccb = start_ccb;  in passstart()
881 		binuptime(&io_req->start_time);  in passstart()
883 					  &io_req->start_time);  in passstart()
912 		struct pass_io_req *io_req;  in passdone()  local
914 		io_req = hdr->ccb_ioreq;  in passdone()
917 			  __func__, io_req->user_ccb_ptr);  in passdone()
920 		    ((io_req->flags & PASS_IO_ABANDONED) == 0)) {  in passdone()
938 		bcopy(done_ccb, &io_req->ccb, sizeof(*done_ccb));  in passdone()
952 			    &io_req->start_time);  in passdone()
962 			    &io_req->start_time);  in passdone()
979 			    &io_req->start_time);  in passdone()
985 			    &io_req->start_time);  in passdone()
994 		if ((io_req->flags & PASS_IO_ABANDONED) == 0) {  in passdone()
995 			TAILQ_REMOVE(&softc->active_queue, io_req, links);  in passdone()
996 			TAILQ_INSERT_TAIL(&softc->done_queue, io_req, links);  in passdone()
1005 			TAILQ_REMOVE(&softc->abandoned_queue, io_req, links);  in passdone()
1006 			passiocleanup(softc, io_req);  in passdone()
1007 			uma_zfree(softc->pass_zone, io_req);  in passdone()
1123 passiocleanup(struct pass_softc *softc, struct pass_io_req *io_req)  in passiocleanup()  argument
1130 	ccb = &io_req->ccb;  in passiocleanup()
1135 		numbufs = min(io_req->num_bufs, 2);  in passiocleanup()
1147 		numbufs = min(io_req->num_bufs, 1);  in passiocleanup()
1151 		numbufs = min(io_req->num_bufs, 1);  in passiocleanup()
1154 		numbufs = min(io_req->num_bufs, 2);  in passiocleanup()
1159 		numbufs = min(io_req->num_bufs, 1);  in passiocleanup()
1165 		numbufs = min(io_req->num_bufs, 1);  in passiocleanup()
1173 	if (io_req->flags & PASS_IO_USER_SEG_MALLOC) {  in passiocleanup()
1174 		free(io_req->user_segptr, M_SCSIPASS);  in passiocleanup()
1175 		io_req->user_segptr = NULL;  in passiocleanup()
1181 	if (io_req->data_flags == CAM_DATA_VADDR) {  in passiocleanup()
1182 		for (i = 0; i < io_req->num_bufs; i++) {  in passiocleanup()
1183 			if (io_req->kern_bufs[i] == NULL)  in passiocleanup()
1186 			free(io_req->kern_bufs[i], M_SCSIPASS);  in passiocleanup()
1187 			io_req->kern_bufs[i] = NULL;  in passiocleanup()
1189 	} else if (io_req->data_flags == CAM_DATA_SG) {  in passiocleanup()
1190 		for (i = 0; i < io_req->num_kern_segs; i++) {  in passiocleanup()
1192 			    io_req->kern_segptr[i].ds_addr == NULL)  in passiocleanup()
1196 			    io_req->kern_segptr[i].ds_addr);  in passiocleanup()
1197 			io_req->kern_segptr[i].ds_addr = 0;  in passiocleanup()
1201 	if (io_req->flags & PASS_IO_KERN_SEG_MALLOC) {  in passiocleanup()
1202 		free(io_req->kern_segptr, M_SCSIPASS);  in passiocleanup()
1203 		io_req->kern_segptr = NULL;  in passiocleanup()
1206 	if (io_req->data_flags != CAM_DATA_PADDR) {  in passiocleanup()
1212 			if (io_req->user_bufs[i] != NULL)  in passiocleanup()
1213 				*data_ptrs[i] = io_req->user_bufs[i];  in passiocleanup()
1220 passcopysglist(struct cam_periph *periph, struct pass_io_req *io_req,  in passcopysglist()  argument
1231 	user_sglist = io_req->user_segptr;  in passcopysglist()
1232 	kern_sglist = io_req->kern_segptr;  in passcopysglist()
1234 	for (i = 0, j = 0; i < io_req->num_user_segs &&  in passcopysglist()
1235 	     j < io_req->num_kern_segs;) {  in passcopysglist()
1286 passmemsetup(struct cam_periph *periph, struct pass_io_req *io_req)  in passmemsetup()  argument
1305 	ccb = &io_req->ccb;  in passmemsetup()
1331 		io_req->data_flags = CAM_DATA_VADDR;  in passmemsetup()
1344 		io_req->data_flags = hdr->flags & CAM_DATA_MASK;  in passmemsetup()
1364 		io_req->data_flags = CAM_DATA_VADDR;  in passmemsetup()
1373 		io_req->data_flags = CAM_DATA_VADDR;  in passmemsetup()
1388 		io_req->data_flags = CAM_DATA_VADDR;  in passmemsetup()
1400 		io_req->data_flags = hdr->flags & CAM_DATA_MASK;  in passmemsetup()
1415 	io_req->num_bufs = numbufs;  in passmemsetup()
1424 		io_req->user_bufs[i] = *data_ptrs[i];  in passmemsetup()
1425 		io_req->dirs[i] = dirs[i];  in passmemsetup()
1426 		io_req->lengths[i] = lengths[i];  in passmemsetup()
1440 	switch (io_req->data_flags) {  in passmemsetup()
1450 			if (io_req->lengths[i] == 0)  in passmemsetup()
1455 			io_req->kern_bufs[i] = tmp_buf;  in passmemsetup()
1461 				  tmp_buf, lengths[i], io_req->user_bufs[i],  in passmemsetup()
1470 			error = copyin(io_req->user_bufs[i],  in passmemsetup()
1471 				       io_req->kern_bufs[i], lengths[i]);  in passmemsetup()
1476 					  io_req->user_bufs[i],  in passmemsetup()
1477 					  io_req->kern_bufs[i], error);  in passmemsetup()
1538 		io_req->num_user_segs = num_segs;  in passmemsetup()
1539 		io_req->num_kern_segs = num_segs_needed;  in passmemsetup()
1542 		io_req->user_bufs[0] = *data_ptrs[0];  in passmemsetup()
1549 			io_req->user_segptr = malloc(sizeof(bus_dma_segment_t) *  in passmemsetup()
1551 			io_req->flags |= PASS_IO_USER_SEG_MALLOC;  in passmemsetup()
1553 			io_req->user_segptr = io_req->user_segs;  in passmemsetup()
1555 		error = copyin(*data_ptrs[0], io_req->user_segptr, sg_length);  in passmemsetup()
1559 				  __func__, *data_ptrs[0], io_req->user_segptr,  in passmemsetup()
1565 			io_req->kern_segptr = malloc(sizeof(bus_dma_segment_t) *  in passmemsetup()
1567 			io_req->flags |= PASS_IO_KERN_SEG_MALLOC;  in passmemsetup()
1569 			io_req->kern_segptr = io_req->kern_segs;  in passmemsetup()
1582 			io_req->kern_segptr[i].ds_addr =  in passmemsetup()
1584 			io_req->kern_segptr[i].ds_len = alloc_size;  in passmemsetup()
1593 		*data_ptrs[0] = (uint8_t *)io_req->kern_segptr;  in passmemsetup()
1594 		*seg_cnt_ptr = io_req->num_kern_segs;  in passmemsetup()
1600 			error = passcopysglist(periph, io_req, dirs[0]);  in passmemsetup()
1639 		io_req->num_user_segs = num_segs;  in passmemsetup()
1640 		io_req->num_kern_segs = io_req->num_user_segs;  in passmemsetup()
1643 		io_req->user_bufs[0] = *data_ptrs[0];  in passmemsetup()
1646 			io_req->user_segptr = malloc(sizeof(bus_dma_segment_t) *  in passmemsetup()
1648 			io_req->flags |= PASS_IO_USER_SEG_MALLOC;  in passmemsetup()
1650 			io_req->user_segptr = io_req->user_segs;  in passmemsetup()
1652 		io_req->kern_segptr = io_req->user_segptr;  in passmemsetup()
1654 		error = copyin(*data_ptrs[0], io_req->user_segptr, sg_length);  in passmemsetup()
1658 				  __func__, *data_ptrs[0], io_req->user_segptr,  in passmemsetup()
1676 		passiocleanup(softc, io_req);  in passmemsetup()
1682 passmemdone(struct cam_periph *periph, struct pass_io_req *io_req)  in passmemdone()  argument
1691 	switch (io_req->data_flags) {  in passmemdone()
1696 		for (i = 0; i < io_req->num_bufs; i++) {  in passmemdone()
1697 			if (io_req->dirs[i] != CAM_DIR_IN)  in passmemdone()
1700 			error = copyout(io_req->kern_bufs[i],  in passmemdone()
1701 			    io_req->user_bufs[i], io_req->lengths[i]);  in passmemdone()
1705 					  io_req->lengths[i],  in passmemdone()
1706 					  io_req->kern_bufs[i],  in passmemdone()
1707 					  io_req->user_bufs[i]);  in passmemdone()
1720 		if (io_req->dirs[0] == CAM_DIR_IN)  in passmemdone()
1721 			error = passcopysglist(periph, io_req, io_req->dirs[0]);  in passmemdone()
1740 	passiocleanup(softc, io_req);  in passmemdone()
1841 		struct pass_io_req *io_req;  in passdoioctl()  local
1863 		io_req = uma_zalloc(softc->pass_zone, M_WAITOK | M_ZERO);  in passdoioctl()
1864 		ccb = &io_req->ccb;  in passdoioctl()
1928 		io_req->user_ccb_ptr = *user_ccb;  in passdoioctl()
1929 		io_req->user_periph_links = ccb->ccb_h.periph_links;  in passdoioctl()
1930 		io_req->user_periph_priv = ccb->ccb_h.periph_priv;  in passdoioctl()
1936 		ccb->ccb_h.ccb_ioreq = io_req;  in passdoioctl()
1966 			error = passmemsetup(periph, io_req);  in passdoioctl()
1970 			io_req->mapinfo.num_bufs_used = 0;  in passdoioctl()
1977 		TAILQ_INSERT_TAIL(&softc->incoming_queue, io_req, links);  in passdoioctl()
1998 		TAILQ_REMOVE(&softc->incoming_queue, io_req, links);  in passdoioctl()
1999 		TAILQ_INSERT_TAIL(&softc->active_queue, io_req, links);  in passdoioctl()
2009 			TAILQ_REMOVE(&softc->active_queue, io_req, links);  in passdoioctl()
2010 			TAILQ_INSERT_TAIL(&softc->done_queue, io_req, links);  in passdoioctl()
2015 		uma_zfree(softc->pass_zone, io_req);  in passdoioctl()
2022 		struct pass_io_req *io_req;  in passdoioctl()  local
2034 		io_req = TAILQ_FIRST(&softc->done_queue);  in passdoioctl()
2035 		if (io_req == NULL) {  in passdoioctl()
2043 		TAILQ_REMOVE(&softc->done_queue, io_req, links);  in passdoioctl()
2055 		error = passmemdone(periph, io_req);  in passdoioctl()
2059 		io_req->ccb.ccb_h.periph_links = io_req->user_periph_links;  in passdoioctl()
2060 		io_req->ccb.ccb_h.periph_priv = io_req->user_periph_priv;  in passdoioctl()
2064 			  "kernel address %p\n", *user_ccb, &io_req->ccb);  in passdoioctl()
2067 		error = copyout(&io_req->ccb, *user_ccb, sizeof(union ccb));  in passdoioctl()
2071 				  *user_ccb, &io_req->ccb, error);  in passdoioctl()
2089 		uma_zfree(softc->pass_zone, io_req);  in passdoioctl()