xref: /freebsd/sys/cam/cam_xpt.c (revision e796cc77c586c2955b2f3940dbf4991b31e8d289)
1 /*-
2  * Implementation of the Common Access Method Transport (XPT) layer.
3  *
4  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
5  *
6  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
7  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
8  * All rights reserved.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions, and the following disclaimer,
15  *    without modification, immediately at the beginning of the file.
16  * 2. The name of the author may not be used to endorse or promote products
17  *    derived from this software without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
23  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  */
31 
32 #include "opt_printf.h"
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #include <sys/param.h>
38 #include <sys/bio.h>
39 #include <sys/bus.h>
40 #include <sys/systm.h>
41 #include <sys/types.h>
42 #include <sys/malloc.h>
43 #include <sys/kernel.h>
44 #include <sys/time.h>
45 #include <sys/conf.h>
46 #include <sys/fcntl.h>
47 #include <sys/interrupt.h>
48 #include <sys/proc.h>
49 #include <sys/sbuf.h>
50 #include <sys/smp.h>
51 #include <sys/taskqueue.h>
52 
53 #include <sys/lock.h>
54 #include <sys/mutex.h>
55 #include <sys/sysctl.h>
56 #include <sys/kthread.h>
57 
58 #include <cam/cam.h>
59 #include <cam/cam_ccb.h>
60 #include <cam/cam_iosched.h>
61 #include <cam/cam_periph.h>
62 #include <cam/cam_queue.h>
63 #include <cam/cam_sim.h>
64 #include <cam/cam_xpt.h>
65 #include <cam/cam_xpt_sim.h>
66 #include <cam/cam_xpt_periph.h>
67 #include <cam/cam_xpt_internal.h>
68 #include <cam/cam_debug.h>
69 #include <cam/cam_compat.h>
70 
71 #include <cam/scsi/scsi_all.h>
72 #include <cam/scsi/scsi_message.h>
73 #include <cam/scsi/scsi_pass.h>
74 
75 #include <machine/md_var.h>	/* geometry translation */
76 #include <machine/stdarg.h>	/* for xpt_print below */
77 
78 #include "opt_cam.h"
79 
80 /* Wild guess based on not wanting to grow the stack too much */
81 #define XPT_PRINT_MAXLEN	512
82 #ifdef PRINTF_BUFR_SIZE
83 #define XPT_PRINT_LEN	PRINTF_BUFR_SIZE
84 #else
85 #define XPT_PRINT_LEN	128
86 #endif
87 _Static_assert(XPT_PRINT_LEN <= XPT_PRINT_MAXLEN, "XPT_PRINT_LEN is too large");
88 
89 /*
90  * This is the maximum number of high powered commands (e.g. start unit)
91  * that can be outstanding at a particular time.
92  */
93 #ifndef CAM_MAX_HIGHPOWER
94 #define CAM_MAX_HIGHPOWER  4
95 #endif
96 
97 /* Datastructures internal to the xpt layer */
98 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
99 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
100 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
101 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
102 
103 /* Object for defering XPT actions to a taskqueue */
104 struct xpt_task {
105 	struct task	task;
106 	void		*data1;
107 	uintptr_t	data2;
108 };
109 
110 struct xpt_softc {
111 	uint32_t		xpt_generation;
112 
113 	/* number of high powered commands that can go through right now */
114 	struct mtx		xpt_highpower_lock;
115 	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
116 	int			num_highpower;
117 
118 	/* queue for handling async rescan requests. */
119 	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
120 	int buses_to_config;
121 	int buses_config_done;
122 	int announce_nosbuf;
123 
124 	/*
125 	 * Registered buses
126 	 *
127 	 * N.B., "busses" is an archaic spelling of "buses".  In new code
128 	 * "buses" is preferred.
129 	 */
130 	TAILQ_HEAD(,cam_eb)	xpt_busses;
131 	u_int			bus_generation;
132 
133 	struct intr_config_hook	*xpt_config_hook;
134 
135 	int			boot_delay;
136 	struct callout 		boot_callout;
137 
138 	struct mtx		xpt_topo_lock;
139 	struct mtx		xpt_lock;
140 	struct taskqueue	*xpt_taskq;
141 };
142 
143 typedef enum {
144 	DM_RET_COPY		= 0x01,
145 	DM_RET_FLAG_MASK	= 0x0f,
146 	DM_RET_NONE		= 0x00,
147 	DM_RET_STOP		= 0x10,
148 	DM_RET_DESCEND		= 0x20,
149 	DM_RET_ERROR		= 0x30,
150 	DM_RET_ACTION_MASK	= 0xf0
151 } dev_match_ret;
152 
153 typedef enum {
154 	XPT_DEPTH_BUS,
155 	XPT_DEPTH_TARGET,
156 	XPT_DEPTH_DEVICE,
157 	XPT_DEPTH_PERIPH
158 } xpt_traverse_depth;
159 
160 struct xpt_traverse_config {
161 	xpt_traverse_depth	depth;
162 	void			*tr_func;
163 	void			*tr_arg;
164 };
165 
166 typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
167 typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
168 typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
169 typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
170 typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
171 
172 /* Transport layer configuration information */
173 static struct xpt_softc xsoftc;
174 
175 MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
176 
177 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
178            &xsoftc.boot_delay, 0, "Bus registration wait time");
179 SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
180 	    &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
181 SYSCTL_INT(_kern_cam, OID_AUTO, announce_nosbuf, CTLFLAG_RWTUN,
182 	    &xsoftc.announce_nosbuf, 0, "Don't use sbuf for announcements");
183 
184 struct cam_doneq {
185 	struct mtx_padalign	cam_doneq_mtx;
186 	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
187 	int			cam_doneq_sleep;
188 };
189 
190 static struct cam_doneq cam_doneqs[MAXCPU];
191 static int cam_num_doneqs;
192 static struct proc *cam_proc;
193 
194 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
195            &cam_num_doneqs, 0, "Number of completion queues/threads");
196 
197 struct cam_periph *xpt_periph;
198 
199 static periph_init_t xpt_periph_init;
200 
201 static struct periph_driver xpt_driver =
202 {
203 	xpt_periph_init, "xpt",
204 	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
205 	CAM_PERIPH_DRV_EARLY
206 };
207 
208 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
209 
210 static d_open_t xptopen;
211 static d_close_t xptclose;
212 static d_ioctl_t xptioctl;
213 static d_ioctl_t xptdoioctl;
214 
215 static struct cdevsw xpt_cdevsw = {
216 	.d_version =	D_VERSION,
217 	.d_flags =	0,
218 	.d_open =	xptopen,
219 	.d_close =	xptclose,
220 	.d_ioctl =	xptioctl,
221 	.d_name =	"xpt",
222 };
223 
224 /* Storage for debugging datastructures */
225 struct cam_path *cam_dpath;
226 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
227 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RWTUN,
228 	&cam_dflags, 0, "Enabled debug flags");
229 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
230 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RWTUN,
231 	&cam_debug_delay, 0, "Delay in us after each debug message");
232 
233 /* Our boot-time initialization hook */
234 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
235 
236 static moduledata_t cam_moduledata = {
237 	"cam",
238 	cam_module_event_handler,
239 	NULL
240 };
241 
242 static int	xpt_init(void *);
243 
244 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
245 MODULE_VERSION(cam, 1);
246 
247 
248 static void		xpt_async_bcast(struct async_list *async_head,
249 					u_int32_t async_code,
250 					struct cam_path *path,
251 					void *async_arg);
252 static path_id_t xptnextfreepathid(void);
253 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
254 static union ccb *xpt_get_ccb(struct cam_periph *periph);
255 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
256 static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
257 static void	 xpt_run_allocq_task(void *context, int pending);
258 static void	 xpt_run_devq(struct cam_devq *devq);
259 static timeout_t xpt_release_devq_timeout;
260 static void	 xpt_release_simq_timeout(void *arg) __unused;
261 static void	 xpt_acquire_bus(struct cam_eb *bus);
262 static void	 xpt_release_bus(struct cam_eb *bus);
263 static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
264 static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
265 		    int run_queue);
266 static struct cam_et*
267 		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
268 static void	 xpt_acquire_target(struct cam_et *target);
269 static void	 xpt_release_target(struct cam_et *target);
270 static struct cam_eb*
271 		 xpt_find_bus(path_id_t path_id);
272 static struct cam_et*
273 		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
274 static struct cam_ed*
275 		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
276 static void	 xpt_config(void *arg);
277 static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
278 				 u_int32_t new_priority);
279 static xpt_devicefunc_t xptpassannouncefunc;
280 static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
281 static void	 xptpoll(struct cam_sim *sim);
282 static void	 camisr_runqueue(void);
283 static void	 xpt_done_process(struct ccb_hdr *ccb_h);
284 static void	 xpt_done_td(void *);
285 static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
286 				    u_int num_patterns, struct cam_eb *bus);
287 static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
288 				       u_int num_patterns,
289 				       struct cam_ed *device);
290 static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
291 				       u_int num_patterns,
292 				       struct cam_periph *periph);
293 static xpt_busfunc_t	xptedtbusfunc;
294 static xpt_targetfunc_t	xptedttargetfunc;
295 static xpt_devicefunc_t	xptedtdevicefunc;
296 static xpt_periphfunc_t	xptedtperiphfunc;
297 static xpt_pdrvfunc_t	xptplistpdrvfunc;
298 static xpt_periphfunc_t	xptplistperiphfunc;
299 static int		xptedtmatch(struct ccb_dev_match *cdm);
300 static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
301 static int		xptbustraverse(struct cam_eb *start_bus,
302 				       xpt_busfunc_t *tr_func, void *arg);
303 static int		xpttargettraverse(struct cam_eb *bus,
304 					  struct cam_et *start_target,
305 					  xpt_targetfunc_t *tr_func, void *arg);
306 static int		xptdevicetraverse(struct cam_et *target,
307 					  struct cam_ed *start_device,
308 					  xpt_devicefunc_t *tr_func, void *arg);
309 static int		xptperiphtraverse(struct cam_ed *device,
310 					  struct cam_periph *start_periph,
311 					  xpt_periphfunc_t *tr_func, void *arg);
312 static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
313 					xpt_pdrvfunc_t *tr_func, void *arg);
314 static int		xptpdperiphtraverse(struct periph_driver **pdrv,
315 					    struct cam_periph *start_periph,
316 					    xpt_periphfunc_t *tr_func,
317 					    void *arg);
318 static xpt_busfunc_t	xptdefbusfunc;
319 static xpt_targetfunc_t	xptdeftargetfunc;
320 static xpt_devicefunc_t	xptdefdevicefunc;
321 static xpt_periphfunc_t	xptdefperiphfunc;
322 static void		xpt_finishconfig_task(void *context, int pending);
323 static void		xpt_dev_async_default(u_int32_t async_code,
324 					      struct cam_eb *bus,
325 					      struct cam_et *target,
326 					      struct cam_ed *device,
327 					      void *async_arg);
328 static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
329 						 struct cam_et *target,
330 						 lun_id_t lun_id);
331 static xpt_devicefunc_t	xptsetasyncfunc;
332 static xpt_busfunc_t	xptsetasyncbusfunc;
333 static cam_status	xptregister(struct cam_periph *periph,
334 				    void *arg);
335 static __inline int device_is_queued(struct cam_ed *device);
336 
337 static __inline int
338 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
339 {
340 	int	retval;
341 
342 	mtx_assert(&devq->send_mtx, MA_OWNED);
343 	if ((dev->ccbq.queue.entries > 0) &&
344 	    (dev->ccbq.dev_openings > 0) &&
345 	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
346 		/*
347 		 * The priority of a device waiting for controller
348 		 * resources is that of the highest priority CCB
349 		 * enqueued.
350 		 */
351 		retval =
352 		    xpt_schedule_dev(&devq->send_queue,
353 				     &dev->devq_entry,
354 				     CAMQ_GET_PRIO(&dev->ccbq.queue));
355 	} else {
356 		retval = 0;
357 	}
358 	return (retval);
359 }
360 
361 static __inline int
362 device_is_queued(struct cam_ed *device)
363 {
364 	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
365 }
366 
367 static void
368 xpt_periph_init()
369 {
370 	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
371 }
372 
373 static int
374 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
375 {
376 
377 	/*
378 	 * Only allow read-write access.
379 	 */
380 	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
381 		return(EPERM);
382 
383 	/*
384 	 * We don't allow nonblocking access.
385 	 */
386 	if ((flags & O_NONBLOCK) != 0) {
387 		printf("%s: can't do nonblocking access\n", devtoname(dev));
388 		return(ENODEV);
389 	}
390 
391 	return(0);
392 }
393 
394 static int
395 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
396 {
397 
398 	return(0);
399 }
400 
401 /*
402  * Don't automatically grab the xpt softc lock here even though this is going
403  * through the xpt device.  The xpt device is really just a back door for
404  * accessing other devices and SIMs, so the right thing to do is to grab
405  * the appropriate SIM lock once the bus/SIM is located.
406  */
407 static int
408 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
409 {
410 	int error;
411 
412 	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
413 		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
414 	}
415 	return (error);
416 }
417 
418 static int
419 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
420 {
421 	int error;
422 
423 	error = 0;
424 
425 	switch(cmd) {
426 	/*
427 	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
428 	 * to accept CCB types that don't quite make sense to send through a
429 	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
430 	 * in the CAM spec.
431 	 */
432 	case CAMIOCOMMAND: {
433 		union ccb *ccb;
434 		union ccb *inccb;
435 		struct cam_eb *bus;
436 
437 		inccb = (union ccb *)addr;
438 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
439 		if (inccb->ccb_h.func_code == XPT_SCSI_IO)
440 			inccb->csio.bio = NULL;
441 #endif
442 
443 		if (inccb->ccb_h.flags & CAM_UNLOCKED)
444 			return (EINVAL);
445 
446 		bus = xpt_find_bus(inccb->ccb_h.path_id);
447 		if (bus == NULL)
448 			return (EINVAL);
449 
450 		switch (inccb->ccb_h.func_code) {
451 		case XPT_SCAN_BUS:
452 		case XPT_RESET_BUS:
453 			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
454 			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
455 				xpt_release_bus(bus);
456 				return (EINVAL);
457 			}
458 			break;
459 		case XPT_SCAN_TGT:
460 			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
461 			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
462 				xpt_release_bus(bus);
463 				return (EINVAL);
464 			}
465 			break;
466 		default:
467 			break;
468 		}
469 
470 		switch(inccb->ccb_h.func_code) {
471 		case XPT_SCAN_BUS:
472 		case XPT_RESET_BUS:
473 		case XPT_PATH_INQ:
474 		case XPT_ENG_INQ:
475 		case XPT_SCAN_LUN:
476 		case XPT_SCAN_TGT:
477 
478 			ccb = xpt_alloc_ccb();
479 
480 			/*
481 			 * Create a path using the bus, target, and lun the
482 			 * user passed in.
483 			 */
484 			if (xpt_create_path(&ccb->ccb_h.path, NULL,
485 					    inccb->ccb_h.path_id,
486 					    inccb->ccb_h.target_id,
487 					    inccb->ccb_h.target_lun) !=
488 					    CAM_REQ_CMP){
489 				error = EINVAL;
490 				xpt_free_ccb(ccb);
491 				break;
492 			}
493 			/* Ensure all of our fields are correct */
494 			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
495 				      inccb->ccb_h.pinfo.priority);
496 			xpt_merge_ccb(ccb, inccb);
497 			xpt_path_lock(ccb->ccb_h.path);
498 			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
499 			xpt_path_unlock(ccb->ccb_h.path);
500 			bcopy(ccb, inccb, sizeof(union ccb));
501 			xpt_free_path(ccb->ccb_h.path);
502 			xpt_free_ccb(ccb);
503 			break;
504 
505 		case XPT_DEBUG: {
506 			union ccb ccb;
507 
508 			/*
509 			 * This is an immediate CCB, so it's okay to
510 			 * allocate it on the stack.
511 			 */
512 
513 			/*
514 			 * Create a path using the bus, target, and lun the
515 			 * user passed in.
516 			 */
517 			if (xpt_create_path(&ccb.ccb_h.path, NULL,
518 					    inccb->ccb_h.path_id,
519 					    inccb->ccb_h.target_id,
520 					    inccb->ccb_h.target_lun) !=
521 					    CAM_REQ_CMP){
522 				error = EINVAL;
523 				break;
524 			}
525 			/* Ensure all of our fields are correct */
526 			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
527 				      inccb->ccb_h.pinfo.priority);
528 			xpt_merge_ccb(&ccb, inccb);
529 			xpt_action(&ccb);
530 			bcopy(&ccb, inccb, sizeof(union ccb));
531 			xpt_free_path(ccb.ccb_h.path);
532 			break;
533 
534 		}
535 		case XPT_DEV_MATCH: {
536 			struct cam_periph_map_info mapinfo;
537 			struct cam_path *old_path;
538 
539 			/*
540 			 * We can't deal with physical addresses for this
541 			 * type of transaction.
542 			 */
543 			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
544 			    CAM_DATA_VADDR) {
545 				error = EINVAL;
546 				break;
547 			}
548 
549 			/*
550 			 * Save this in case the caller had it set to
551 			 * something in particular.
552 			 */
553 			old_path = inccb->ccb_h.path;
554 
555 			/*
556 			 * We really don't need a path for the matching
557 			 * code.  The path is needed because of the
558 			 * debugging statements in xpt_action().  They
559 			 * assume that the CCB has a valid path.
560 			 */
561 			inccb->ccb_h.path = xpt_periph->path;
562 
563 			bzero(&mapinfo, sizeof(mapinfo));
564 
565 			/*
566 			 * Map the pattern and match buffers into kernel
567 			 * virtual address space.
568 			 */
569 			error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
570 
571 			if (error) {
572 				inccb->ccb_h.path = old_path;
573 				break;
574 			}
575 
576 			/*
577 			 * This is an immediate CCB, we can send it on directly.
578 			 */
579 			xpt_action(inccb);
580 
581 			/*
582 			 * Map the buffers back into user space.
583 			 */
584 			cam_periph_unmapmem(inccb, &mapinfo);
585 
586 			inccb->ccb_h.path = old_path;
587 
588 			error = 0;
589 			break;
590 		}
591 		default:
592 			error = ENOTSUP;
593 			break;
594 		}
595 		xpt_release_bus(bus);
596 		break;
597 	}
598 	/*
599 	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
600 	 * with the periphal driver name and unit name filled in.  The other
601 	 * fields don't really matter as input.  The passthrough driver name
602 	 * ("pass"), and unit number are passed back in the ccb.  The current
603 	 * device generation number, and the index into the device peripheral
604 	 * driver list, and the status are also passed back.  Note that
605 	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
606 	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
607 	 * (or rather should be) impossible for the device peripheral driver
608 	 * list to change since we look at the whole thing in one pass, and
609 	 * we do it with lock protection.
610 	 *
611 	 */
612 	case CAMGETPASSTHRU: {
613 		union ccb *ccb;
614 		struct cam_periph *periph;
615 		struct periph_driver **p_drv;
616 		char   *name;
617 		u_int unit;
618 		int base_periph_found;
619 
620 		ccb = (union ccb *)addr;
621 		unit = ccb->cgdl.unit_number;
622 		name = ccb->cgdl.periph_name;
623 		base_periph_found = 0;
624 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
625 		if (ccb->ccb_h.func_code == XPT_SCSI_IO)
626 			ccb->csio.bio = NULL;
627 #endif
628 
629 		/*
630 		 * Sanity check -- make sure we don't get a null peripheral
631 		 * driver name.
632 		 */
633 		if (*ccb->cgdl.periph_name == '\0') {
634 			error = EINVAL;
635 			break;
636 		}
637 
638 		/* Keep the list from changing while we traverse it */
639 		xpt_lock_buses();
640 
641 		/* first find our driver in the list of drivers */
642 		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
643 			if (strcmp((*p_drv)->driver_name, name) == 0)
644 				break;
645 
646 		if (*p_drv == NULL) {
647 			xpt_unlock_buses();
648 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
649 			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
650 			*ccb->cgdl.periph_name = '\0';
651 			ccb->cgdl.unit_number = 0;
652 			error = ENOENT;
653 			break;
654 		}
655 
656 		/*
657 		 * Run through every peripheral instance of this driver
658 		 * and check to see whether it matches the unit passed
659 		 * in by the user.  If it does, get out of the loops and
660 		 * find the passthrough driver associated with that
661 		 * peripheral driver.
662 		 */
663 		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
664 		     periph = TAILQ_NEXT(periph, unit_links)) {
665 
666 			if (periph->unit_number == unit)
667 				break;
668 		}
669 		/*
670 		 * If we found the peripheral driver that the user passed
671 		 * in, go through all of the peripheral drivers for that
672 		 * particular device and look for a passthrough driver.
673 		 */
674 		if (periph != NULL) {
675 			struct cam_ed *device;
676 			int i;
677 
678 			base_periph_found = 1;
679 			device = periph->path->device;
680 			for (i = 0, periph = SLIST_FIRST(&device->periphs);
681 			     periph != NULL;
682 			     periph = SLIST_NEXT(periph, periph_links), i++) {
683 				/*
684 				 * Check to see whether we have a
685 				 * passthrough device or not.
686 				 */
687 				if (strcmp(periph->periph_name, "pass") == 0) {
688 					/*
689 					 * Fill in the getdevlist fields.
690 					 */
691 					strlcpy(ccb->cgdl.periph_name,
692 					       periph->periph_name,
693 					       sizeof(ccb->cgdl.periph_name));
694 					ccb->cgdl.unit_number =
695 						periph->unit_number;
696 					if (SLIST_NEXT(periph, periph_links))
697 						ccb->cgdl.status =
698 							CAM_GDEVLIST_MORE_DEVS;
699 					else
700 						ccb->cgdl.status =
701 						       CAM_GDEVLIST_LAST_DEVICE;
702 					ccb->cgdl.generation =
703 						device->generation;
704 					ccb->cgdl.index = i;
705 					/*
706 					 * Fill in some CCB header fields
707 					 * that the user may want.
708 					 */
709 					ccb->ccb_h.path_id =
710 						periph->path->bus->path_id;
711 					ccb->ccb_h.target_id =
712 						periph->path->target->target_id;
713 					ccb->ccb_h.target_lun =
714 						periph->path->device->lun_id;
715 					ccb->ccb_h.status = CAM_REQ_CMP;
716 					break;
717 				}
718 			}
719 		}
720 
721 		/*
722 		 * If the periph is null here, one of two things has
723 		 * happened.  The first possibility is that we couldn't
724 		 * find the unit number of the particular peripheral driver
725 		 * that the user is asking about.  e.g. the user asks for
726 		 * the passthrough driver for "da11".  We find the list of
727 		 * "da" peripherals all right, but there is no unit 11.
728 		 * The other possibility is that we went through the list
729 		 * of peripheral drivers attached to the device structure,
730 		 * but didn't find one with the name "pass".  Either way,
731 		 * we return ENOENT, since we couldn't find something.
732 		 */
733 		if (periph == NULL) {
734 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
735 			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
736 			*ccb->cgdl.periph_name = '\0';
737 			ccb->cgdl.unit_number = 0;
738 			error = ENOENT;
739 			/*
740 			 * It is unfortunate that this is even necessary,
741 			 * but there are many, many clueless users out there.
742 			 * If this is true, the user is looking for the
743 			 * passthrough driver, but doesn't have one in his
744 			 * kernel.
745 			 */
746 			if (base_periph_found == 1) {
747 				printf("xptioctl: pass driver is not in the "
748 				       "kernel\n");
749 				printf("xptioctl: put \"device pass\" in "
750 				       "your kernel config file\n");
751 			}
752 		}
753 		xpt_unlock_buses();
754 		break;
755 		}
756 	default:
757 		error = ENOTTY;
758 		break;
759 	}
760 
761 	return(error);
762 }
763 
764 static int
765 cam_module_event_handler(module_t mod, int what, void *arg)
766 {
767 	int error;
768 
769 	switch (what) {
770 	case MOD_LOAD:
771 		if ((error = xpt_init(NULL)) != 0)
772 			return (error);
773 		break;
774 	case MOD_UNLOAD:
775 		return EBUSY;
776 	default:
777 		return EOPNOTSUPP;
778 	}
779 
780 	return 0;
781 }
782 
783 static struct xpt_proto *
784 xpt_proto_find(cam_proto proto)
785 {
786 	struct xpt_proto **pp;
787 
788 	SET_FOREACH(pp, cam_xpt_proto_set) {
789 		if ((*pp)->proto == proto)
790 			return *pp;
791 	}
792 
793 	return NULL;
794 }
795 
796 static void
797 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
798 {
799 
800 	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
801 		xpt_free_path(done_ccb->ccb_h.path);
802 		xpt_free_ccb(done_ccb);
803 	} else {
804 		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
805 		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
806 	}
807 	xpt_release_boot();
808 }
809 
810 /* thread to handle bus rescans */
811 static void
812 xpt_scanner_thread(void *dummy)
813 {
814 	union ccb	*ccb;
815 	struct cam_path	 path;
816 
817 	xpt_lock_buses();
818 	for (;;) {
819 		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
820 			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
821 			       "-", 0);
822 		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
823 			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
824 			xpt_unlock_buses();
825 
826 			/*
827 			 * Since lock can be dropped inside and path freed
828 			 * by completion callback even before return here,
829 			 * take our own path copy for reference.
830 			 */
831 			xpt_copy_path(&path, ccb->ccb_h.path);
832 			xpt_path_lock(&path);
833 			xpt_action(ccb);
834 			xpt_path_unlock(&path);
835 			xpt_release_path(&path);
836 
837 			xpt_lock_buses();
838 		}
839 	}
840 }
841 
842 void
843 xpt_rescan(union ccb *ccb)
844 {
845 	struct ccb_hdr *hdr;
846 
847 	/* Prepare request */
848 	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
849 	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
850 		ccb->ccb_h.func_code = XPT_SCAN_BUS;
851 	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
852 	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
853 		ccb->ccb_h.func_code = XPT_SCAN_TGT;
854 	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
855 	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
856 		ccb->ccb_h.func_code = XPT_SCAN_LUN;
857 	else {
858 		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
859 		xpt_free_path(ccb->ccb_h.path);
860 		xpt_free_ccb(ccb);
861 		return;
862 	}
863 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
864 	    ("xpt_rescan: func %#x %s\n", ccb->ccb_h.func_code,
865  		xpt_action_name(ccb->ccb_h.func_code)));
866 
867 	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
868 	ccb->ccb_h.cbfcnp = xpt_rescan_done;
869 	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
870 	/* Don't make duplicate entries for the same paths. */
871 	xpt_lock_buses();
872 	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
873 		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
874 			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
875 				wakeup(&xsoftc.ccb_scanq);
876 				xpt_unlock_buses();
877 				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
878 				xpt_free_path(ccb->ccb_h.path);
879 				xpt_free_ccb(ccb);
880 				return;
881 			}
882 		}
883 	}
884 	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
885 	xsoftc.buses_to_config++;
886 	wakeup(&xsoftc.ccb_scanq);
887 	xpt_unlock_buses();
888 }
889 
890 /* Functions accessed by the peripheral drivers */
891 static int
892 xpt_init(void *dummy)
893 {
894 	struct cam_sim *xpt_sim;
895 	struct cam_path *path;
896 	struct cam_devq *devq;
897 	cam_status status;
898 	int error, i;
899 
900 	TAILQ_INIT(&xsoftc.xpt_busses);
901 	TAILQ_INIT(&xsoftc.ccb_scanq);
902 	STAILQ_INIT(&xsoftc.highpowerq);
903 	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
904 
905 	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
906 	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
907 	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
908 	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
909 
910 #ifdef CAM_BOOT_DELAY
911 	/*
912 	 * Override this value at compile time to assist our users
913 	 * who don't use loader to boot a kernel.
914 	 */
915 	xsoftc.boot_delay = CAM_BOOT_DELAY;
916 #endif
917 	/*
918 	 * The xpt layer is, itself, the equivalent of a SIM.
919 	 * Allow 16 ccbs in the ccb pool for it.  This should
920 	 * give decent parallelism when we probe buses and
921 	 * perform other XPT functions.
922 	 */
923 	devq = cam_simq_alloc(16);
924 	xpt_sim = cam_sim_alloc(xptaction,
925 				xptpoll,
926 				"xpt",
927 				/*softc*/NULL,
928 				/*unit*/0,
929 				/*mtx*/&xsoftc.xpt_lock,
930 				/*max_dev_transactions*/0,
931 				/*max_tagged_dev_transactions*/0,
932 				devq);
933 	if (xpt_sim == NULL)
934 		return (ENOMEM);
935 
936 	mtx_lock(&xsoftc.xpt_lock);
937 	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
938 		mtx_unlock(&xsoftc.xpt_lock);
939 		printf("xpt_init: xpt_bus_register failed with status %#x,"
940 		       " failing attach\n", status);
941 		return (EINVAL);
942 	}
943 	mtx_unlock(&xsoftc.xpt_lock);
944 
945 	/*
946 	 * Looking at the XPT from the SIM layer, the XPT is
947 	 * the equivalent of a peripheral driver.  Allocate
948 	 * a peripheral driver entry for us.
949 	 */
950 	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
951 				      CAM_TARGET_WILDCARD,
952 				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
953 		printf("xpt_init: xpt_create_path failed with status %#x,"
954 		       " failing attach\n", status);
955 		return (EINVAL);
956 	}
957 	xpt_path_lock(path);
958 	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
959 			 path, NULL, 0, xpt_sim);
960 	xpt_path_unlock(path);
961 	xpt_free_path(path);
962 
963 	if (cam_num_doneqs < 1)
964 		cam_num_doneqs = 1 + mp_ncpus / 6;
965 	else if (cam_num_doneqs > MAXCPU)
966 		cam_num_doneqs = MAXCPU;
967 	for (i = 0; i < cam_num_doneqs; i++) {
968 		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
969 		    MTX_DEF);
970 		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
971 		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
972 		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
973 		if (error != 0) {
974 			cam_num_doneqs = i;
975 			break;
976 		}
977 	}
978 	if (cam_num_doneqs < 1) {
979 		printf("xpt_init: Cannot init completion queues "
980 		       "- failing attach\n");
981 		return (ENOMEM);
982 	}
983 	/*
984 	 * Register a callback for when interrupts are enabled.
985 	 */
986 	xsoftc.xpt_config_hook =
987 	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
988 					      M_CAMXPT, M_NOWAIT | M_ZERO);
989 	if (xsoftc.xpt_config_hook == NULL) {
990 		printf("xpt_init: Cannot malloc config hook "
991 		       "- failing attach\n");
992 		return (ENOMEM);
993 	}
994 	xsoftc.xpt_config_hook->ich_func = xpt_config;
995 	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
996 		free (xsoftc.xpt_config_hook, M_CAMXPT);
997 		printf("xpt_init: config_intrhook_establish failed "
998 		       "- failing attach\n");
999 	}
1000 
1001 	return (0);
1002 }
1003 
1004 static cam_status
1005 xptregister(struct cam_periph *periph, void *arg)
1006 {
1007 	struct cam_sim *xpt_sim;
1008 
1009 	if (periph == NULL) {
1010 		printf("xptregister: periph was NULL!!\n");
1011 		return(CAM_REQ_CMP_ERR);
1012 	}
1013 
1014 	xpt_sim = (struct cam_sim *)arg;
1015 	xpt_sim->softc = periph;
1016 	xpt_periph = periph;
1017 	periph->softc = NULL;
1018 
1019 	return(CAM_REQ_CMP);
1020 }
1021 
1022 int32_t
1023 xpt_add_periph(struct cam_periph *periph)
1024 {
1025 	struct cam_ed *device;
1026 	int32_t	 status;
1027 
1028 	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
1029 	device = periph->path->device;
1030 	status = CAM_REQ_CMP;
1031 	if (device != NULL) {
1032 		mtx_lock(&device->target->bus->eb_mtx);
1033 		device->generation++;
1034 		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
1035 		mtx_unlock(&device->target->bus->eb_mtx);
1036 		atomic_add_32(&xsoftc.xpt_generation, 1);
1037 	}
1038 
1039 	return (status);
1040 }
1041 
1042 void
1043 xpt_remove_periph(struct cam_periph *periph)
1044 {
1045 	struct cam_ed *device;
1046 
1047 	device = periph->path->device;
1048 	if (device != NULL) {
1049 		mtx_lock(&device->target->bus->eb_mtx);
1050 		device->generation++;
1051 		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1052 		mtx_unlock(&device->target->bus->eb_mtx);
1053 		atomic_add_32(&xsoftc.xpt_generation, 1);
1054 	}
1055 }
1056 
1057 
1058 void
1059 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1060 {
1061 	struct	cam_path *path = periph->path;
1062 	struct  xpt_proto *proto;
1063 
1064 	cam_periph_assert(periph, MA_OWNED);
1065 	periph->flags |= CAM_PERIPH_ANNOUNCED;
1066 
1067 	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1068 	       periph->periph_name, periph->unit_number,
1069 	       path->bus->sim->sim_name,
1070 	       path->bus->sim->unit_number,
1071 	       path->bus->sim->bus_id,
1072 	       path->bus->path_id,
1073 	       path->target->target_id,
1074 	       (uintmax_t)path->device->lun_id);
1075 	printf("%s%d: ", periph->periph_name, periph->unit_number);
1076 	proto = xpt_proto_find(path->device->protocol);
1077 	if (proto)
1078 		proto->ops->announce(path->device);
1079 	else
1080 		printf("%s%d: Unknown protocol device %d\n",
1081 		    periph->periph_name, periph->unit_number,
1082 		    path->device->protocol);
1083 	if (path->device->serial_num_len > 0) {
1084 		/* Don't wrap the screen  - print only the first 60 chars */
1085 		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1086 		       periph->unit_number, path->device->serial_num);
1087 	}
1088 	/* Announce transport details. */
1089 	path->bus->xport->ops->announce(periph);
1090 	/* Announce command queueing. */
1091 	if (path->device->inq_flags & SID_CmdQue
1092 	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1093 		printf("%s%d: Command Queueing enabled\n",
1094 		       periph->periph_name, periph->unit_number);
1095 	}
1096 	/* Announce caller's details if they've passed in. */
1097 	if (announce_string != NULL)
1098 		printf("%s%d: %s\n", periph->periph_name,
1099 		       periph->unit_number, announce_string);
1100 }
1101 
1102 void
1103 xpt_announce_periph_sbuf(struct cam_periph *periph, struct sbuf *sb,
1104     char *announce_string)
1105 {
1106 	struct	cam_path *path = periph->path;
1107 	struct  xpt_proto *proto;
1108 
1109 	cam_periph_assert(periph, MA_OWNED);
1110 	periph->flags |= CAM_PERIPH_ANNOUNCED;
1111 
1112 	/* Fall back to the non-sbuf method if necessary */
1113 	if (xsoftc.announce_nosbuf != 0) {
1114 		xpt_announce_periph(periph, announce_string);
1115 		return;
1116 	}
1117 	proto = xpt_proto_find(path->device->protocol);
1118 	if (((proto != NULL) && (proto->ops->announce_sbuf == NULL)) ||
1119 	    (path->bus->xport->ops->announce_sbuf == NULL)) {
1120 		xpt_announce_periph(periph, announce_string);
1121 		return;
1122 	}
1123 
1124 	sbuf_printf(sb, "%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1125 	    periph->periph_name, periph->unit_number,
1126 	    path->bus->sim->sim_name,
1127 	    path->bus->sim->unit_number,
1128 	    path->bus->sim->bus_id,
1129 	    path->bus->path_id,
1130 	    path->target->target_id,
1131 	    (uintmax_t)path->device->lun_id);
1132 	sbuf_printf(sb, "%s%d: ", periph->periph_name, periph->unit_number);
1133 
1134 	if (proto)
1135 		proto->ops->announce_sbuf(path->device, sb);
1136 	else
1137 		sbuf_printf(sb, "%s%d: Unknown protocol device %d\n",
1138 		    periph->periph_name, periph->unit_number,
1139 		    path->device->protocol);
1140 	if (path->device->serial_num_len > 0) {
1141 		/* Don't wrap the screen  - print only the first 60 chars */
1142 		sbuf_printf(sb, "%s%d: Serial Number %.60s\n",
1143 		    periph->periph_name, periph->unit_number,
1144 		    path->device->serial_num);
1145 	}
1146 	/* Announce transport details. */
1147 	path->bus->xport->ops->announce_sbuf(periph, sb);
1148 	/* Announce command queueing. */
1149 	if (path->device->inq_flags & SID_CmdQue
1150 	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1151 		sbuf_printf(sb, "%s%d: Command Queueing enabled\n",
1152 		    periph->periph_name, periph->unit_number);
1153 	}
1154 	/* Announce caller's details if they've passed in. */
1155 	if (announce_string != NULL)
1156 		sbuf_printf(sb, "%s%d: %s\n", periph->periph_name,
1157 		    periph->unit_number, announce_string);
1158 }
1159 
1160 void
1161 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1162 {
1163 	if (quirks != 0) {
1164 		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1165 		    periph->unit_number, quirks, bit_string);
1166 	}
1167 }
1168 
1169 void
1170 xpt_announce_quirks_sbuf(struct cam_periph *periph, struct sbuf *sb,
1171 			 int quirks, char *bit_string)
1172 {
1173 	if (xsoftc.announce_nosbuf != 0) {
1174 		xpt_announce_quirks(periph, quirks, bit_string);
1175 		return;
1176 	}
1177 
1178 	if (quirks != 0) {
1179 		sbuf_printf(sb, "%s%d: quirks=0x%b\n", periph->periph_name,
1180 		    periph->unit_number, quirks, bit_string);
1181 	}
1182 }
1183 
1184 void
1185 xpt_denounce_periph(struct cam_periph *periph)
1186 {
1187 	struct	cam_path *path = periph->path;
1188 	struct  xpt_proto *proto;
1189 
1190 	cam_periph_assert(periph, MA_OWNED);
1191 	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1192 	       periph->periph_name, periph->unit_number,
1193 	       path->bus->sim->sim_name,
1194 	       path->bus->sim->unit_number,
1195 	       path->bus->sim->bus_id,
1196 	       path->bus->path_id,
1197 	       path->target->target_id,
1198 	       (uintmax_t)path->device->lun_id);
1199 	printf("%s%d: ", periph->periph_name, periph->unit_number);
1200 	proto = xpt_proto_find(path->device->protocol);
1201 	if (proto)
1202 		proto->ops->denounce(path->device);
1203 	else
1204 		printf("%s%d: Unknown protocol device %d\n",
1205 		    periph->periph_name, periph->unit_number,
1206 		    path->device->protocol);
1207 	if (path->device->serial_num_len > 0)
1208 		printf(" s/n %.60s", path->device->serial_num);
1209 	printf(" detached\n");
1210 }
1211 
1212 void
1213 xpt_denounce_periph_sbuf(struct cam_periph *periph, struct sbuf *sb)
1214 {
1215 	struct cam_path *path = periph->path;
1216 	struct xpt_proto *proto;
1217 
1218 	cam_periph_assert(periph, MA_OWNED);
1219 
1220 	/* Fall back to the non-sbuf method if necessary */
1221 	if (xsoftc.announce_nosbuf != 0) {
1222 		xpt_denounce_periph(periph);
1223 		return;
1224 	}
1225 	proto = xpt_proto_find(path->device->protocol);
1226 	if ((proto != NULL) && (proto->ops->denounce_sbuf == NULL)) {
1227 		xpt_denounce_periph(periph);
1228 		return;
1229 	}
1230 
1231 	sbuf_printf(sb, "%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1232 	    periph->periph_name, periph->unit_number,
1233 	    path->bus->sim->sim_name,
1234 	    path->bus->sim->unit_number,
1235 	    path->bus->sim->bus_id,
1236 	    path->bus->path_id,
1237 	    path->target->target_id,
1238 	    (uintmax_t)path->device->lun_id);
1239 	sbuf_printf(sb, "%s%d: ", periph->periph_name, periph->unit_number);
1240 
1241 	if (proto)
1242 		proto->ops->denounce_sbuf(path->device, sb);
1243 	else
1244 		sbuf_printf(sb, "%s%d: Unknown protocol device %d\n",
1245 		    periph->periph_name, periph->unit_number,
1246 		    path->device->protocol);
1247 	if (path->device->serial_num_len > 0)
1248 		sbuf_printf(sb, " s/n %.60s", path->device->serial_num);
1249 	sbuf_printf(sb, " detached\n");
1250 }
1251 
1252 int
1253 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1254 {
1255 	int ret = -1, l, o;
1256 	struct ccb_dev_advinfo cdai;
1257 	struct scsi_vpd_id_descriptor *idd;
1258 
1259 	xpt_path_assert(path, MA_OWNED);
1260 
1261 	memset(&cdai, 0, sizeof(cdai));
1262 	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1263 	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1264 	cdai.flags = CDAI_FLAG_NONE;
1265 	cdai.bufsiz = len;
1266 
1267 	if (!strcmp(attr, "GEOM::ident"))
1268 		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1269 	else if (!strcmp(attr, "GEOM::physpath"))
1270 		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1271 	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1272 		 strcmp(attr, "GEOM::lunname") == 0) {
1273 		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1274 		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1275 	} else
1276 		goto out;
1277 
1278 	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1279 	if (cdai.buf == NULL) {
1280 		ret = ENOMEM;
1281 		goto out;
1282 	}
1283 	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1284 	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1285 		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1286 	if (cdai.provsiz == 0)
1287 		goto out;
1288 	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1289 		if (strcmp(attr, "GEOM::lunid") == 0) {
1290 			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1291 			    cdai.provsiz, scsi_devid_is_lun_naa);
1292 			if (idd == NULL)
1293 				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1294 				    cdai.provsiz, scsi_devid_is_lun_eui64);
1295 			if (idd == NULL)
1296 				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1297 				    cdai.provsiz, scsi_devid_is_lun_uuid);
1298 			if (idd == NULL)
1299 				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1300 				    cdai.provsiz, scsi_devid_is_lun_md5);
1301 		} else
1302 			idd = NULL;
1303 		if (idd == NULL)
1304 			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1305 			    cdai.provsiz, scsi_devid_is_lun_t10);
1306 		if (idd == NULL)
1307 			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1308 			    cdai.provsiz, scsi_devid_is_lun_name);
1309 		if (idd == NULL)
1310 			goto out;
1311 		ret = 0;
1312 		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1313 			if (idd->length < len) {
1314 				for (l = 0; l < idd->length; l++)
1315 					buf[l] = idd->identifier[l] ?
1316 					    idd->identifier[l] : ' ';
1317 				buf[l] = 0;
1318 			} else
1319 				ret = EFAULT;
1320 		} else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1321 			l = strnlen(idd->identifier, idd->length);
1322 			if (l < len) {
1323 				bcopy(idd->identifier, buf, l);
1324 				buf[l] = 0;
1325 			} else
1326 				ret = EFAULT;
1327 		} else if ((idd->id_type & SVPD_ID_TYPE_MASK) == SVPD_ID_TYPE_UUID
1328 		    && idd->identifier[0] == 0x10) {
1329 			if ((idd->length - 2) * 2 + 4 < len) {
1330 				for (l = 2, o = 0; l < idd->length; l++) {
1331 					if (l == 6 || l == 8 || l == 10 || l == 12)
1332 					    o += sprintf(buf + o, "-");
1333 					o += sprintf(buf + o, "%02x",
1334 					    idd->identifier[l]);
1335 				}
1336 			} else
1337 				ret = EFAULT;
1338 		} else {
1339 			if (idd->length * 2 < len) {
1340 				for (l = 0; l < idd->length; l++)
1341 					sprintf(buf + l * 2, "%02x",
1342 					    idd->identifier[l]);
1343 			} else
1344 				ret = EFAULT;
1345 		}
1346 	} else {
1347 		ret = 0;
1348 		if (strlcpy(buf, cdai.buf, len) >= len)
1349 			ret = EFAULT;
1350 	}
1351 
1352 out:
1353 	if (cdai.buf != NULL)
1354 		free(cdai.buf, M_CAMXPT);
1355 	return ret;
1356 }
1357 
1358 static dev_match_ret
1359 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1360 	    struct cam_eb *bus)
1361 {
1362 	dev_match_ret retval;
1363 	u_int i;
1364 
1365 	retval = DM_RET_NONE;
1366 
1367 	/*
1368 	 * If we aren't given something to match against, that's an error.
1369 	 */
1370 	if (bus == NULL)
1371 		return(DM_RET_ERROR);
1372 
1373 	/*
1374 	 * If there are no match entries, then this bus matches no
1375 	 * matter what.
1376 	 */
1377 	if ((patterns == NULL) || (num_patterns == 0))
1378 		return(DM_RET_DESCEND | DM_RET_COPY);
1379 
1380 	for (i = 0; i < num_patterns; i++) {
1381 		struct bus_match_pattern *cur_pattern;
1382 
1383 		/*
1384 		 * If the pattern in question isn't for a bus node, we
1385 		 * aren't interested.  However, we do indicate to the
1386 		 * calling routine that we should continue descending the
1387 		 * tree, since the user wants to match against lower-level
1388 		 * EDT elements.
1389 		 */
1390 		if (patterns[i].type != DEV_MATCH_BUS) {
1391 			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1392 				retval |= DM_RET_DESCEND;
1393 			continue;
1394 		}
1395 
1396 		cur_pattern = &patterns[i].pattern.bus_pattern;
1397 
1398 		/*
1399 		 * If they want to match any bus node, we give them any
1400 		 * device node.
1401 		 */
1402 		if (cur_pattern->flags == BUS_MATCH_ANY) {
1403 			/* set the copy flag */
1404 			retval |= DM_RET_COPY;
1405 
1406 			/*
1407 			 * If we've already decided on an action, go ahead
1408 			 * and return.
1409 			 */
1410 			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1411 				return(retval);
1412 		}
1413 
1414 		/*
1415 		 * Not sure why someone would do this...
1416 		 */
1417 		if (cur_pattern->flags == BUS_MATCH_NONE)
1418 			continue;
1419 
1420 		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1421 		 && (cur_pattern->path_id != bus->path_id))
1422 			continue;
1423 
1424 		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1425 		 && (cur_pattern->bus_id != bus->sim->bus_id))
1426 			continue;
1427 
1428 		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1429 		 && (cur_pattern->unit_number != bus->sim->unit_number))
1430 			continue;
1431 
1432 		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1433 		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1434 			     DEV_IDLEN) != 0))
1435 			continue;
1436 
1437 		/*
1438 		 * If we get to this point, the user definitely wants
1439 		 * information on this bus.  So tell the caller to copy the
1440 		 * data out.
1441 		 */
1442 		retval |= DM_RET_COPY;
1443 
1444 		/*
1445 		 * If the return action has been set to descend, then we
1446 		 * know that we've already seen a non-bus matching
1447 		 * expression, therefore we need to further descend the tree.
1448 		 * This won't change by continuing around the loop, so we
1449 		 * go ahead and return.  If we haven't seen a non-bus
1450 		 * matching expression, we keep going around the loop until
1451 		 * we exhaust the matching expressions.  We'll set the stop
1452 		 * flag once we fall out of the loop.
1453 		 */
1454 		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1455 			return(retval);
1456 	}
1457 
1458 	/*
1459 	 * If the return action hasn't been set to descend yet, that means
1460 	 * we haven't seen anything other than bus matching patterns.  So
1461 	 * tell the caller to stop descending the tree -- the user doesn't
1462 	 * want to match against lower level tree elements.
1463 	 */
1464 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1465 		retval |= DM_RET_STOP;
1466 
1467 	return(retval);
1468 }
1469 
1470 static dev_match_ret
1471 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1472 	       struct cam_ed *device)
1473 {
1474 	dev_match_ret retval;
1475 	u_int i;
1476 
1477 	retval = DM_RET_NONE;
1478 
1479 	/*
1480 	 * If we aren't given something to match against, that's an error.
1481 	 */
1482 	if (device == NULL)
1483 		return(DM_RET_ERROR);
1484 
1485 	/*
1486 	 * If there are no match entries, then this device matches no
1487 	 * matter what.
1488 	 */
1489 	if ((patterns == NULL) || (num_patterns == 0))
1490 		return(DM_RET_DESCEND | DM_RET_COPY);
1491 
1492 	for (i = 0; i < num_patterns; i++) {
1493 		struct device_match_pattern *cur_pattern;
1494 		struct scsi_vpd_device_id *device_id_page;
1495 
1496 		/*
1497 		 * If the pattern in question isn't for a device node, we
1498 		 * aren't interested.
1499 		 */
1500 		if (patterns[i].type != DEV_MATCH_DEVICE) {
1501 			if ((patterns[i].type == DEV_MATCH_PERIPH)
1502 			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1503 				retval |= DM_RET_DESCEND;
1504 			continue;
1505 		}
1506 
1507 		cur_pattern = &patterns[i].pattern.device_pattern;
1508 
1509 		/* Error out if mutually exclusive options are specified. */
1510 		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1511 		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1512 			return(DM_RET_ERROR);
1513 
1514 		/*
1515 		 * If they want to match any device node, we give them any
1516 		 * device node.
1517 		 */
1518 		if (cur_pattern->flags == DEV_MATCH_ANY)
1519 			goto copy_dev_node;
1520 
1521 		/*
1522 		 * Not sure why someone would do this...
1523 		 */
1524 		if (cur_pattern->flags == DEV_MATCH_NONE)
1525 			continue;
1526 
1527 		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1528 		 && (cur_pattern->path_id != device->target->bus->path_id))
1529 			continue;
1530 
1531 		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1532 		 && (cur_pattern->target_id != device->target->target_id))
1533 			continue;
1534 
1535 		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1536 		 && (cur_pattern->target_lun != device->lun_id))
1537 			continue;
1538 
1539 		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1540 		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1541 				    (caddr_t)&cur_pattern->data.inq_pat,
1542 				    1, sizeof(cur_pattern->data.inq_pat),
1543 				    scsi_static_inquiry_match) == NULL))
1544 			continue;
1545 
1546 		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1547 		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1548 		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1549 		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1550 				      device->device_id_len
1551 				    - SVPD_DEVICE_ID_HDR_LEN,
1552 				      cur_pattern->data.devid_pat.id,
1553 				      cur_pattern->data.devid_pat.id_len) != 0))
1554 			continue;
1555 
1556 copy_dev_node:
1557 		/*
1558 		 * If we get to this point, the user definitely wants
1559 		 * information on this device.  So tell the caller to copy
1560 		 * the data out.
1561 		 */
1562 		retval |= DM_RET_COPY;
1563 
1564 		/*
1565 		 * If the return action has been set to descend, then we
1566 		 * know that we've already seen a peripheral matching
1567 		 * expression, therefore we need to further descend the tree.
1568 		 * This won't change by continuing around the loop, so we
1569 		 * go ahead and return.  If we haven't seen a peripheral
1570 		 * matching expression, we keep going around the loop until
1571 		 * we exhaust the matching expressions.  We'll set the stop
1572 		 * flag once we fall out of the loop.
1573 		 */
1574 		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1575 			return(retval);
1576 	}
1577 
1578 	/*
1579 	 * If the return action hasn't been set to descend yet, that means
1580 	 * we haven't seen any peripheral matching patterns.  So tell the
1581 	 * caller to stop descending the tree -- the user doesn't want to
1582 	 * match against lower level tree elements.
1583 	 */
1584 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1585 		retval |= DM_RET_STOP;
1586 
1587 	return(retval);
1588 }
1589 
1590 /*
1591  * Match a single peripheral against any number of match patterns.
1592  */
1593 static dev_match_ret
1594 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1595 	       struct cam_periph *periph)
1596 {
1597 	dev_match_ret retval;
1598 	u_int i;
1599 
1600 	/*
1601 	 * If we aren't given something to match against, that's an error.
1602 	 */
1603 	if (periph == NULL)
1604 		return(DM_RET_ERROR);
1605 
1606 	/*
1607 	 * If there are no match entries, then this peripheral matches no
1608 	 * matter what.
1609 	 */
1610 	if ((patterns == NULL) || (num_patterns == 0))
1611 		return(DM_RET_STOP | DM_RET_COPY);
1612 
1613 	/*
1614 	 * There aren't any nodes below a peripheral node, so there's no
1615 	 * reason to descend the tree any further.
1616 	 */
1617 	retval = DM_RET_STOP;
1618 
1619 	for (i = 0; i < num_patterns; i++) {
1620 		struct periph_match_pattern *cur_pattern;
1621 
1622 		/*
1623 		 * If the pattern in question isn't for a peripheral, we
1624 		 * aren't interested.
1625 		 */
1626 		if (patterns[i].type != DEV_MATCH_PERIPH)
1627 			continue;
1628 
1629 		cur_pattern = &patterns[i].pattern.periph_pattern;
1630 
1631 		/*
1632 		 * If they want to match on anything, then we will do so.
1633 		 */
1634 		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1635 			/* set the copy flag */
1636 			retval |= DM_RET_COPY;
1637 
1638 			/*
1639 			 * We've already set the return action to stop,
1640 			 * since there are no nodes below peripherals in
1641 			 * the tree.
1642 			 */
1643 			return(retval);
1644 		}
1645 
1646 		/*
1647 		 * Not sure why someone would do this...
1648 		 */
1649 		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1650 			continue;
1651 
1652 		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1653 		 && (cur_pattern->path_id != periph->path->bus->path_id))
1654 			continue;
1655 
1656 		/*
1657 		 * For the target and lun id's, we have to make sure the
1658 		 * target and lun pointers aren't NULL.  The xpt peripheral
1659 		 * has a wildcard target and device.
1660 		 */
1661 		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1662 		 && ((periph->path->target == NULL)
1663 		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1664 			continue;
1665 
1666 		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1667 		 && ((periph->path->device == NULL)
1668 		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1669 			continue;
1670 
1671 		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1672 		 && (cur_pattern->unit_number != periph->unit_number))
1673 			continue;
1674 
1675 		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1676 		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1677 			     DEV_IDLEN) != 0))
1678 			continue;
1679 
1680 		/*
1681 		 * If we get to this point, the user definitely wants
1682 		 * information on this peripheral.  So tell the caller to
1683 		 * copy the data out.
1684 		 */
1685 		retval |= DM_RET_COPY;
1686 
1687 		/*
1688 		 * The return action has already been set to stop, since
1689 		 * peripherals don't have any nodes below them in the EDT.
1690 		 */
1691 		return(retval);
1692 	}
1693 
1694 	/*
1695 	 * If we get to this point, the peripheral that was passed in
1696 	 * doesn't match any of the patterns.
1697 	 */
1698 	return(retval);
1699 }
1700 
1701 static int
1702 xptedtbusfunc(struct cam_eb *bus, void *arg)
1703 {
1704 	struct ccb_dev_match *cdm;
1705 	struct cam_et *target;
1706 	dev_match_ret retval;
1707 
1708 	cdm = (struct ccb_dev_match *)arg;
1709 
1710 	/*
1711 	 * If our position is for something deeper in the tree, that means
1712 	 * that we've already seen this node.  So, we keep going down.
1713 	 */
1714 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1715 	 && (cdm->pos.cookie.bus == bus)
1716 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1717 	 && (cdm->pos.cookie.target != NULL))
1718 		retval = DM_RET_DESCEND;
1719 	else
1720 		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1721 
1722 	/*
1723 	 * If we got an error, bail out of the search.
1724 	 */
1725 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1726 		cdm->status = CAM_DEV_MATCH_ERROR;
1727 		return(0);
1728 	}
1729 
1730 	/*
1731 	 * If the copy flag is set, copy this bus out.
1732 	 */
1733 	if (retval & DM_RET_COPY) {
1734 		int spaceleft, j;
1735 
1736 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1737 			sizeof(struct dev_match_result));
1738 
1739 		/*
1740 		 * If we don't have enough space to put in another
1741 		 * match result, save our position and tell the
1742 		 * user there are more devices to check.
1743 		 */
1744 		if (spaceleft < sizeof(struct dev_match_result)) {
1745 			bzero(&cdm->pos, sizeof(cdm->pos));
1746 			cdm->pos.position_type =
1747 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1748 
1749 			cdm->pos.cookie.bus = bus;
1750 			cdm->pos.generations[CAM_BUS_GENERATION]=
1751 				xsoftc.bus_generation;
1752 			cdm->status = CAM_DEV_MATCH_MORE;
1753 			return(0);
1754 		}
1755 		j = cdm->num_matches;
1756 		cdm->num_matches++;
1757 		cdm->matches[j].type = DEV_MATCH_BUS;
1758 		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1759 		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1760 		cdm->matches[j].result.bus_result.unit_number =
1761 			bus->sim->unit_number;
1762 		strlcpy(cdm->matches[j].result.bus_result.dev_name,
1763 			bus->sim->sim_name,
1764 			sizeof(cdm->matches[j].result.bus_result.dev_name));
1765 	}
1766 
1767 	/*
1768 	 * If the user is only interested in buses, there's no
1769 	 * reason to descend to the next level in the tree.
1770 	 */
1771 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1772 		return(1);
1773 
1774 	/*
1775 	 * If there is a target generation recorded, check it to
1776 	 * make sure the target list hasn't changed.
1777 	 */
1778 	mtx_lock(&bus->eb_mtx);
1779 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1780 	 && (cdm->pos.cookie.bus == bus)
1781 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1782 	 && (cdm->pos.cookie.target != NULL)) {
1783 		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1784 		    bus->generation)) {
1785 			mtx_unlock(&bus->eb_mtx);
1786 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1787 			return (0);
1788 		}
1789 		target = (struct cam_et *)cdm->pos.cookie.target;
1790 		target->refcount++;
1791 	} else
1792 		target = NULL;
1793 	mtx_unlock(&bus->eb_mtx);
1794 
1795 	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1796 }
1797 
1798 static int
1799 xptedttargetfunc(struct cam_et *target, void *arg)
1800 {
1801 	struct ccb_dev_match *cdm;
1802 	struct cam_eb *bus;
1803 	struct cam_ed *device;
1804 
1805 	cdm = (struct ccb_dev_match *)arg;
1806 	bus = target->bus;
1807 
1808 	/*
1809 	 * If there is a device list generation recorded, check it to
1810 	 * make sure the device list hasn't changed.
1811 	 */
1812 	mtx_lock(&bus->eb_mtx);
1813 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1814 	 && (cdm->pos.cookie.bus == bus)
1815 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1816 	 && (cdm->pos.cookie.target == target)
1817 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1818 	 && (cdm->pos.cookie.device != NULL)) {
1819 		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1820 		    target->generation) {
1821 			mtx_unlock(&bus->eb_mtx);
1822 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1823 			return(0);
1824 		}
1825 		device = (struct cam_ed *)cdm->pos.cookie.device;
1826 		device->refcount++;
1827 	} else
1828 		device = NULL;
1829 	mtx_unlock(&bus->eb_mtx);
1830 
1831 	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1832 }
1833 
1834 static int
1835 xptedtdevicefunc(struct cam_ed *device, void *arg)
1836 {
1837 	struct cam_eb *bus;
1838 	struct cam_periph *periph;
1839 	struct ccb_dev_match *cdm;
1840 	dev_match_ret retval;
1841 
1842 	cdm = (struct ccb_dev_match *)arg;
1843 	bus = device->target->bus;
1844 
1845 	/*
1846 	 * If our position is for something deeper in the tree, that means
1847 	 * that we've already seen this node.  So, we keep going down.
1848 	 */
1849 	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1850 	 && (cdm->pos.cookie.device == device)
1851 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1852 	 && (cdm->pos.cookie.periph != NULL))
1853 		retval = DM_RET_DESCEND;
1854 	else
1855 		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1856 					device);
1857 
1858 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1859 		cdm->status = CAM_DEV_MATCH_ERROR;
1860 		return(0);
1861 	}
1862 
1863 	/*
1864 	 * If the copy flag is set, copy this device out.
1865 	 */
1866 	if (retval & DM_RET_COPY) {
1867 		int spaceleft, j;
1868 
1869 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1870 			sizeof(struct dev_match_result));
1871 
1872 		/*
1873 		 * If we don't have enough space to put in another
1874 		 * match result, save our position and tell the
1875 		 * user there are more devices to check.
1876 		 */
1877 		if (spaceleft < sizeof(struct dev_match_result)) {
1878 			bzero(&cdm->pos, sizeof(cdm->pos));
1879 			cdm->pos.position_type =
1880 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1881 				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1882 
1883 			cdm->pos.cookie.bus = device->target->bus;
1884 			cdm->pos.generations[CAM_BUS_GENERATION]=
1885 				xsoftc.bus_generation;
1886 			cdm->pos.cookie.target = device->target;
1887 			cdm->pos.generations[CAM_TARGET_GENERATION] =
1888 				device->target->bus->generation;
1889 			cdm->pos.cookie.device = device;
1890 			cdm->pos.generations[CAM_DEV_GENERATION] =
1891 				device->target->generation;
1892 			cdm->status = CAM_DEV_MATCH_MORE;
1893 			return(0);
1894 		}
1895 		j = cdm->num_matches;
1896 		cdm->num_matches++;
1897 		cdm->matches[j].type = DEV_MATCH_DEVICE;
1898 		cdm->matches[j].result.device_result.path_id =
1899 			device->target->bus->path_id;
1900 		cdm->matches[j].result.device_result.target_id =
1901 			device->target->target_id;
1902 		cdm->matches[j].result.device_result.target_lun =
1903 			device->lun_id;
1904 		cdm->matches[j].result.device_result.protocol =
1905 			device->protocol;
1906 		bcopy(&device->inq_data,
1907 		      &cdm->matches[j].result.device_result.inq_data,
1908 		      sizeof(struct scsi_inquiry_data));
1909 		bcopy(&device->ident_data,
1910 		      &cdm->matches[j].result.device_result.ident_data,
1911 		      sizeof(struct ata_params));
1912 
1913 		/* Let the user know whether this device is unconfigured */
1914 		if (device->flags & CAM_DEV_UNCONFIGURED)
1915 			cdm->matches[j].result.device_result.flags =
1916 				DEV_RESULT_UNCONFIGURED;
1917 		else
1918 			cdm->matches[j].result.device_result.flags =
1919 				DEV_RESULT_NOFLAG;
1920 	}
1921 
1922 	/*
1923 	 * If the user isn't interested in peripherals, don't descend
1924 	 * the tree any further.
1925 	 */
1926 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1927 		return(1);
1928 
1929 	/*
1930 	 * If there is a peripheral list generation recorded, make sure
1931 	 * it hasn't changed.
1932 	 */
1933 	xpt_lock_buses();
1934 	mtx_lock(&bus->eb_mtx);
1935 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1936 	 && (cdm->pos.cookie.bus == bus)
1937 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1938 	 && (cdm->pos.cookie.target == device->target)
1939 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1940 	 && (cdm->pos.cookie.device == device)
1941 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1942 	 && (cdm->pos.cookie.periph != NULL)) {
1943 		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1944 		    device->generation) {
1945 			mtx_unlock(&bus->eb_mtx);
1946 			xpt_unlock_buses();
1947 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1948 			return(0);
1949 		}
1950 		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1951 		periph->refcount++;
1952 	} else
1953 		periph = NULL;
1954 	mtx_unlock(&bus->eb_mtx);
1955 	xpt_unlock_buses();
1956 
1957 	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1958 }
1959 
1960 static int
1961 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1962 {
1963 	struct ccb_dev_match *cdm;
1964 	dev_match_ret retval;
1965 
1966 	cdm = (struct ccb_dev_match *)arg;
1967 
1968 	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1969 
1970 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1971 		cdm->status = CAM_DEV_MATCH_ERROR;
1972 		return(0);
1973 	}
1974 
1975 	/*
1976 	 * If the copy flag is set, copy this peripheral out.
1977 	 */
1978 	if (retval & DM_RET_COPY) {
1979 		int spaceleft, j;
1980 		size_t l;
1981 
1982 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1983 			sizeof(struct dev_match_result));
1984 
1985 		/*
1986 		 * If we don't have enough space to put in another
1987 		 * match result, save our position and tell the
1988 		 * user there are more devices to check.
1989 		 */
1990 		if (spaceleft < sizeof(struct dev_match_result)) {
1991 			bzero(&cdm->pos, sizeof(cdm->pos));
1992 			cdm->pos.position_type =
1993 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1994 				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1995 				CAM_DEV_POS_PERIPH;
1996 
1997 			cdm->pos.cookie.bus = periph->path->bus;
1998 			cdm->pos.generations[CAM_BUS_GENERATION]=
1999 				xsoftc.bus_generation;
2000 			cdm->pos.cookie.target = periph->path->target;
2001 			cdm->pos.generations[CAM_TARGET_GENERATION] =
2002 				periph->path->bus->generation;
2003 			cdm->pos.cookie.device = periph->path->device;
2004 			cdm->pos.generations[CAM_DEV_GENERATION] =
2005 				periph->path->target->generation;
2006 			cdm->pos.cookie.periph = periph;
2007 			cdm->pos.generations[CAM_PERIPH_GENERATION] =
2008 				periph->path->device->generation;
2009 			cdm->status = CAM_DEV_MATCH_MORE;
2010 			return(0);
2011 		}
2012 
2013 		j = cdm->num_matches;
2014 		cdm->num_matches++;
2015 		cdm->matches[j].type = DEV_MATCH_PERIPH;
2016 		cdm->matches[j].result.periph_result.path_id =
2017 			periph->path->bus->path_id;
2018 		cdm->matches[j].result.periph_result.target_id =
2019 			periph->path->target->target_id;
2020 		cdm->matches[j].result.periph_result.target_lun =
2021 			periph->path->device->lun_id;
2022 		cdm->matches[j].result.periph_result.unit_number =
2023 			periph->unit_number;
2024 		l = sizeof(cdm->matches[j].result.periph_result.periph_name);
2025 		strlcpy(cdm->matches[j].result.periph_result.periph_name,
2026 			periph->periph_name, l);
2027 	}
2028 
2029 	return(1);
2030 }
2031 
2032 static int
2033 xptedtmatch(struct ccb_dev_match *cdm)
2034 {
2035 	struct cam_eb *bus;
2036 	int ret;
2037 
2038 	cdm->num_matches = 0;
2039 
2040 	/*
2041 	 * Check the bus list generation.  If it has changed, the user
2042 	 * needs to reset everything and start over.
2043 	 */
2044 	xpt_lock_buses();
2045 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
2046 	 && (cdm->pos.cookie.bus != NULL)) {
2047 		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
2048 		    xsoftc.bus_generation) {
2049 			xpt_unlock_buses();
2050 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2051 			return(0);
2052 		}
2053 		bus = (struct cam_eb *)cdm->pos.cookie.bus;
2054 		bus->refcount++;
2055 	} else
2056 		bus = NULL;
2057 	xpt_unlock_buses();
2058 
2059 	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
2060 
2061 	/*
2062 	 * If we get back 0, that means that we had to stop before fully
2063 	 * traversing the EDT.  It also means that one of the subroutines
2064 	 * has set the status field to the proper value.  If we get back 1,
2065 	 * we've fully traversed the EDT and copied out any matching entries.
2066 	 */
2067 	if (ret == 1)
2068 		cdm->status = CAM_DEV_MATCH_LAST;
2069 
2070 	return(ret);
2071 }
2072 
2073 static int
2074 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
2075 {
2076 	struct cam_periph *periph;
2077 	struct ccb_dev_match *cdm;
2078 
2079 	cdm = (struct ccb_dev_match *)arg;
2080 
2081 	xpt_lock_buses();
2082 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2083 	 && (cdm->pos.cookie.pdrv == pdrv)
2084 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2085 	 && (cdm->pos.cookie.periph != NULL)) {
2086 		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
2087 		    (*pdrv)->generation) {
2088 			xpt_unlock_buses();
2089 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2090 			return(0);
2091 		}
2092 		periph = (struct cam_periph *)cdm->pos.cookie.periph;
2093 		periph->refcount++;
2094 	} else
2095 		periph = NULL;
2096 	xpt_unlock_buses();
2097 
2098 	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
2099 }
2100 
2101 static int
2102 xptplistperiphfunc(struct cam_periph *periph, void *arg)
2103 {
2104 	struct ccb_dev_match *cdm;
2105 	dev_match_ret retval;
2106 
2107 	cdm = (struct ccb_dev_match *)arg;
2108 
2109 	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
2110 
2111 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
2112 		cdm->status = CAM_DEV_MATCH_ERROR;
2113 		return(0);
2114 	}
2115 
2116 	/*
2117 	 * If the copy flag is set, copy this peripheral out.
2118 	 */
2119 	if (retval & DM_RET_COPY) {
2120 		int spaceleft, j;
2121 		size_t l;
2122 
2123 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
2124 			sizeof(struct dev_match_result));
2125 
2126 		/*
2127 		 * If we don't have enough space to put in another
2128 		 * match result, save our position and tell the
2129 		 * user there are more devices to check.
2130 		 */
2131 		if (spaceleft < sizeof(struct dev_match_result)) {
2132 			struct periph_driver **pdrv;
2133 
2134 			pdrv = NULL;
2135 			bzero(&cdm->pos, sizeof(cdm->pos));
2136 			cdm->pos.position_type =
2137 				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
2138 				CAM_DEV_POS_PERIPH;
2139 
2140 			/*
2141 			 * This may look a bit non-sensical, but it is
2142 			 * actually quite logical.  There are very few
2143 			 * peripheral drivers, and bloating every peripheral
2144 			 * structure with a pointer back to its parent
2145 			 * peripheral driver linker set entry would cost
2146 			 * more in the long run than doing this quick lookup.
2147 			 */
2148 			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
2149 				if (strcmp((*pdrv)->driver_name,
2150 				    periph->periph_name) == 0)
2151 					break;
2152 			}
2153 
2154 			if (*pdrv == NULL) {
2155 				cdm->status = CAM_DEV_MATCH_ERROR;
2156 				return(0);
2157 			}
2158 
2159 			cdm->pos.cookie.pdrv = pdrv;
2160 			/*
2161 			 * The periph generation slot does double duty, as
2162 			 * does the periph pointer slot.  They are used for
2163 			 * both edt and pdrv lookups and positioning.
2164 			 */
2165 			cdm->pos.cookie.periph = periph;
2166 			cdm->pos.generations[CAM_PERIPH_GENERATION] =
2167 				(*pdrv)->generation;
2168 			cdm->status = CAM_DEV_MATCH_MORE;
2169 			return(0);
2170 		}
2171 
2172 		j = cdm->num_matches;
2173 		cdm->num_matches++;
2174 		cdm->matches[j].type = DEV_MATCH_PERIPH;
2175 		cdm->matches[j].result.periph_result.path_id =
2176 			periph->path->bus->path_id;
2177 
2178 		/*
2179 		 * The transport layer peripheral doesn't have a target or
2180 		 * lun.
2181 		 */
2182 		if (periph->path->target)
2183 			cdm->matches[j].result.periph_result.target_id =
2184 				periph->path->target->target_id;
2185 		else
2186 			cdm->matches[j].result.periph_result.target_id =
2187 				CAM_TARGET_WILDCARD;
2188 
2189 		if (periph->path->device)
2190 			cdm->matches[j].result.periph_result.target_lun =
2191 				periph->path->device->lun_id;
2192 		else
2193 			cdm->matches[j].result.periph_result.target_lun =
2194 				CAM_LUN_WILDCARD;
2195 
2196 		cdm->matches[j].result.periph_result.unit_number =
2197 			periph->unit_number;
2198 		l = sizeof(cdm->matches[j].result.periph_result.periph_name);
2199 		strlcpy(cdm->matches[j].result.periph_result.periph_name,
2200 			periph->periph_name, l);
2201 	}
2202 
2203 	return(1);
2204 }
2205 
2206 static int
2207 xptperiphlistmatch(struct ccb_dev_match *cdm)
2208 {
2209 	int ret;
2210 
2211 	cdm->num_matches = 0;
2212 
2213 	/*
2214 	 * At this point in the edt traversal function, we check the bus
2215 	 * list generation to make sure that no buses have been added or
2216 	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2217 	 * For the peripheral driver list traversal function, however, we
2218 	 * don't have to worry about new peripheral driver types coming or
2219 	 * going; they're in a linker set, and therefore can't change
2220 	 * without a recompile.
2221 	 */
2222 
2223 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2224 	 && (cdm->pos.cookie.pdrv != NULL))
2225 		ret = xptpdrvtraverse(
2226 				(struct periph_driver **)cdm->pos.cookie.pdrv,
2227 				xptplistpdrvfunc, cdm);
2228 	else
2229 		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2230 
2231 	/*
2232 	 * If we get back 0, that means that we had to stop before fully
2233 	 * traversing the peripheral driver tree.  It also means that one of
2234 	 * the subroutines has set the status field to the proper value.  If
2235 	 * we get back 1, we've fully traversed the EDT and copied out any
2236 	 * matching entries.
2237 	 */
2238 	if (ret == 1)
2239 		cdm->status = CAM_DEV_MATCH_LAST;
2240 
2241 	return(ret);
2242 }
2243 
2244 static int
2245 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2246 {
2247 	struct cam_eb *bus, *next_bus;
2248 	int retval;
2249 
2250 	retval = 1;
2251 	if (start_bus)
2252 		bus = start_bus;
2253 	else {
2254 		xpt_lock_buses();
2255 		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2256 		if (bus == NULL) {
2257 			xpt_unlock_buses();
2258 			return (retval);
2259 		}
2260 		bus->refcount++;
2261 		xpt_unlock_buses();
2262 	}
2263 	for (; bus != NULL; bus = next_bus) {
2264 		retval = tr_func(bus, arg);
2265 		if (retval == 0) {
2266 			xpt_release_bus(bus);
2267 			break;
2268 		}
2269 		xpt_lock_buses();
2270 		next_bus = TAILQ_NEXT(bus, links);
2271 		if (next_bus)
2272 			next_bus->refcount++;
2273 		xpt_unlock_buses();
2274 		xpt_release_bus(bus);
2275 	}
2276 	return(retval);
2277 }
2278 
2279 static int
2280 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2281 		  xpt_targetfunc_t *tr_func, void *arg)
2282 {
2283 	struct cam_et *target, *next_target;
2284 	int retval;
2285 
2286 	retval = 1;
2287 	if (start_target)
2288 		target = start_target;
2289 	else {
2290 		mtx_lock(&bus->eb_mtx);
2291 		target = TAILQ_FIRST(&bus->et_entries);
2292 		if (target == NULL) {
2293 			mtx_unlock(&bus->eb_mtx);
2294 			return (retval);
2295 		}
2296 		target->refcount++;
2297 		mtx_unlock(&bus->eb_mtx);
2298 	}
2299 	for (; target != NULL; target = next_target) {
2300 		retval = tr_func(target, arg);
2301 		if (retval == 0) {
2302 			xpt_release_target(target);
2303 			break;
2304 		}
2305 		mtx_lock(&bus->eb_mtx);
2306 		next_target = TAILQ_NEXT(target, links);
2307 		if (next_target)
2308 			next_target->refcount++;
2309 		mtx_unlock(&bus->eb_mtx);
2310 		xpt_release_target(target);
2311 	}
2312 	return(retval);
2313 }
2314 
2315 static int
2316 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2317 		  xpt_devicefunc_t *tr_func, void *arg)
2318 {
2319 	struct cam_eb *bus;
2320 	struct cam_ed *device, *next_device;
2321 	int retval;
2322 
2323 	retval = 1;
2324 	bus = target->bus;
2325 	if (start_device)
2326 		device = start_device;
2327 	else {
2328 		mtx_lock(&bus->eb_mtx);
2329 		device = TAILQ_FIRST(&target->ed_entries);
2330 		if (device == NULL) {
2331 			mtx_unlock(&bus->eb_mtx);
2332 			return (retval);
2333 		}
2334 		device->refcount++;
2335 		mtx_unlock(&bus->eb_mtx);
2336 	}
2337 	for (; device != NULL; device = next_device) {
2338 		mtx_lock(&device->device_mtx);
2339 		retval = tr_func(device, arg);
2340 		mtx_unlock(&device->device_mtx);
2341 		if (retval == 0) {
2342 			xpt_release_device(device);
2343 			break;
2344 		}
2345 		mtx_lock(&bus->eb_mtx);
2346 		next_device = TAILQ_NEXT(device, links);
2347 		if (next_device)
2348 			next_device->refcount++;
2349 		mtx_unlock(&bus->eb_mtx);
2350 		xpt_release_device(device);
2351 	}
2352 	return(retval);
2353 }
2354 
2355 static int
2356 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2357 		  xpt_periphfunc_t *tr_func, void *arg)
2358 {
2359 	struct cam_eb *bus;
2360 	struct cam_periph *periph, *next_periph;
2361 	int retval;
2362 
2363 	retval = 1;
2364 
2365 	bus = device->target->bus;
2366 	if (start_periph)
2367 		periph = start_periph;
2368 	else {
2369 		xpt_lock_buses();
2370 		mtx_lock(&bus->eb_mtx);
2371 		periph = SLIST_FIRST(&device->periphs);
2372 		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2373 			periph = SLIST_NEXT(periph, periph_links);
2374 		if (periph == NULL) {
2375 			mtx_unlock(&bus->eb_mtx);
2376 			xpt_unlock_buses();
2377 			return (retval);
2378 		}
2379 		periph->refcount++;
2380 		mtx_unlock(&bus->eb_mtx);
2381 		xpt_unlock_buses();
2382 	}
2383 	for (; periph != NULL; periph = next_periph) {
2384 		retval = tr_func(periph, arg);
2385 		if (retval == 0) {
2386 			cam_periph_release_locked(periph);
2387 			break;
2388 		}
2389 		xpt_lock_buses();
2390 		mtx_lock(&bus->eb_mtx);
2391 		next_periph = SLIST_NEXT(periph, periph_links);
2392 		while (next_periph != NULL &&
2393 		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2394 			next_periph = SLIST_NEXT(next_periph, periph_links);
2395 		if (next_periph)
2396 			next_periph->refcount++;
2397 		mtx_unlock(&bus->eb_mtx);
2398 		xpt_unlock_buses();
2399 		cam_periph_release_locked(periph);
2400 	}
2401 	return(retval);
2402 }
2403 
2404 static int
2405 xptpdrvtraverse(struct periph_driver **start_pdrv,
2406 		xpt_pdrvfunc_t *tr_func, void *arg)
2407 {
2408 	struct periph_driver **pdrv;
2409 	int retval;
2410 
2411 	retval = 1;
2412 
2413 	/*
2414 	 * We don't traverse the peripheral driver list like we do the
2415 	 * other lists, because it is a linker set, and therefore cannot be
2416 	 * changed during runtime.  If the peripheral driver list is ever
2417 	 * re-done to be something other than a linker set (i.e. it can
2418 	 * change while the system is running), the list traversal should
2419 	 * be modified to work like the other traversal functions.
2420 	 */
2421 	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2422 	     *pdrv != NULL; pdrv++) {
2423 		retval = tr_func(pdrv, arg);
2424 
2425 		if (retval == 0)
2426 			return(retval);
2427 	}
2428 
2429 	return(retval);
2430 }
2431 
2432 static int
2433 xptpdperiphtraverse(struct periph_driver **pdrv,
2434 		    struct cam_periph *start_periph,
2435 		    xpt_periphfunc_t *tr_func, void *arg)
2436 {
2437 	struct cam_periph *periph, *next_periph;
2438 	int retval;
2439 
2440 	retval = 1;
2441 
2442 	if (start_periph)
2443 		periph = start_periph;
2444 	else {
2445 		xpt_lock_buses();
2446 		periph = TAILQ_FIRST(&(*pdrv)->units);
2447 		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2448 			periph = TAILQ_NEXT(periph, unit_links);
2449 		if (periph == NULL) {
2450 			xpt_unlock_buses();
2451 			return (retval);
2452 		}
2453 		periph->refcount++;
2454 		xpt_unlock_buses();
2455 	}
2456 	for (; periph != NULL; periph = next_periph) {
2457 		cam_periph_lock(periph);
2458 		retval = tr_func(periph, arg);
2459 		cam_periph_unlock(periph);
2460 		if (retval == 0) {
2461 			cam_periph_release(periph);
2462 			break;
2463 		}
2464 		xpt_lock_buses();
2465 		next_periph = TAILQ_NEXT(periph, unit_links);
2466 		while (next_periph != NULL &&
2467 		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2468 			next_periph = TAILQ_NEXT(next_periph, unit_links);
2469 		if (next_periph)
2470 			next_periph->refcount++;
2471 		xpt_unlock_buses();
2472 		cam_periph_release(periph);
2473 	}
2474 	return(retval);
2475 }
2476 
2477 static int
2478 xptdefbusfunc(struct cam_eb *bus, void *arg)
2479 {
2480 	struct xpt_traverse_config *tr_config;
2481 
2482 	tr_config = (struct xpt_traverse_config *)arg;
2483 
2484 	if (tr_config->depth == XPT_DEPTH_BUS) {
2485 		xpt_busfunc_t *tr_func;
2486 
2487 		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2488 
2489 		return(tr_func(bus, tr_config->tr_arg));
2490 	} else
2491 		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2492 }
2493 
2494 static int
2495 xptdeftargetfunc(struct cam_et *target, void *arg)
2496 {
2497 	struct xpt_traverse_config *tr_config;
2498 
2499 	tr_config = (struct xpt_traverse_config *)arg;
2500 
2501 	if (tr_config->depth == XPT_DEPTH_TARGET) {
2502 		xpt_targetfunc_t *tr_func;
2503 
2504 		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2505 
2506 		return(tr_func(target, tr_config->tr_arg));
2507 	} else
2508 		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2509 }
2510 
2511 static int
2512 xptdefdevicefunc(struct cam_ed *device, void *arg)
2513 {
2514 	struct xpt_traverse_config *tr_config;
2515 
2516 	tr_config = (struct xpt_traverse_config *)arg;
2517 
2518 	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2519 		xpt_devicefunc_t *tr_func;
2520 
2521 		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2522 
2523 		return(tr_func(device, tr_config->tr_arg));
2524 	} else
2525 		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2526 }
2527 
2528 static int
2529 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2530 {
2531 	struct xpt_traverse_config *tr_config;
2532 	xpt_periphfunc_t *tr_func;
2533 
2534 	tr_config = (struct xpt_traverse_config *)arg;
2535 
2536 	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2537 
2538 	/*
2539 	 * Unlike the other default functions, we don't check for depth
2540 	 * here.  The peripheral driver level is the last level in the EDT,
2541 	 * so if we're here, we should execute the function in question.
2542 	 */
2543 	return(tr_func(periph, tr_config->tr_arg));
2544 }
2545 
2546 /*
2547  * Execute the given function for every bus in the EDT.
2548  */
2549 static int
2550 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2551 {
2552 	struct xpt_traverse_config tr_config;
2553 
2554 	tr_config.depth = XPT_DEPTH_BUS;
2555 	tr_config.tr_func = tr_func;
2556 	tr_config.tr_arg = arg;
2557 
2558 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2559 }
2560 
2561 /*
2562  * Execute the given function for every device in the EDT.
2563  */
2564 static int
2565 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2566 {
2567 	struct xpt_traverse_config tr_config;
2568 
2569 	tr_config.depth = XPT_DEPTH_DEVICE;
2570 	tr_config.tr_func = tr_func;
2571 	tr_config.tr_arg = arg;
2572 
2573 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2574 }
2575 
2576 static int
2577 xptsetasyncfunc(struct cam_ed *device, void *arg)
2578 {
2579 	struct cam_path path;
2580 	struct ccb_getdev cgd;
2581 	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2582 
2583 	/*
2584 	 * Don't report unconfigured devices (Wildcard devs,
2585 	 * devices only for target mode, device instances
2586 	 * that have been invalidated but are waiting for
2587 	 * their last reference count to be released).
2588 	 */
2589 	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2590 		return (1);
2591 
2592 	xpt_compile_path(&path,
2593 			 NULL,
2594 			 device->target->bus->path_id,
2595 			 device->target->target_id,
2596 			 device->lun_id);
2597 	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2598 	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2599 	xpt_action((union ccb *)&cgd);
2600 	csa->callback(csa->callback_arg,
2601 			    AC_FOUND_DEVICE,
2602 			    &path, &cgd);
2603 	xpt_release_path(&path);
2604 
2605 	return(1);
2606 }
2607 
2608 static int
2609 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2610 {
2611 	struct cam_path path;
2612 	struct ccb_pathinq cpi;
2613 	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2614 
2615 	xpt_compile_path(&path, /*periph*/NULL,
2616 			 bus->path_id,
2617 			 CAM_TARGET_WILDCARD,
2618 			 CAM_LUN_WILDCARD);
2619 	xpt_path_lock(&path);
2620 	xpt_path_inq(&cpi, &path);
2621 	csa->callback(csa->callback_arg,
2622 			    AC_PATH_REGISTERED,
2623 			    &path, &cpi);
2624 	xpt_path_unlock(&path);
2625 	xpt_release_path(&path);
2626 
2627 	return(1);
2628 }
2629 
2630 void
2631 xpt_action(union ccb *start_ccb)
2632 {
2633 
2634 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE,
2635 	    ("xpt_action: func %#x %s\n", start_ccb->ccb_h.func_code,
2636 		xpt_action_name(start_ccb->ccb_h.func_code)));
2637 
2638 	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2639 	(*(start_ccb->ccb_h.path->bus->xport->ops->action))(start_ccb);
2640 }
2641 
2642 void
2643 xpt_action_default(union ccb *start_ccb)
2644 {
2645 	struct cam_path *path;
2646 	struct cam_sim *sim;
2647 	struct mtx *mtx;
2648 
2649 	path = start_ccb->ccb_h.path;
2650 	CAM_DEBUG(path, CAM_DEBUG_TRACE,
2651 	    ("xpt_action_default: func %#x %s\n", start_ccb->ccb_h.func_code,
2652 		xpt_action_name(start_ccb->ccb_h.func_code)));
2653 
2654 	switch (start_ccb->ccb_h.func_code) {
2655 	case XPT_SCSI_IO:
2656 	{
2657 		struct cam_ed *device;
2658 
2659 		/*
2660 		 * For the sake of compatibility with SCSI-1
2661 		 * devices that may not understand the identify
2662 		 * message, we include lun information in the
2663 		 * second byte of all commands.  SCSI-1 specifies
2664 		 * that luns are a 3 bit value and reserves only 3
2665 		 * bits for lun information in the CDB.  Later
2666 		 * revisions of the SCSI spec allow for more than 8
2667 		 * luns, but have deprecated lun information in the
2668 		 * CDB.  So, if the lun won't fit, we must omit.
2669 		 *
2670 		 * Also be aware that during initial probing for devices,
2671 		 * the inquiry information is unknown but initialized to 0.
2672 		 * This means that this code will be exercised while probing
2673 		 * devices with an ANSI revision greater than 2.
2674 		 */
2675 		device = path->device;
2676 		if (device->protocol_version <= SCSI_REV_2
2677 		 && start_ccb->ccb_h.target_lun < 8
2678 		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2679 
2680 			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2681 			    start_ccb->ccb_h.target_lun << 5;
2682 		}
2683 		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2684 	}
2685 	/* FALLTHROUGH */
2686 	case XPT_TARGET_IO:
2687 	case XPT_CONT_TARGET_IO:
2688 		start_ccb->csio.sense_resid = 0;
2689 		start_ccb->csio.resid = 0;
2690 		/* FALLTHROUGH */
2691 	case XPT_ATA_IO:
2692 		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2693 			start_ccb->ataio.resid = 0;
2694 		/* FALLTHROUGH */
2695 	case XPT_NVME_IO:
2696 		/* FALLTHROUGH */
2697 	case XPT_NVME_ADMIN:
2698 		/* FALLTHROUGH */
2699 	case XPT_MMC_IO:
2700 		/* XXX just like nmve_io? */
2701 	case XPT_RESET_DEV:
2702 	case XPT_ENG_EXEC:
2703 	case XPT_SMP_IO:
2704 	{
2705 		struct cam_devq *devq;
2706 
2707 		devq = path->bus->sim->devq;
2708 		mtx_lock(&devq->send_mtx);
2709 		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2710 		if (xpt_schedule_devq(devq, path->device) != 0)
2711 			xpt_run_devq(devq);
2712 		mtx_unlock(&devq->send_mtx);
2713 		break;
2714 	}
2715 	case XPT_CALC_GEOMETRY:
2716 		/* Filter out garbage */
2717 		if (start_ccb->ccg.block_size == 0
2718 		 || start_ccb->ccg.volume_size == 0) {
2719 			start_ccb->ccg.cylinders = 0;
2720 			start_ccb->ccg.heads = 0;
2721 			start_ccb->ccg.secs_per_track = 0;
2722 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2723 			break;
2724 		}
2725 #if defined(__sparc64__)
2726 		/*
2727 		 * For sparc64, we may need adjust the geometry of large
2728 		 * disks in order to fit the limitations of the 16-bit
2729 		 * fields of the VTOC8 disk label.
2730 		 */
2731 		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2732 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2733 			break;
2734 		}
2735 #endif
2736 		goto call_sim;
2737 	case XPT_ABORT:
2738 	{
2739 		union ccb* abort_ccb;
2740 
2741 		abort_ccb = start_ccb->cab.abort_ccb;
2742 		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2743 			struct cam_ed *device;
2744 			struct cam_devq *devq;
2745 
2746 			device = abort_ccb->ccb_h.path->device;
2747 			devq = device->sim->devq;
2748 
2749 			mtx_lock(&devq->send_mtx);
2750 			if (abort_ccb->ccb_h.pinfo.index > 0) {
2751 				cam_ccbq_remove_ccb(&device->ccbq, abort_ccb);
2752 				abort_ccb->ccb_h.status =
2753 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2754 				xpt_freeze_devq_device(device, 1);
2755 				mtx_unlock(&devq->send_mtx);
2756 				xpt_done(abort_ccb);
2757 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2758 				break;
2759 			}
2760 			mtx_unlock(&devq->send_mtx);
2761 
2762 			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2763 			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2764 				/*
2765 				 * We've caught this ccb en route to
2766 				 * the SIM.  Flag it for abort and the
2767 				 * SIM will do so just before starting
2768 				 * real work on the CCB.
2769 				 */
2770 				abort_ccb->ccb_h.status =
2771 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2772 				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2773 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2774 				break;
2775 			}
2776 		}
2777 		if (XPT_FC_IS_QUEUED(abort_ccb)
2778 		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2779 			/*
2780 			 * It's already completed but waiting
2781 			 * for our SWI to get to it.
2782 			 */
2783 			start_ccb->ccb_h.status = CAM_UA_ABORT;
2784 			break;
2785 		}
2786 		/*
2787 		 * If we weren't able to take care of the abort request
2788 		 * in the XPT, pass the request down to the SIM for processing.
2789 		 */
2790 	}
2791 	/* FALLTHROUGH */
2792 	case XPT_ACCEPT_TARGET_IO:
2793 	case XPT_EN_LUN:
2794 	case XPT_IMMED_NOTIFY:
2795 	case XPT_NOTIFY_ACK:
2796 	case XPT_RESET_BUS:
2797 	case XPT_IMMEDIATE_NOTIFY:
2798 	case XPT_NOTIFY_ACKNOWLEDGE:
2799 	case XPT_GET_SIM_KNOB_OLD:
2800 	case XPT_GET_SIM_KNOB:
2801 	case XPT_SET_SIM_KNOB:
2802 	case XPT_GET_TRAN_SETTINGS:
2803 	case XPT_SET_TRAN_SETTINGS:
2804 	case XPT_PATH_INQ:
2805 call_sim:
2806 		sim = path->bus->sim;
2807 		mtx = sim->mtx;
2808 		if (mtx && !mtx_owned(mtx))
2809 			mtx_lock(mtx);
2810 		else
2811 			mtx = NULL;
2812 
2813 		CAM_DEBUG(path, CAM_DEBUG_TRACE,
2814 		    ("Calling sim->sim_action(): func=%#x\n", start_ccb->ccb_h.func_code));
2815 		(*(sim->sim_action))(sim, start_ccb);
2816 		CAM_DEBUG(path, CAM_DEBUG_TRACE,
2817 		    ("sim->sim_action returned: status=%#x\n", start_ccb->ccb_h.status));
2818 		if (mtx)
2819 			mtx_unlock(mtx);
2820 		break;
2821 	case XPT_PATH_STATS:
2822 		start_ccb->cpis.last_reset = path->bus->last_reset;
2823 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2824 		break;
2825 	case XPT_GDEV_TYPE:
2826 	{
2827 		struct cam_ed *dev;
2828 
2829 		dev = path->device;
2830 		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2831 			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2832 		} else {
2833 			struct ccb_getdev *cgd;
2834 
2835 			cgd = &start_ccb->cgd;
2836 			cgd->protocol = dev->protocol;
2837 			cgd->inq_data = dev->inq_data;
2838 			cgd->ident_data = dev->ident_data;
2839 			cgd->inq_flags = dev->inq_flags;
2840 			cgd->ccb_h.status = CAM_REQ_CMP;
2841 			cgd->serial_num_len = dev->serial_num_len;
2842 			if ((dev->serial_num_len > 0)
2843 			 && (dev->serial_num != NULL))
2844 				bcopy(dev->serial_num, cgd->serial_num,
2845 				      dev->serial_num_len);
2846 		}
2847 		break;
2848 	}
2849 	case XPT_GDEV_STATS:
2850 	{
2851 		struct ccb_getdevstats *cgds = &start_ccb->cgds;
2852 		struct cam_ed *dev = path->device;
2853 		struct cam_eb *bus = path->bus;
2854 		struct cam_et *tar = path->target;
2855 		struct cam_devq *devq = bus->sim->devq;
2856 
2857 		mtx_lock(&devq->send_mtx);
2858 		cgds->dev_openings = dev->ccbq.dev_openings;
2859 		cgds->dev_active = dev->ccbq.dev_active;
2860 		cgds->allocated = dev->ccbq.allocated;
2861 		cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2862 		cgds->held = cgds->allocated - cgds->dev_active - cgds->queued;
2863 		cgds->last_reset = tar->last_reset;
2864 		cgds->maxtags = dev->maxtags;
2865 		cgds->mintags = dev->mintags;
2866 		if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2867 			cgds->last_reset = bus->last_reset;
2868 		mtx_unlock(&devq->send_mtx);
2869 		cgds->ccb_h.status = CAM_REQ_CMP;
2870 		break;
2871 	}
2872 	case XPT_GDEVLIST:
2873 	{
2874 		struct cam_periph	*nperiph;
2875 		struct periph_list	*periph_head;
2876 		struct ccb_getdevlist	*cgdl;
2877 		u_int			i;
2878 		struct cam_ed		*device;
2879 		int			found;
2880 
2881 
2882 		found = 0;
2883 
2884 		/*
2885 		 * Don't want anyone mucking with our data.
2886 		 */
2887 		device = path->device;
2888 		periph_head = &device->periphs;
2889 		cgdl = &start_ccb->cgdl;
2890 
2891 		/*
2892 		 * Check and see if the list has changed since the user
2893 		 * last requested a list member.  If so, tell them that the
2894 		 * list has changed, and therefore they need to start over
2895 		 * from the beginning.
2896 		 */
2897 		if ((cgdl->index != 0) &&
2898 		    (cgdl->generation != device->generation)) {
2899 			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2900 			break;
2901 		}
2902 
2903 		/*
2904 		 * Traverse the list of peripherals and attempt to find
2905 		 * the requested peripheral.
2906 		 */
2907 		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2908 		     (nperiph != NULL) && (i <= cgdl->index);
2909 		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2910 			if (i == cgdl->index) {
2911 				strlcpy(cgdl->periph_name,
2912 					nperiph->periph_name,
2913 					sizeof(cgdl->periph_name));
2914 				cgdl->unit_number = nperiph->unit_number;
2915 				found = 1;
2916 			}
2917 		}
2918 		if (found == 0) {
2919 			cgdl->status = CAM_GDEVLIST_ERROR;
2920 			break;
2921 		}
2922 
2923 		if (nperiph == NULL)
2924 			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2925 		else
2926 			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2927 
2928 		cgdl->index++;
2929 		cgdl->generation = device->generation;
2930 
2931 		cgdl->ccb_h.status = CAM_REQ_CMP;
2932 		break;
2933 	}
2934 	case XPT_DEV_MATCH:
2935 	{
2936 		dev_pos_type position_type;
2937 		struct ccb_dev_match *cdm;
2938 
2939 		cdm = &start_ccb->cdm;
2940 
2941 		/*
2942 		 * There are two ways of getting at information in the EDT.
2943 		 * The first way is via the primary EDT tree.  It starts
2944 		 * with a list of buses, then a list of targets on a bus,
2945 		 * then devices/luns on a target, and then peripherals on a
2946 		 * device/lun.  The "other" way is by the peripheral driver
2947 		 * lists.  The peripheral driver lists are organized by
2948 		 * peripheral driver.  (obviously)  So it makes sense to
2949 		 * use the peripheral driver list if the user is looking
2950 		 * for something like "da1", or all "da" devices.  If the
2951 		 * user is looking for something on a particular bus/target
2952 		 * or lun, it's generally better to go through the EDT tree.
2953 		 */
2954 
2955 		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2956 			position_type = cdm->pos.position_type;
2957 		else {
2958 			u_int i;
2959 
2960 			position_type = CAM_DEV_POS_NONE;
2961 
2962 			for (i = 0; i < cdm->num_patterns; i++) {
2963 				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2964 				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2965 					position_type = CAM_DEV_POS_EDT;
2966 					break;
2967 				}
2968 			}
2969 
2970 			if (cdm->num_patterns == 0)
2971 				position_type = CAM_DEV_POS_EDT;
2972 			else if (position_type == CAM_DEV_POS_NONE)
2973 				position_type = CAM_DEV_POS_PDRV;
2974 		}
2975 
2976 		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2977 		case CAM_DEV_POS_EDT:
2978 			xptedtmatch(cdm);
2979 			break;
2980 		case CAM_DEV_POS_PDRV:
2981 			xptperiphlistmatch(cdm);
2982 			break;
2983 		default:
2984 			cdm->status = CAM_DEV_MATCH_ERROR;
2985 			break;
2986 		}
2987 
2988 		if (cdm->status == CAM_DEV_MATCH_ERROR)
2989 			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2990 		else
2991 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2992 
2993 		break;
2994 	}
2995 	case XPT_SASYNC_CB:
2996 	{
2997 		struct ccb_setasync *csa;
2998 		struct async_node *cur_entry;
2999 		struct async_list *async_head;
3000 		u_int32_t added;
3001 
3002 		csa = &start_ccb->csa;
3003 		added = csa->event_enable;
3004 		async_head = &path->device->asyncs;
3005 
3006 		/*
3007 		 * If there is already an entry for us, simply
3008 		 * update it.
3009 		 */
3010 		cur_entry = SLIST_FIRST(async_head);
3011 		while (cur_entry != NULL) {
3012 			if ((cur_entry->callback_arg == csa->callback_arg)
3013 			 && (cur_entry->callback == csa->callback))
3014 				break;
3015 			cur_entry = SLIST_NEXT(cur_entry, links);
3016 		}
3017 
3018 		if (cur_entry != NULL) {
3019 		 	/*
3020 			 * If the request has no flags set,
3021 			 * remove the entry.
3022 			 */
3023 			added &= ~cur_entry->event_enable;
3024 			if (csa->event_enable == 0) {
3025 				SLIST_REMOVE(async_head, cur_entry,
3026 					     async_node, links);
3027 				xpt_release_device(path->device);
3028 				free(cur_entry, M_CAMXPT);
3029 			} else {
3030 				cur_entry->event_enable = csa->event_enable;
3031 			}
3032 			csa->event_enable = added;
3033 		} else {
3034 			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
3035 					   M_NOWAIT);
3036 			if (cur_entry == NULL) {
3037 				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
3038 				break;
3039 			}
3040 			cur_entry->event_enable = csa->event_enable;
3041 			cur_entry->event_lock = (path->bus->sim->mtx &&
3042 			    mtx_owned(path->bus->sim->mtx)) ? 1 : 0;
3043 			cur_entry->callback_arg = csa->callback_arg;
3044 			cur_entry->callback = csa->callback;
3045 			SLIST_INSERT_HEAD(async_head, cur_entry, links);
3046 			xpt_acquire_device(path->device);
3047 		}
3048 		start_ccb->ccb_h.status = CAM_REQ_CMP;
3049 		break;
3050 	}
3051 	case XPT_REL_SIMQ:
3052 	{
3053 		struct ccb_relsim *crs;
3054 		struct cam_ed *dev;
3055 
3056 		crs = &start_ccb->crs;
3057 		dev = path->device;
3058 		if (dev == NULL) {
3059 
3060 			crs->ccb_h.status = CAM_DEV_NOT_THERE;
3061 			break;
3062 		}
3063 
3064 		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
3065 
3066 			/* Don't ever go below one opening */
3067 			if (crs->openings > 0) {
3068 				xpt_dev_ccbq_resize(path, crs->openings);
3069 				if (bootverbose) {
3070 					xpt_print(path,
3071 					    "number of openings is now %d\n",
3072 					    crs->openings);
3073 				}
3074 			}
3075 		}
3076 
3077 		mtx_lock(&dev->sim->devq->send_mtx);
3078 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
3079 
3080 			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
3081 
3082 				/*
3083 				 * Just extend the old timeout and decrement
3084 				 * the freeze count so that a single timeout
3085 				 * is sufficient for releasing the queue.
3086 				 */
3087 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3088 				callout_stop(&dev->callout);
3089 			} else {
3090 
3091 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3092 			}
3093 
3094 			callout_reset_sbt(&dev->callout,
3095 			    SBT_1MS * crs->release_timeout, 0,
3096 			    xpt_release_devq_timeout, dev, 0);
3097 
3098 			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
3099 
3100 		}
3101 
3102 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
3103 
3104 			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
3105 				/*
3106 				 * Decrement the freeze count so that a single
3107 				 * completion is still sufficient to unfreeze
3108 				 * the queue.
3109 				 */
3110 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3111 			} else {
3112 
3113 				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
3114 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3115 			}
3116 		}
3117 
3118 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
3119 
3120 			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
3121 			 || (dev->ccbq.dev_active == 0)) {
3122 
3123 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3124 			} else {
3125 
3126 				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
3127 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3128 			}
3129 		}
3130 		mtx_unlock(&dev->sim->devq->send_mtx);
3131 
3132 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
3133 			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
3134 		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
3135 		start_ccb->ccb_h.status = CAM_REQ_CMP;
3136 		break;
3137 	}
3138 	case XPT_DEBUG: {
3139 		struct cam_path *oldpath;
3140 
3141 		/* Check that all request bits are supported. */
3142 		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
3143 			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
3144 			break;
3145 		}
3146 
3147 		cam_dflags = CAM_DEBUG_NONE;
3148 		if (cam_dpath != NULL) {
3149 			oldpath = cam_dpath;
3150 			cam_dpath = NULL;
3151 			xpt_free_path(oldpath);
3152 		}
3153 		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
3154 			if (xpt_create_path(&cam_dpath, NULL,
3155 					    start_ccb->ccb_h.path_id,
3156 					    start_ccb->ccb_h.target_id,
3157 					    start_ccb->ccb_h.target_lun) !=
3158 					    CAM_REQ_CMP) {
3159 				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3160 			} else {
3161 				cam_dflags = start_ccb->cdbg.flags;
3162 				start_ccb->ccb_h.status = CAM_REQ_CMP;
3163 				xpt_print(cam_dpath, "debugging flags now %x\n",
3164 				    cam_dflags);
3165 			}
3166 		} else
3167 			start_ccb->ccb_h.status = CAM_REQ_CMP;
3168 		break;
3169 	}
3170 	case XPT_NOOP:
3171 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
3172 			xpt_freeze_devq(path, 1);
3173 		start_ccb->ccb_h.status = CAM_REQ_CMP;
3174 		break;
3175 	case XPT_REPROBE_LUN:
3176 		xpt_async(AC_INQ_CHANGED, path, NULL);
3177 		start_ccb->ccb_h.status = CAM_REQ_CMP;
3178 		xpt_done(start_ccb);
3179 		break;
3180 	default:
3181 	case XPT_SDEV_TYPE:
3182 	case XPT_TERM_IO:
3183 	case XPT_ENG_INQ:
3184 		/* XXX Implement */
3185 		xpt_print(start_ccb->ccb_h.path,
3186 		    "%s: CCB type %#x %s not supported\n", __func__,
3187 		    start_ccb->ccb_h.func_code,
3188 		    xpt_action_name(start_ccb->ccb_h.func_code));
3189 		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3190 		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3191 			xpt_done(start_ccb);
3192 		}
3193 		break;
3194 	}
3195 	CAM_DEBUG(path, CAM_DEBUG_TRACE,
3196 	    ("xpt_action_default: func= %#x %s status %#x\n",
3197 		start_ccb->ccb_h.func_code,
3198  		xpt_action_name(start_ccb->ccb_h.func_code),
3199 		start_ccb->ccb_h.status));
3200 }
3201 
3202 uint32_t
3203 xpt_poll_setup(union ccb *start_ccb)
3204 {
3205 	u_int32_t timeout;
3206 	struct	  cam_sim *sim;
3207 	struct	  cam_devq *devq;
3208 	struct	  cam_ed *dev;
3209 	struct mtx *mtx;
3210 
3211 	timeout = start_ccb->ccb_h.timeout * 10;
3212 	sim = start_ccb->ccb_h.path->bus->sim;
3213 	devq = sim->devq;
3214 	mtx = sim->mtx;
3215 	dev = start_ccb->ccb_h.path->device;
3216 
3217 	/*
3218 	 * Steal an opening so that no other queued requests
3219 	 * can get it before us while we simulate interrupts.
3220 	 */
3221 	mtx_lock(&devq->send_mtx);
3222 	dev->ccbq.dev_openings--;
3223 	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3224 	    (--timeout > 0)) {
3225 		mtx_unlock(&devq->send_mtx);
3226 		DELAY(100);
3227 		if (mtx)
3228 			mtx_lock(mtx);
3229 		(*(sim->sim_poll))(sim);
3230 		if (mtx)
3231 			mtx_unlock(mtx);
3232 		camisr_runqueue();
3233 		mtx_lock(&devq->send_mtx);
3234 	}
3235 	dev->ccbq.dev_openings++;
3236 	mtx_unlock(&devq->send_mtx);
3237 
3238 	return (timeout);
3239 }
3240 
3241 void
3242 xpt_pollwait(union ccb *start_ccb, uint32_t timeout)
3243 {
3244 	struct cam_sim	*sim;
3245 	struct mtx	*mtx;
3246 
3247 	sim = start_ccb->ccb_h.path->bus->sim;
3248 	mtx = sim->mtx;
3249 
3250 	while (--timeout > 0) {
3251 		if (mtx)
3252 			mtx_lock(mtx);
3253 		(*(sim->sim_poll))(sim);
3254 		if (mtx)
3255 			mtx_unlock(mtx);
3256 		camisr_runqueue();
3257 		if ((start_ccb->ccb_h.status & CAM_STATUS_MASK)
3258 		    != CAM_REQ_INPROG)
3259 			break;
3260 		DELAY(100);
3261 	}
3262 
3263 	if (timeout == 0) {
3264 		/*
3265 		 * XXX Is it worth adding a sim_timeout entry
3266 		 * point so we can attempt recovery?  If
3267 		 * this is only used for dumps, I don't think
3268 		 * it is.
3269 		 */
3270 		start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3271 	}
3272 }
3273 
3274 void
3275 xpt_polled_action(union ccb *start_ccb)
3276 {
3277 	uint32_t	timeout;
3278 	struct cam_ed	*dev;
3279 
3280 	timeout = start_ccb->ccb_h.timeout * 10;
3281 	dev = start_ccb->ccb_h.path->device;
3282 
3283 	mtx_unlock(&dev->device_mtx);
3284 
3285 	timeout = xpt_poll_setup(start_ccb);
3286 	if (timeout > 0) {
3287 		xpt_action(start_ccb);
3288 		xpt_pollwait(start_ccb, timeout);
3289 	} else {
3290 		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3291 	}
3292 
3293 	mtx_lock(&dev->device_mtx);
3294 }
3295 
3296 /*
3297  * Schedule a peripheral driver to receive a ccb when its
3298  * target device has space for more transactions.
3299  */
3300 void
3301 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3302 {
3303 
3304 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3305 	cam_periph_assert(periph, MA_OWNED);
3306 	if (new_priority < periph->scheduled_priority) {
3307 		periph->scheduled_priority = new_priority;
3308 		xpt_run_allocq(periph, 0);
3309 	}
3310 }
3311 
3312 
3313 /*
3314  * Schedule a device to run on a given queue.
3315  * If the device was inserted as a new entry on the queue,
3316  * return 1 meaning the device queue should be run. If we
3317  * were already queued, implying someone else has already
3318  * started the queue, return 0 so the caller doesn't attempt
3319  * to run the queue.
3320  */
3321 static int
3322 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3323 		 u_int32_t new_priority)
3324 {
3325 	int retval;
3326 	u_int32_t old_priority;
3327 
3328 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3329 
3330 
3331 	old_priority = pinfo->priority;
3332 
3333 	/*
3334 	 * Are we already queued?
3335 	 */
3336 	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3337 		/* Simply reorder based on new priority */
3338 		if (new_priority < old_priority) {
3339 			camq_change_priority(queue, pinfo->index,
3340 					     new_priority);
3341 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3342 					("changed priority to %d\n",
3343 					 new_priority));
3344 			retval = 1;
3345 		} else
3346 			retval = 0;
3347 	} else {
3348 		/* New entry on the queue */
3349 		if (new_priority < old_priority)
3350 			pinfo->priority = new_priority;
3351 
3352 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3353 				("Inserting onto queue\n"));
3354 		pinfo->generation = ++queue->generation;
3355 		camq_insert(queue, pinfo);
3356 		retval = 1;
3357 	}
3358 	return (retval);
3359 }
3360 
3361 static void
3362 xpt_run_allocq_task(void *context, int pending)
3363 {
3364 	struct cam_periph *periph = context;
3365 
3366 	cam_periph_lock(periph);
3367 	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3368 	xpt_run_allocq(periph, 1);
3369 	cam_periph_unlock(periph);
3370 	cam_periph_release(periph);
3371 }
3372 
3373 static void
3374 xpt_run_allocq(struct cam_periph *periph, int sleep)
3375 {
3376 	struct cam_ed	*device;
3377 	union ccb	*ccb;
3378 	uint32_t	 prio;
3379 
3380 	cam_periph_assert(periph, MA_OWNED);
3381 	if (periph->periph_allocating)
3382 		return;
3383 	cam_periph_doacquire(periph);
3384 	periph->periph_allocating = 1;
3385 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3386 	device = periph->path->device;
3387 	ccb = NULL;
3388 restart:
3389 	while ((prio = min(periph->scheduled_priority,
3390 	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3391 	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3392 	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3393 
3394 		if (ccb == NULL &&
3395 		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3396 			if (sleep) {
3397 				ccb = xpt_get_ccb(periph);
3398 				goto restart;
3399 			}
3400 			if (periph->flags & CAM_PERIPH_RUN_TASK)
3401 				break;
3402 			cam_periph_doacquire(periph);
3403 			periph->flags |= CAM_PERIPH_RUN_TASK;
3404 			taskqueue_enqueue(xsoftc.xpt_taskq,
3405 			    &periph->periph_run_task);
3406 			break;
3407 		}
3408 		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3409 		if (prio == periph->immediate_priority) {
3410 			periph->immediate_priority = CAM_PRIORITY_NONE;
3411 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3412 					("waking cam_periph_getccb()\n"));
3413 			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3414 					  periph_links.sle);
3415 			wakeup(&periph->ccb_list);
3416 		} else {
3417 			periph->scheduled_priority = CAM_PRIORITY_NONE;
3418 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3419 					("calling periph_start()\n"));
3420 			periph->periph_start(periph, ccb);
3421 		}
3422 		ccb = NULL;
3423 	}
3424 	if (ccb != NULL)
3425 		xpt_release_ccb(ccb);
3426 	periph->periph_allocating = 0;
3427 	cam_periph_release_locked(periph);
3428 }
3429 
3430 static void
3431 xpt_run_devq(struct cam_devq *devq)
3432 {
3433 	struct mtx *mtx;
3434 
3435 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3436 
3437 	devq->send_queue.qfrozen_cnt++;
3438 	while ((devq->send_queue.entries > 0)
3439 	    && (devq->send_openings > 0)
3440 	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3441 		struct	cam_ed *device;
3442 		union ccb *work_ccb;
3443 		struct	cam_sim *sim;
3444 		struct xpt_proto *proto;
3445 
3446 		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3447 							   CAMQ_HEAD);
3448 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3449 				("running device %p\n", device));
3450 
3451 		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3452 		if (work_ccb == NULL) {
3453 			printf("device on run queue with no ccbs???\n");
3454 			continue;
3455 		}
3456 
3457 		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3458 
3459 			mtx_lock(&xsoftc.xpt_highpower_lock);
3460 		 	if (xsoftc.num_highpower <= 0) {
3461 				/*
3462 				 * We got a high power command, but we
3463 				 * don't have any available slots.  Freeze
3464 				 * the device queue until we have a slot
3465 				 * available.
3466 				 */
3467 				xpt_freeze_devq_device(device, 1);
3468 				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3469 						   highpowerq_entry);
3470 
3471 				mtx_unlock(&xsoftc.xpt_highpower_lock);
3472 				continue;
3473 			} else {
3474 				/*
3475 				 * Consume a high power slot while
3476 				 * this ccb runs.
3477 				 */
3478 				xsoftc.num_highpower--;
3479 			}
3480 			mtx_unlock(&xsoftc.xpt_highpower_lock);
3481 		}
3482 		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3483 		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3484 		devq->send_openings--;
3485 		devq->send_active++;
3486 		xpt_schedule_devq(devq, device);
3487 		mtx_unlock(&devq->send_mtx);
3488 
3489 		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3490 			/*
3491 			 * The client wants to freeze the queue
3492 			 * after this CCB is sent.
3493 			 */
3494 			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3495 		}
3496 
3497 		/* In Target mode, the peripheral driver knows best... */
3498 		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3499 			if ((device->inq_flags & SID_CmdQue) != 0
3500 			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3501 				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3502 			else
3503 				/*
3504 				 * Clear this in case of a retried CCB that
3505 				 * failed due to a rejected tag.
3506 				 */
3507 				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3508 		}
3509 
3510 		KASSERT(device == work_ccb->ccb_h.path->device,
3511 		    ("device (%p) / path->device (%p) mismatch",
3512 			device, work_ccb->ccb_h.path->device));
3513 		proto = xpt_proto_find(device->protocol);
3514 		if (proto && proto->ops->debug_out)
3515 			proto->ops->debug_out(work_ccb);
3516 
3517 		/*
3518 		 * Device queues can be shared among multiple SIM instances
3519 		 * that reside on different buses.  Use the SIM from the
3520 		 * queued device, rather than the one from the calling bus.
3521 		 */
3522 		sim = device->sim;
3523 		mtx = sim->mtx;
3524 		if (mtx && !mtx_owned(mtx))
3525 			mtx_lock(mtx);
3526 		else
3527 			mtx = NULL;
3528 		work_ccb->ccb_h.qos.periph_data = cam_iosched_now();
3529 		(*(sim->sim_action))(sim, work_ccb);
3530 		if (mtx)
3531 			mtx_unlock(mtx);
3532 		mtx_lock(&devq->send_mtx);
3533 	}
3534 	devq->send_queue.qfrozen_cnt--;
3535 }
3536 
3537 /*
3538  * This function merges stuff from the slave ccb into the master ccb, while
3539  * keeping important fields in the master ccb constant.
3540  */
3541 void
3542 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3543 {
3544 
3545 	/*
3546 	 * Pull fields that are valid for peripheral drivers to set
3547 	 * into the master CCB along with the CCB "payload".
3548 	 */
3549 	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3550 	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3551 	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3552 	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3553 	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3554 	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3555 }
3556 
3557 void
3558 xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3559 		    u_int32_t priority, u_int32_t flags)
3560 {
3561 
3562 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3563 	ccb_h->pinfo.priority = priority;
3564 	ccb_h->path = path;
3565 	ccb_h->path_id = path->bus->path_id;
3566 	if (path->target)
3567 		ccb_h->target_id = path->target->target_id;
3568 	else
3569 		ccb_h->target_id = CAM_TARGET_WILDCARD;
3570 	if (path->device) {
3571 		ccb_h->target_lun = path->device->lun_id;
3572 		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3573 	} else {
3574 		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3575 	}
3576 	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3577 	ccb_h->flags = flags;
3578 	ccb_h->xflags = 0;
3579 }
3580 
3581 void
3582 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3583 {
3584 	xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3585 }
3586 
3587 /* Path manipulation functions */
3588 cam_status
3589 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3590 		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3591 {
3592 	struct	   cam_path *path;
3593 	cam_status status;
3594 
3595 	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3596 
3597 	if (path == NULL) {
3598 		status = CAM_RESRC_UNAVAIL;
3599 		return(status);
3600 	}
3601 	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3602 	if (status != CAM_REQ_CMP) {
3603 		free(path, M_CAMPATH);
3604 		path = NULL;
3605 	}
3606 	*new_path_ptr = path;
3607 	return (status);
3608 }
3609 
3610 cam_status
3611 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3612 			 struct cam_periph *periph, path_id_t path_id,
3613 			 target_id_t target_id, lun_id_t lun_id)
3614 {
3615 
3616 	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3617 	    lun_id));
3618 }
3619 
3620 cam_status
3621 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3622 		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3623 {
3624 	struct	     cam_eb *bus;
3625 	struct	     cam_et *target;
3626 	struct	     cam_ed *device;
3627 	cam_status   status;
3628 
3629 	status = CAM_REQ_CMP;	/* Completed without error */
3630 	target = NULL;		/* Wildcarded */
3631 	device = NULL;		/* Wildcarded */
3632 
3633 	/*
3634 	 * We will potentially modify the EDT, so block interrupts
3635 	 * that may attempt to create cam paths.
3636 	 */
3637 	bus = xpt_find_bus(path_id);
3638 	if (bus == NULL) {
3639 		status = CAM_PATH_INVALID;
3640 	} else {
3641 		xpt_lock_buses();
3642 		mtx_lock(&bus->eb_mtx);
3643 		target = xpt_find_target(bus, target_id);
3644 		if (target == NULL) {
3645 			/* Create one */
3646 			struct cam_et *new_target;
3647 
3648 			new_target = xpt_alloc_target(bus, target_id);
3649 			if (new_target == NULL) {
3650 				status = CAM_RESRC_UNAVAIL;
3651 			} else {
3652 				target = new_target;
3653 			}
3654 		}
3655 		xpt_unlock_buses();
3656 		if (target != NULL) {
3657 			device = xpt_find_device(target, lun_id);
3658 			if (device == NULL) {
3659 				/* Create one */
3660 				struct cam_ed *new_device;
3661 
3662 				new_device =
3663 				    (*(bus->xport->ops->alloc_device))(bus,
3664 								       target,
3665 								       lun_id);
3666 				if (new_device == NULL) {
3667 					status = CAM_RESRC_UNAVAIL;
3668 				} else {
3669 					device = new_device;
3670 				}
3671 			}
3672 		}
3673 		mtx_unlock(&bus->eb_mtx);
3674 	}
3675 
3676 	/*
3677 	 * Only touch the user's data if we are successful.
3678 	 */
3679 	if (status == CAM_REQ_CMP) {
3680 		new_path->periph = perph;
3681 		new_path->bus = bus;
3682 		new_path->target = target;
3683 		new_path->device = device;
3684 		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3685 	} else {
3686 		if (device != NULL)
3687 			xpt_release_device(device);
3688 		if (target != NULL)
3689 			xpt_release_target(target);
3690 		if (bus != NULL)
3691 			xpt_release_bus(bus);
3692 	}
3693 	return (status);
3694 }
3695 
3696 cam_status
3697 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3698 {
3699 	struct	   cam_path *new_path;
3700 
3701 	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3702 	if (new_path == NULL)
3703 		return(CAM_RESRC_UNAVAIL);
3704 	xpt_copy_path(new_path, path);
3705 	*new_path_ptr = new_path;
3706 	return (CAM_REQ_CMP);
3707 }
3708 
3709 void
3710 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3711 {
3712 
3713 	*new_path = *path;
3714 	if (path->bus != NULL)
3715 		xpt_acquire_bus(path->bus);
3716 	if (path->target != NULL)
3717 		xpt_acquire_target(path->target);
3718 	if (path->device != NULL)
3719 		xpt_acquire_device(path->device);
3720 }
3721 
3722 void
3723 xpt_release_path(struct cam_path *path)
3724 {
3725 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3726 	if (path->device != NULL) {
3727 		xpt_release_device(path->device);
3728 		path->device = NULL;
3729 	}
3730 	if (path->target != NULL) {
3731 		xpt_release_target(path->target);
3732 		path->target = NULL;
3733 	}
3734 	if (path->bus != NULL) {
3735 		xpt_release_bus(path->bus);
3736 		path->bus = NULL;
3737 	}
3738 }
3739 
3740 void
3741 xpt_free_path(struct cam_path *path)
3742 {
3743 
3744 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3745 	xpt_release_path(path);
3746 	free(path, M_CAMPATH);
3747 }
3748 
3749 void
3750 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3751     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3752 {
3753 
3754 	xpt_lock_buses();
3755 	if (bus_ref) {
3756 		if (path->bus)
3757 			*bus_ref = path->bus->refcount;
3758 		else
3759 			*bus_ref = 0;
3760 	}
3761 	if (periph_ref) {
3762 		if (path->periph)
3763 			*periph_ref = path->periph->refcount;
3764 		else
3765 			*periph_ref = 0;
3766 	}
3767 	xpt_unlock_buses();
3768 	if (target_ref) {
3769 		if (path->target)
3770 			*target_ref = path->target->refcount;
3771 		else
3772 			*target_ref = 0;
3773 	}
3774 	if (device_ref) {
3775 		if (path->device)
3776 			*device_ref = path->device->refcount;
3777 		else
3778 			*device_ref = 0;
3779 	}
3780 }
3781 
3782 /*
3783  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3784  * in path1, 2 for match with wildcards in path2.
3785  */
3786 int
3787 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3788 {
3789 	int retval = 0;
3790 
3791 	if (path1->bus != path2->bus) {
3792 		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3793 			retval = 1;
3794 		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3795 			retval = 2;
3796 		else
3797 			return (-1);
3798 	}
3799 	if (path1->target != path2->target) {
3800 		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3801 			if (retval == 0)
3802 				retval = 1;
3803 		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3804 			retval = 2;
3805 		else
3806 			return (-1);
3807 	}
3808 	if (path1->device != path2->device) {
3809 		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3810 			if (retval == 0)
3811 				retval = 1;
3812 		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3813 			retval = 2;
3814 		else
3815 			return (-1);
3816 	}
3817 	return (retval);
3818 }
3819 
3820 int
3821 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3822 {
3823 	int retval = 0;
3824 
3825 	if (path->bus != dev->target->bus) {
3826 		if (path->bus->path_id == CAM_BUS_WILDCARD)
3827 			retval = 1;
3828 		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3829 			retval = 2;
3830 		else
3831 			return (-1);
3832 	}
3833 	if (path->target != dev->target) {
3834 		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3835 			if (retval == 0)
3836 				retval = 1;
3837 		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3838 			retval = 2;
3839 		else
3840 			return (-1);
3841 	}
3842 	if (path->device != dev) {
3843 		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3844 			if (retval == 0)
3845 				retval = 1;
3846 		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3847 			retval = 2;
3848 		else
3849 			return (-1);
3850 	}
3851 	return (retval);
3852 }
3853 
3854 void
3855 xpt_print_path(struct cam_path *path)
3856 {
3857 	struct sbuf sb;
3858 	char buffer[XPT_PRINT_LEN];
3859 
3860 	sbuf_new(&sb, buffer, XPT_PRINT_LEN, SBUF_FIXEDLEN);
3861 	xpt_path_sbuf(path, &sb);
3862 	sbuf_finish(&sb);
3863 	printf("%s", sbuf_data(&sb));
3864 	sbuf_delete(&sb);
3865 }
3866 
3867 void
3868 xpt_print_device(struct cam_ed *device)
3869 {
3870 
3871 	if (device == NULL)
3872 		printf("(nopath): ");
3873 	else {
3874 		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3875 		       device->sim->unit_number,
3876 		       device->sim->bus_id,
3877 		       device->target->target_id,
3878 		       (uintmax_t)device->lun_id);
3879 	}
3880 }
3881 
3882 void
3883 xpt_print(struct cam_path *path, const char *fmt, ...)
3884 {
3885 	va_list ap;
3886 	struct sbuf sb;
3887 	char buffer[XPT_PRINT_LEN];
3888 
3889 	sbuf_new(&sb, buffer, XPT_PRINT_LEN, SBUF_FIXEDLEN);
3890 
3891 	xpt_path_sbuf(path, &sb);
3892 	va_start(ap, fmt);
3893 	sbuf_vprintf(&sb, fmt, ap);
3894 	va_end(ap);
3895 
3896 	sbuf_finish(&sb);
3897 	printf("%s", sbuf_data(&sb));
3898 	sbuf_delete(&sb);
3899 }
3900 
3901 int
3902 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3903 {
3904 	struct sbuf sb;
3905 	int len;
3906 
3907 	sbuf_new(&sb, str, str_len, 0);
3908 	len = xpt_path_sbuf(path, &sb);
3909 	sbuf_finish(&sb);
3910 	return (len);
3911 }
3912 
3913 int
3914 xpt_path_sbuf(struct cam_path *path, struct sbuf *sb)
3915 {
3916 
3917 	if (path == NULL)
3918 		sbuf_printf(sb, "(nopath): ");
3919 	else {
3920 		if (path->periph != NULL)
3921 			sbuf_printf(sb, "(%s%d:", path->periph->periph_name,
3922 				    path->periph->unit_number);
3923 		else
3924 			sbuf_printf(sb, "(noperiph:");
3925 
3926 		if (path->bus != NULL)
3927 			sbuf_printf(sb, "%s%d:%d:", path->bus->sim->sim_name,
3928 				    path->bus->sim->unit_number,
3929 				    path->bus->sim->bus_id);
3930 		else
3931 			sbuf_printf(sb, "nobus:");
3932 
3933 		if (path->target != NULL)
3934 			sbuf_printf(sb, "%d:", path->target->target_id);
3935 		else
3936 			sbuf_printf(sb, "X:");
3937 
3938 		if (path->device != NULL)
3939 			sbuf_printf(sb, "%jx): ",
3940 			    (uintmax_t)path->device->lun_id);
3941 		else
3942 			sbuf_printf(sb, "X): ");
3943 	}
3944 
3945 	return(sbuf_len(sb));
3946 }
3947 
3948 path_id_t
3949 xpt_path_path_id(struct cam_path *path)
3950 {
3951 	return(path->bus->path_id);
3952 }
3953 
3954 target_id_t
3955 xpt_path_target_id(struct cam_path *path)
3956 {
3957 	if (path->target != NULL)
3958 		return (path->target->target_id);
3959 	else
3960 		return (CAM_TARGET_WILDCARD);
3961 }
3962 
3963 lun_id_t
3964 xpt_path_lun_id(struct cam_path *path)
3965 {
3966 	if (path->device != NULL)
3967 		return (path->device->lun_id);
3968 	else
3969 		return (CAM_LUN_WILDCARD);
3970 }
3971 
3972 struct cam_sim *
3973 xpt_path_sim(struct cam_path *path)
3974 {
3975 
3976 	return (path->bus->sim);
3977 }
3978 
3979 struct cam_periph*
3980 xpt_path_periph(struct cam_path *path)
3981 {
3982 
3983 	return (path->periph);
3984 }
3985 
3986 /*
3987  * Release a CAM control block for the caller.  Remit the cost of the structure
3988  * to the device referenced by the path.  If the this device had no 'credits'
3989  * and peripheral drivers have registered async callbacks for this notification
3990  * call them now.
3991  */
3992 void
3993 xpt_release_ccb(union ccb *free_ccb)
3994 {
3995 	struct	 cam_ed *device;
3996 	struct	 cam_periph *periph;
3997 
3998 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3999 	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
4000 	device = free_ccb->ccb_h.path->device;
4001 	periph = free_ccb->ccb_h.path->periph;
4002 
4003 	xpt_free_ccb(free_ccb);
4004 	periph->periph_allocated--;
4005 	cam_ccbq_release_opening(&device->ccbq);
4006 	xpt_run_allocq(periph, 0);
4007 }
4008 
4009 /* Functions accessed by SIM drivers */
4010 
4011 static struct xpt_xport_ops xport_default_ops = {
4012 	.alloc_device = xpt_alloc_device_default,
4013 	.action = xpt_action_default,
4014 	.async = xpt_dev_async_default,
4015 };
4016 static struct xpt_xport xport_default = {
4017 	.xport = XPORT_UNKNOWN,
4018 	.name = "unknown",
4019 	.ops = &xport_default_ops,
4020 };
4021 
4022 CAM_XPT_XPORT(xport_default);
4023 
4024 /*
4025  * A sim structure, listing the SIM entry points and instance
4026  * identification info is passed to xpt_bus_register to hook the SIM
4027  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
4028  * for this new bus and places it in the array of buses and assigns
4029  * it a path_id.  The path_id may be influenced by "hard wiring"
4030  * information specified by the user.  Once interrupt services are
4031  * available, the bus will be probed.
4032  */
4033 int32_t
4034 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
4035 {
4036 	struct cam_eb *new_bus;
4037 	struct cam_eb *old_bus;
4038 	struct ccb_pathinq cpi;
4039 	struct cam_path *path;
4040 	cam_status status;
4041 
4042 	sim->bus_id = bus;
4043 	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
4044 					  M_CAMXPT, M_NOWAIT|M_ZERO);
4045 	if (new_bus == NULL) {
4046 		/* Couldn't satisfy request */
4047 		return (CAM_RESRC_UNAVAIL);
4048 	}
4049 
4050 	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
4051 	TAILQ_INIT(&new_bus->et_entries);
4052 	cam_sim_hold(sim);
4053 	new_bus->sim = sim;
4054 	timevalclear(&new_bus->last_reset);
4055 	new_bus->flags = 0;
4056 	new_bus->refcount = 1;	/* Held until a bus_deregister event */
4057 	new_bus->generation = 0;
4058 
4059 	xpt_lock_buses();
4060 	sim->path_id = new_bus->path_id =
4061 	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
4062 	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4063 	while (old_bus != NULL
4064 	    && old_bus->path_id < new_bus->path_id)
4065 		old_bus = TAILQ_NEXT(old_bus, links);
4066 	if (old_bus != NULL)
4067 		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
4068 	else
4069 		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
4070 	xsoftc.bus_generation++;
4071 	xpt_unlock_buses();
4072 
4073 	/*
4074 	 * Set a default transport so that a PATH_INQ can be issued to
4075 	 * the SIM.  This will then allow for probing and attaching of
4076 	 * a more appropriate transport.
4077 	 */
4078 	new_bus->xport = &xport_default;
4079 
4080 	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
4081 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4082 	if (status != CAM_REQ_CMP) {
4083 		xpt_release_bus(new_bus);
4084 		return (CAM_RESRC_UNAVAIL);
4085 	}
4086 
4087 	xpt_path_inq(&cpi, path);
4088 
4089 	if (cpi.ccb_h.status == CAM_REQ_CMP) {
4090 		struct xpt_xport **xpt;
4091 
4092 		SET_FOREACH(xpt, cam_xpt_xport_set) {
4093 			if ((*xpt)->xport == cpi.transport) {
4094 				new_bus->xport = *xpt;
4095 				break;
4096 			}
4097 		}
4098 		if (new_bus->xport == NULL) {
4099 			xpt_print(path,
4100 			    "No transport found for %d\n", cpi.transport);
4101 			xpt_release_bus(new_bus);
4102 			free(path, M_CAMXPT);
4103 			return (CAM_RESRC_UNAVAIL);
4104 		}
4105 	}
4106 
4107 	/* Notify interested parties */
4108 	if (sim->path_id != CAM_XPT_PATH_ID) {
4109 
4110 		xpt_async(AC_PATH_REGISTERED, path, &cpi);
4111 		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
4112 			union	ccb *scan_ccb;
4113 
4114 			/* Initiate bus rescan. */
4115 			scan_ccb = xpt_alloc_ccb_nowait();
4116 			if (scan_ccb != NULL) {
4117 				scan_ccb->ccb_h.path = path;
4118 				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
4119 				scan_ccb->crcn.flags = 0;
4120 				xpt_rescan(scan_ccb);
4121 			} else {
4122 				xpt_print(path,
4123 					  "Can't allocate CCB to scan bus\n");
4124 				xpt_free_path(path);
4125 			}
4126 		} else
4127 			xpt_free_path(path);
4128 	} else
4129 		xpt_free_path(path);
4130 	return (CAM_SUCCESS);
4131 }
4132 
4133 int32_t
4134 xpt_bus_deregister(path_id_t pathid)
4135 {
4136 	struct cam_path bus_path;
4137 	cam_status status;
4138 
4139 	status = xpt_compile_path(&bus_path, NULL, pathid,
4140 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4141 	if (status != CAM_REQ_CMP)
4142 		return (status);
4143 
4144 	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
4145 	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
4146 
4147 	/* Release the reference count held while registered. */
4148 	xpt_release_bus(bus_path.bus);
4149 	xpt_release_path(&bus_path);
4150 
4151 	return (CAM_REQ_CMP);
4152 }
4153 
4154 static path_id_t
4155 xptnextfreepathid(void)
4156 {
4157 	struct cam_eb *bus;
4158 	path_id_t pathid;
4159 	const char *strval;
4160 
4161 	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4162 	pathid = 0;
4163 	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4164 retry:
4165 	/* Find an unoccupied pathid */
4166 	while (bus != NULL && bus->path_id <= pathid) {
4167 		if (bus->path_id == pathid)
4168 			pathid++;
4169 		bus = TAILQ_NEXT(bus, links);
4170 	}
4171 
4172 	/*
4173 	 * Ensure that this pathid is not reserved for
4174 	 * a bus that may be registered in the future.
4175 	 */
4176 	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4177 		++pathid;
4178 		/* Start the search over */
4179 		goto retry;
4180 	}
4181 	return (pathid);
4182 }
4183 
4184 static path_id_t
4185 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4186 {
4187 	path_id_t pathid;
4188 	int i, dunit, val;
4189 	char buf[32];
4190 	const char *dname;
4191 
4192 	pathid = CAM_XPT_PATH_ID;
4193 	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4194 	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4195 		return (pathid);
4196 	i = 0;
4197 	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4198 		if (strcmp(dname, "scbus")) {
4199 			/* Avoid a bit of foot shooting. */
4200 			continue;
4201 		}
4202 		if (dunit < 0)		/* unwired?! */
4203 			continue;
4204 		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4205 			if (sim_bus == val) {
4206 				pathid = dunit;
4207 				break;
4208 			}
4209 		} else if (sim_bus == 0) {
4210 			/* Unspecified matches bus 0 */
4211 			pathid = dunit;
4212 			break;
4213 		} else {
4214 			printf("Ambiguous scbus configuration for %s%d "
4215 			       "bus %d, cannot wire down.  The kernel "
4216 			       "config entry for scbus%d should "
4217 			       "specify a controller bus.\n"
4218 			       "Scbus will be assigned dynamically.\n",
4219 			       sim_name, sim_unit, sim_bus, dunit);
4220 			break;
4221 		}
4222 	}
4223 
4224 	if (pathid == CAM_XPT_PATH_ID)
4225 		pathid = xptnextfreepathid();
4226 	return (pathid);
4227 }
4228 
4229 static const char *
4230 xpt_async_string(u_int32_t async_code)
4231 {
4232 
4233 	switch (async_code) {
4234 	case AC_BUS_RESET: return ("AC_BUS_RESET");
4235 	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4236 	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4237 	case AC_SENT_BDR: return ("AC_SENT_BDR");
4238 	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4239 	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4240 	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4241 	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4242 	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4243 	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4244 	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4245 	case AC_CONTRACT: return ("AC_CONTRACT");
4246 	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4247 	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4248 	}
4249 	return ("AC_UNKNOWN");
4250 }
4251 
4252 static int
4253 xpt_async_size(u_int32_t async_code)
4254 {
4255 
4256 	switch (async_code) {
4257 	case AC_BUS_RESET: return (0);
4258 	case AC_UNSOL_RESEL: return (0);
4259 	case AC_SCSI_AEN: return (0);
4260 	case AC_SENT_BDR: return (0);
4261 	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4262 	case AC_PATH_DEREGISTERED: return (0);
4263 	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4264 	case AC_LOST_DEVICE: return (0);
4265 	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4266 	case AC_INQ_CHANGED: return (0);
4267 	case AC_GETDEV_CHANGED: return (0);
4268 	case AC_CONTRACT: return (sizeof(struct ac_contract));
4269 	case AC_ADVINFO_CHANGED: return (-1);
4270 	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4271 	}
4272 	return (0);
4273 }
4274 
4275 static int
4276 xpt_async_process_dev(struct cam_ed *device, void *arg)
4277 {
4278 	union ccb *ccb = arg;
4279 	struct cam_path *path = ccb->ccb_h.path;
4280 	void *async_arg = ccb->casync.async_arg_ptr;
4281 	u_int32_t async_code = ccb->casync.async_code;
4282 	int relock;
4283 
4284 	if (path->device != device
4285 	 && path->device->lun_id != CAM_LUN_WILDCARD
4286 	 && device->lun_id != CAM_LUN_WILDCARD)
4287 		return (1);
4288 
4289 	/*
4290 	 * The async callback could free the device.
4291 	 * If it is a broadcast async, it doesn't hold
4292 	 * device reference, so take our own reference.
4293 	 */
4294 	xpt_acquire_device(device);
4295 
4296 	/*
4297 	 * If async for specific device is to be delivered to
4298 	 * the wildcard client, take the specific device lock.
4299 	 * XXX: We may need a way for client to specify it.
4300 	 */
4301 	if ((device->lun_id == CAM_LUN_WILDCARD &&
4302 	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4303 	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4304 	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4305 	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4306 	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4307 		mtx_unlock(&device->device_mtx);
4308 		xpt_path_lock(path);
4309 		relock = 1;
4310 	} else
4311 		relock = 0;
4312 
4313 	(*(device->target->bus->xport->ops->async))(async_code,
4314 	    device->target->bus, device->target, device, async_arg);
4315 	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4316 
4317 	if (relock) {
4318 		xpt_path_unlock(path);
4319 		mtx_lock(&device->device_mtx);
4320 	}
4321 	xpt_release_device(device);
4322 	return (1);
4323 }
4324 
4325 static int
4326 xpt_async_process_tgt(struct cam_et *target, void *arg)
4327 {
4328 	union ccb *ccb = arg;
4329 	struct cam_path *path = ccb->ccb_h.path;
4330 
4331 	if (path->target != target
4332 	 && path->target->target_id != CAM_TARGET_WILDCARD
4333 	 && target->target_id != CAM_TARGET_WILDCARD)
4334 		return (1);
4335 
4336 	if (ccb->casync.async_code == AC_SENT_BDR) {
4337 		/* Update our notion of when the last reset occurred */
4338 		microtime(&target->last_reset);
4339 	}
4340 
4341 	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4342 }
4343 
4344 static void
4345 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4346 {
4347 	struct cam_eb *bus;
4348 	struct cam_path *path;
4349 	void *async_arg;
4350 	u_int32_t async_code;
4351 
4352 	path = ccb->ccb_h.path;
4353 	async_code = ccb->casync.async_code;
4354 	async_arg = ccb->casync.async_arg_ptr;
4355 	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4356 	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4357 	bus = path->bus;
4358 
4359 	if (async_code == AC_BUS_RESET) {
4360 		/* Update our notion of when the last reset occurred */
4361 		microtime(&bus->last_reset);
4362 	}
4363 
4364 	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4365 
4366 	/*
4367 	 * If this wasn't a fully wildcarded async, tell all
4368 	 * clients that want all async events.
4369 	 */
4370 	if (bus != xpt_periph->path->bus) {
4371 		xpt_path_lock(xpt_periph->path);
4372 		xpt_async_process_dev(xpt_periph->path->device, ccb);
4373 		xpt_path_unlock(xpt_periph->path);
4374 	}
4375 
4376 	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4377 		xpt_release_devq(path, 1, TRUE);
4378 	else
4379 		xpt_release_simq(path->bus->sim, TRUE);
4380 	if (ccb->casync.async_arg_size > 0)
4381 		free(async_arg, M_CAMXPT);
4382 	xpt_free_path(path);
4383 	xpt_free_ccb(ccb);
4384 }
4385 
4386 static void
4387 xpt_async_bcast(struct async_list *async_head,
4388 		u_int32_t async_code,
4389 		struct cam_path *path, void *async_arg)
4390 {
4391 	struct async_node *cur_entry;
4392 	struct mtx *mtx;
4393 
4394 	cur_entry = SLIST_FIRST(async_head);
4395 	while (cur_entry != NULL) {
4396 		struct async_node *next_entry;
4397 		/*
4398 		 * Grab the next list entry before we call the current
4399 		 * entry's callback.  This is because the callback function
4400 		 * can delete its async callback entry.
4401 		 */
4402 		next_entry = SLIST_NEXT(cur_entry, links);
4403 		if ((cur_entry->event_enable & async_code) != 0) {
4404 			mtx = cur_entry->event_lock ?
4405 			    path->device->sim->mtx : NULL;
4406 			if (mtx)
4407 				mtx_lock(mtx);
4408 			cur_entry->callback(cur_entry->callback_arg,
4409 					    async_code, path,
4410 					    async_arg);
4411 			if (mtx)
4412 				mtx_unlock(mtx);
4413 		}
4414 		cur_entry = next_entry;
4415 	}
4416 }
4417 
4418 void
4419 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4420 {
4421 	union ccb *ccb;
4422 	int size;
4423 
4424 	ccb = xpt_alloc_ccb_nowait();
4425 	if (ccb == NULL) {
4426 		xpt_print(path, "Can't allocate CCB to send %s\n",
4427 		    xpt_async_string(async_code));
4428 		return;
4429 	}
4430 
4431 	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4432 		xpt_print(path, "Can't allocate path to send %s\n",
4433 		    xpt_async_string(async_code));
4434 		xpt_free_ccb(ccb);
4435 		return;
4436 	}
4437 	ccb->ccb_h.path->periph = NULL;
4438 	ccb->ccb_h.func_code = XPT_ASYNC;
4439 	ccb->ccb_h.cbfcnp = xpt_async_process;
4440 	ccb->ccb_h.flags |= CAM_UNLOCKED;
4441 	ccb->casync.async_code = async_code;
4442 	ccb->casync.async_arg_size = 0;
4443 	size = xpt_async_size(async_code);
4444 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
4445 	    ("xpt_async: func %#x %s aync_code %d %s\n",
4446 		ccb->ccb_h.func_code,
4447 		xpt_action_name(ccb->ccb_h.func_code),
4448 		async_code,
4449 		xpt_async_string(async_code)));
4450 	if (size > 0 && async_arg != NULL) {
4451 		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4452 		if (ccb->casync.async_arg_ptr == NULL) {
4453 			xpt_print(path, "Can't allocate argument to send %s\n",
4454 			    xpt_async_string(async_code));
4455 			xpt_free_path(ccb->ccb_h.path);
4456 			xpt_free_ccb(ccb);
4457 			return;
4458 		}
4459 		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4460 		ccb->casync.async_arg_size = size;
4461 	} else if (size < 0) {
4462 		ccb->casync.async_arg_ptr = async_arg;
4463 		ccb->casync.async_arg_size = size;
4464 	}
4465 	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4466 		xpt_freeze_devq(path, 1);
4467 	else
4468 		xpt_freeze_simq(path->bus->sim, 1);
4469 	xpt_done(ccb);
4470 }
4471 
4472 static void
4473 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4474 		      struct cam_et *target, struct cam_ed *device,
4475 		      void *async_arg)
4476 {
4477 
4478 	/*
4479 	 * We only need to handle events for real devices.
4480 	 */
4481 	if (target->target_id == CAM_TARGET_WILDCARD
4482 	 || device->lun_id == CAM_LUN_WILDCARD)
4483 		return;
4484 
4485 	printf("%s called\n", __func__);
4486 }
4487 
4488 static uint32_t
4489 xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4490 {
4491 	struct cam_devq	*devq;
4492 	uint32_t freeze;
4493 
4494 	devq = dev->sim->devq;
4495 	mtx_assert(&devq->send_mtx, MA_OWNED);
4496 	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4497 	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4498 	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4499 	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4500 	/* Remove frozen device from sendq. */
4501 	if (device_is_queued(dev))
4502 		camq_remove(&devq->send_queue, dev->devq_entry.index);
4503 	return (freeze);
4504 }
4505 
4506 u_int32_t
4507 xpt_freeze_devq(struct cam_path *path, u_int count)
4508 {
4509 	struct cam_ed	*dev = path->device;
4510 	struct cam_devq	*devq;
4511 	uint32_t	 freeze;
4512 
4513 	devq = dev->sim->devq;
4514 	mtx_lock(&devq->send_mtx);
4515 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4516 	freeze = xpt_freeze_devq_device(dev, count);
4517 	mtx_unlock(&devq->send_mtx);
4518 	return (freeze);
4519 }
4520 
4521 u_int32_t
4522 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4523 {
4524 	struct cam_devq	*devq;
4525 	uint32_t	 freeze;
4526 
4527 	devq = sim->devq;
4528 	mtx_lock(&devq->send_mtx);
4529 	freeze = (devq->send_queue.qfrozen_cnt += count);
4530 	mtx_unlock(&devq->send_mtx);
4531 	return (freeze);
4532 }
4533 
4534 static void
4535 xpt_release_devq_timeout(void *arg)
4536 {
4537 	struct cam_ed *dev;
4538 	struct cam_devq *devq;
4539 
4540 	dev = (struct cam_ed *)arg;
4541 	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4542 	devq = dev->sim->devq;
4543 	mtx_assert(&devq->send_mtx, MA_OWNED);
4544 	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4545 		xpt_run_devq(devq);
4546 }
4547 
4548 void
4549 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4550 {
4551 	struct cam_ed *dev;
4552 	struct cam_devq *devq;
4553 
4554 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4555 	    count, run_queue));
4556 	dev = path->device;
4557 	devq = dev->sim->devq;
4558 	mtx_lock(&devq->send_mtx);
4559 	if (xpt_release_devq_device(dev, count, run_queue))
4560 		xpt_run_devq(dev->sim->devq);
4561 	mtx_unlock(&devq->send_mtx);
4562 }
4563 
4564 static int
4565 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4566 {
4567 
4568 	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4569 	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4570 	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4571 	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4572 	if (count > dev->ccbq.queue.qfrozen_cnt) {
4573 #ifdef INVARIANTS
4574 		printf("xpt_release_devq(): requested %u > present %u\n",
4575 		    count, dev->ccbq.queue.qfrozen_cnt);
4576 #endif
4577 		count = dev->ccbq.queue.qfrozen_cnt;
4578 	}
4579 	dev->ccbq.queue.qfrozen_cnt -= count;
4580 	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4581 		/*
4582 		 * No longer need to wait for a successful
4583 		 * command completion.
4584 		 */
4585 		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4586 		/*
4587 		 * Remove any timeouts that might be scheduled
4588 		 * to release this queue.
4589 		 */
4590 		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4591 			callout_stop(&dev->callout);
4592 			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4593 		}
4594 		/*
4595 		 * Now that we are unfrozen schedule the
4596 		 * device so any pending transactions are
4597 		 * run.
4598 		 */
4599 		xpt_schedule_devq(dev->sim->devq, dev);
4600 	} else
4601 		run_queue = 0;
4602 	return (run_queue);
4603 }
4604 
4605 void
4606 xpt_release_simq(struct cam_sim *sim, int run_queue)
4607 {
4608 	struct cam_devq	*devq;
4609 
4610 	devq = sim->devq;
4611 	mtx_lock(&devq->send_mtx);
4612 	if (devq->send_queue.qfrozen_cnt <= 0) {
4613 #ifdef INVARIANTS
4614 		printf("xpt_release_simq: requested 1 > present %u\n",
4615 		    devq->send_queue.qfrozen_cnt);
4616 #endif
4617 	} else
4618 		devq->send_queue.qfrozen_cnt--;
4619 	if (devq->send_queue.qfrozen_cnt == 0) {
4620 		/*
4621 		 * If there is a timeout scheduled to release this
4622 		 * sim queue, remove it.  The queue frozen count is
4623 		 * already at 0.
4624 		 */
4625 		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4626 			callout_stop(&sim->callout);
4627 			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4628 		}
4629 		if (run_queue) {
4630 			/*
4631 			 * Now that we are unfrozen run the send queue.
4632 			 */
4633 			xpt_run_devq(sim->devq);
4634 		}
4635 	}
4636 	mtx_unlock(&devq->send_mtx);
4637 }
4638 
4639 /*
4640  * XXX Appears to be unused.
4641  */
4642 static void
4643 xpt_release_simq_timeout(void *arg)
4644 {
4645 	struct cam_sim *sim;
4646 
4647 	sim = (struct cam_sim *)arg;
4648 	xpt_release_simq(sim, /* run_queue */ TRUE);
4649 }
4650 
4651 void
4652 xpt_done(union ccb *done_ccb)
4653 {
4654 	struct cam_doneq *queue;
4655 	int	run, hash;
4656 
4657 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4658 	if (done_ccb->ccb_h.func_code == XPT_SCSI_IO &&
4659 	    done_ccb->csio.bio != NULL)
4660 		biotrack(done_ccb->csio.bio, __func__);
4661 #endif
4662 
4663 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4664 	    ("xpt_done: func= %#x %s status %#x\n",
4665 		done_ccb->ccb_h.func_code,
4666 		xpt_action_name(done_ccb->ccb_h.func_code),
4667 		done_ccb->ccb_h.status));
4668 	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4669 		return;
4670 
4671 	/* Store the time the ccb was in the sim */
4672 	done_ccb->ccb_h.qos.periph_data = cam_iosched_delta_t(done_ccb->ccb_h.qos.periph_data);
4673 	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4674 	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4675 	queue = &cam_doneqs[hash];
4676 	mtx_lock(&queue->cam_doneq_mtx);
4677 	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4678 	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4679 	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4680 	mtx_unlock(&queue->cam_doneq_mtx);
4681 	if (run)
4682 		wakeup(&queue->cam_doneq);
4683 }
4684 
4685 void
4686 xpt_done_direct(union ccb *done_ccb)
4687 {
4688 
4689 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4690 	    ("xpt_done_direct: status %#x\n", done_ccb->ccb_h.status));
4691 	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4692 		return;
4693 
4694 	/* Store the time the ccb was in the sim */
4695 	done_ccb->ccb_h.qos.periph_data = cam_iosched_delta_t(done_ccb->ccb_h.qos.periph_data);
4696 	xpt_done_process(&done_ccb->ccb_h);
4697 }
4698 
4699 union ccb *
4700 xpt_alloc_ccb()
4701 {
4702 	union ccb *new_ccb;
4703 
4704 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4705 	return (new_ccb);
4706 }
4707 
4708 union ccb *
4709 xpt_alloc_ccb_nowait()
4710 {
4711 	union ccb *new_ccb;
4712 
4713 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4714 	return (new_ccb);
4715 }
4716 
4717 void
4718 xpt_free_ccb(union ccb *free_ccb)
4719 {
4720 	free(free_ccb, M_CAMCCB);
4721 }
4722 
4723 
4724 
4725 /* Private XPT functions */
4726 
4727 /*
4728  * Get a CAM control block for the caller. Charge the structure to the device
4729  * referenced by the path.  If we don't have sufficient resources to allocate
4730  * more ccbs, we return NULL.
4731  */
4732 static union ccb *
4733 xpt_get_ccb_nowait(struct cam_periph *periph)
4734 {
4735 	union ccb *new_ccb;
4736 
4737 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4738 	if (new_ccb == NULL)
4739 		return (NULL);
4740 	periph->periph_allocated++;
4741 	cam_ccbq_take_opening(&periph->path->device->ccbq);
4742 	return (new_ccb);
4743 }
4744 
4745 static union ccb *
4746 xpt_get_ccb(struct cam_periph *periph)
4747 {
4748 	union ccb *new_ccb;
4749 
4750 	cam_periph_unlock(periph);
4751 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4752 	cam_periph_lock(periph);
4753 	periph->periph_allocated++;
4754 	cam_ccbq_take_opening(&periph->path->device->ccbq);
4755 	return (new_ccb);
4756 }
4757 
4758 union ccb *
4759 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4760 {
4761 	struct ccb_hdr *ccb_h;
4762 
4763 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4764 	cam_periph_assert(periph, MA_OWNED);
4765 	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4766 	    ccb_h->pinfo.priority != priority) {
4767 		if (priority < periph->immediate_priority) {
4768 			periph->immediate_priority = priority;
4769 			xpt_run_allocq(periph, 0);
4770 		} else
4771 			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4772 			    "cgticb", 0);
4773 	}
4774 	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4775 	return ((union ccb *)ccb_h);
4776 }
4777 
4778 static void
4779 xpt_acquire_bus(struct cam_eb *bus)
4780 {
4781 
4782 	xpt_lock_buses();
4783 	bus->refcount++;
4784 	xpt_unlock_buses();
4785 }
4786 
4787 static void
4788 xpt_release_bus(struct cam_eb *bus)
4789 {
4790 
4791 	xpt_lock_buses();
4792 	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4793 	if (--bus->refcount > 0) {
4794 		xpt_unlock_buses();
4795 		return;
4796 	}
4797 	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4798 	xsoftc.bus_generation++;
4799 	xpt_unlock_buses();
4800 	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4801 	    ("destroying bus, but target list is not empty"));
4802 	cam_sim_release(bus->sim);
4803 	mtx_destroy(&bus->eb_mtx);
4804 	free(bus, M_CAMXPT);
4805 }
4806 
4807 static struct cam_et *
4808 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4809 {
4810 	struct cam_et *cur_target, *target;
4811 
4812 	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4813 	mtx_assert(&bus->eb_mtx, MA_OWNED);
4814 	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4815 					 M_NOWAIT|M_ZERO);
4816 	if (target == NULL)
4817 		return (NULL);
4818 
4819 	TAILQ_INIT(&target->ed_entries);
4820 	target->bus = bus;
4821 	target->target_id = target_id;
4822 	target->refcount = 1;
4823 	target->generation = 0;
4824 	target->luns = NULL;
4825 	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4826 	timevalclear(&target->last_reset);
4827 	/*
4828 	 * Hold a reference to our parent bus so it
4829 	 * will not go away before we do.
4830 	 */
4831 	bus->refcount++;
4832 
4833 	/* Insertion sort into our bus's target list */
4834 	cur_target = TAILQ_FIRST(&bus->et_entries);
4835 	while (cur_target != NULL && cur_target->target_id < target_id)
4836 		cur_target = TAILQ_NEXT(cur_target, links);
4837 	if (cur_target != NULL) {
4838 		TAILQ_INSERT_BEFORE(cur_target, target, links);
4839 	} else {
4840 		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4841 	}
4842 	bus->generation++;
4843 	return (target);
4844 }
4845 
4846 static void
4847 xpt_acquire_target(struct cam_et *target)
4848 {
4849 	struct cam_eb *bus = target->bus;
4850 
4851 	mtx_lock(&bus->eb_mtx);
4852 	target->refcount++;
4853 	mtx_unlock(&bus->eb_mtx);
4854 }
4855 
4856 static void
4857 xpt_release_target(struct cam_et *target)
4858 {
4859 	struct cam_eb *bus = target->bus;
4860 
4861 	mtx_lock(&bus->eb_mtx);
4862 	if (--target->refcount > 0) {
4863 		mtx_unlock(&bus->eb_mtx);
4864 		return;
4865 	}
4866 	TAILQ_REMOVE(&bus->et_entries, target, links);
4867 	bus->generation++;
4868 	mtx_unlock(&bus->eb_mtx);
4869 	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4870 	    ("destroying target, but device list is not empty"));
4871 	xpt_release_bus(bus);
4872 	mtx_destroy(&target->luns_mtx);
4873 	if (target->luns)
4874 		free(target->luns, M_CAMXPT);
4875 	free(target, M_CAMXPT);
4876 }
4877 
4878 static struct cam_ed *
4879 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4880 			 lun_id_t lun_id)
4881 {
4882 	struct cam_ed *device;
4883 
4884 	device = xpt_alloc_device(bus, target, lun_id);
4885 	if (device == NULL)
4886 		return (NULL);
4887 
4888 	device->mintags = 1;
4889 	device->maxtags = 1;
4890 	return (device);
4891 }
4892 
4893 static void
4894 xpt_destroy_device(void *context, int pending)
4895 {
4896 	struct cam_ed	*device = context;
4897 
4898 	mtx_lock(&device->device_mtx);
4899 	mtx_destroy(&device->device_mtx);
4900 	free(device, M_CAMDEV);
4901 }
4902 
4903 struct cam_ed *
4904 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4905 {
4906 	struct cam_ed	*cur_device, *device;
4907 	struct cam_devq	*devq;
4908 	cam_status status;
4909 
4910 	mtx_assert(&bus->eb_mtx, MA_OWNED);
4911 	/* Make space for us in the device queue on our bus */
4912 	devq = bus->sim->devq;
4913 	mtx_lock(&devq->send_mtx);
4914 	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4915 	mtx_unlock(&devq->send_mtx);
4916 	if (status != CAM_REQ_CMP)
4917 		return (NULL);
4918 
4919 	device = (struct cam_ed *)malloc(sizeof(*device),
4920 					 M_CAMDEV, M_NOWAIT|M_ZERO);
4921 	if (device == NULL)
4922 		return (NULL);
4923 
4924 	cam_init_pinfo(&device->devq_entry);
4925 	device->target = target;
4926 	device->lun_id = lun_id;
4927 	device->sim = bus->sim;
4928 	if (cam_ccbq_init(&device->ccbq,
4929 			  bus->sim->max_dev_openings) != 0) {
4930 		free(device, M_CAMDEV);
4931 		return (NULL);
4932 	}
4933 	SLIST_INIT(&device->asyncs);
4934 	SLIST_INIT(&device->periphs);
4935 	device->generation = 0;
4936 	device->flags = CAM_DEV_UNCONFIGURED;
4937 	device->tag_delay_count = 0;
4938 	device->tag_saved_openings = 0;
4939 	device->refcount = 1;
4940 	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4941 	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4942 	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4943 	/*
4944 	 * Hold a reference to our parent bus so it
4945 	 * will not go away before we do.
4946 	 */
4947 	target->refcount++;
4948 
4949 	cur_device = TAILQ_FIRST(&target->ed_entries);
4950 	while (cur_device != NULL && cur_device->lun_id < lun_id)
4951 		cur_device = TAILQ_NEXT(cur_device, links);
4952 	if (cur_device != NULL)
4953 		TAILQ_INSERT_BEFORE(cur_device, device, links);
4954 	else
4955 		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4956 	target->generation++;
4957 	return (device);
4958 }
4959 
4960 void
4961 xpt_acquire_device(struct cam_ed *device)
4962 {
4963 	struct cam_eb *bus = device->target->bus;
4964 
4965 	mtx_lock(&bus->eb_mtx);
4966 	device->refcount++;
4967 	mtx_unlock(&bus->eb_mtx);
4968 }
4969 
4970 void
4971 xpt_release_device(struct cam_ed *device)
4972 {
4973 	struct cam_eb *bus = device->target->bus;
4974 	struct cam_devq *devq;
4975 
4976 	mtx_lock(&bus->eb_mtx);
4977 	if (--device->refcount > 0) {
4978 		mtx_unlock(&bus->eb_mtx);
4979 		return;
4980 	}
4981 
4982 	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4983 	device->target->generation++;
4984 	mtx_unlock(&bus->eb_mtx);
4985 
4986 	/* Release our slot in the devq */
4987 	devq = bus->sim->devq;
4988 	mtx_lock(&devq->send_mtx);
4989 	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4990 	mtx_unlock(&devq->send_mtx);
4991 
4992 	KASSERT(SLIST_EMPTY(&device->periphs),
4993 	    ("destroying device, but periphs list is not empty"));
4994 	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4995 	    ("destroying device while still queued for ccbs"));
4996 
4997 	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4998 		callout_stop(&device->callout);
4999 
5000 	xpt_release_target(device->target);
5001 
5002 	cam_ccbq_fini(&device->ccbq);
5003 	/*
5004 	 * Free allocated memory.  free(9) does nothing if the
5005 	 * supplied pointer is NULL, so it is safe to call without
5006 	 * checking.
5007 	 */
5008 	free(device->supported_vpds, M_CAMXPT);
5009 	free(device->device_id, M_CAMXPT);
5010 	free(device->ext_inq, M_CAMXPT);
5011 	free(device->physpath, M_CAMXPT);
5012 	free(device->rcap_buf, M_CAMXPT);
5013 	free(device->serial_num, M_CAMXPT);
5014 	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
5015 }
5016 
5017 u_int32_t
5018 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
5019 {
5020 	int	result;
5021 	struct	cam_ed *dev;
5022 
5023 	dev = path->device;
5024 	mtx_lock(&dev->sim->devq->send_mtx);
5025 	result = cam_ccbq_resize(&dev->ccbq, newopenings);
5026 	mtx_unlock(&dev->sim->devq->send_mtx);
5027 	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5028 	 || (dev->inq_flags & SID_CmdQue) != 0)
5029 		dev->tag_saved_openings = newopenings;
5030 	return (result);
5031 }
5032 
5033 static struct cam_eb *
5034 xpt_find_bus(path_id_t path_id)
5035 {
5036 	struct cam_eb *bus;
5037 
5038 	xpt_lock_buses();
5039 	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
5040 	     bus != NULL;
5041 	     bus = TAILQ_NEXT(bus, links)) {
5042 		if (bus->path_id == path_id) {
5043 			bus->refcount++;
5044 			break;
5045 		}
5046 	}
5047 	xpt_unlock_buses();
5048 	return (bus);
5049 }
5050 
5051 static struct cam_et *
5052 xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
5053 {
5054 	struct cam_et *target;
5055 
5056 	mtx_assert(&bus->eb_mtx, MA_OWNED);
5057 	for (target = TAILQ_FIRST(&bus->et_entries);
5058 	     target != NULL;
5059 	     target = TAILQ_NEXT(target, links)) {
5060 		if (target->target_id == target_id) {
5061 			target->refcount++;
5062 			break;
5063 		}
5064 	}
5065 	return (target);
5066 }
5067 
5068 static struct cam_ed *
5069 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
5070 {
5071 	struct cam_ed *device;
5072 
5073 	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
5074 	for (device = TAILQ_FIRST(&target->ed_entries);
5075 	     device != NULL;
5076 	     device = TAILQ_NEXT(device, links)) {
5077 		if (device->lun_id == lun_id) {
5078 			device->refcount++;
5079 			break;
5080 		}
5081 	}
5082 	return (device);
5083 }
5084 
5085 void
5086 xpt_start_tags(struct cam_path *path)
5087 {
5088 	struct ccb_relsim crs;
5089 	struct cam_ed *device;
5090 	struct cam_sim *sim;
5091 	int    newopenings;
5092 
5093 	device = path->device;
5094 	sim = path->bus->sim;
5095 	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
5096 	xpt_freeze_devq(path, /*count*/1);
5097 	device->inq_flags |= SID_CmdQue;
5098 	if (device->tag_saved_openings != 0)
5099 		newopenings = device->tag_saved_openings;
5100 	else
5101 		newopenings = min(device->maxtags,
5102 				  sim->max_tagged_dev_openings);
5103 	xpt_dev_ccbq_resize(path, newopenings);
5104 	xpt_async(AC_GETDEV_CHANGED, path, NULL);
5105 	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
5106 	crs.ccb_h.func_code = XPT_REL_SIMQ;
5107 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
5108 	crs.openings
5109 	    = crs.release_timeout
5110 	    = crs.qfrozen_cnt
5111 	    = 0;
5112 	xpt_action((union ccb *)&crs);
5113 }
5114 
5115 void
5116 xpt_stop_tags(struct cam_path *path)
5117 {
5118 	struct ccb_relsim crs;
5119 	struct cam_ed *device;
5120 	struct cam_sim *sim;
5121 
5122 	device = path->device;
5123 	sim = path->bus->sim;
5124 	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
5125 	device->tag_delay_count = 0;
5126 	xpt_freeze_devq(path, /*count*/1);
5127 	device->inq_flags &= ~SID_CmdQue;
5128 	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
5129 	xpt_async(AC_GETDEV_CHANGED, path, NULL);
5130 	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
5131 	crs.ccb_h.func_code = XPT_REL_SIMQ;
5132 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
5133 	crs.openings
5134 	    = crs.release_timeout
5135 	    = crs.qfrozen_cnt
5136 	    = 0;
5137 	xpt_action((union ccb *)&crs);
5138 }
5139 
5140 static void
5141 xpt_boot_delay(void *arg)
5142 {
5143 
5144 	xpt_release_boot();
5145 }
5146 
5147 static void
5148 xpt_config(void *arg)
5149 {
5150 	/*
5151 	 * Now that interrupts are enabled, go find our devices
5152 	 */
5153 	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
5154 		printf("xpt_config: failed to create taskqueue thread.\n");
5155 
5156 	/* Setup debugging path */
5157 	if (cam_dflags != CAM_DEBUG_NONE) {
5158 		if (xpt_create_path(&cam_dpath, NULL,
5159 				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
5160 				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
5161 			printf("xpt_config: xpt_create_path() failed for debug"
5162 			       " target %d:%d:%d, debugging disabled\n",
5163 			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
5164 			cam_dflags = CAM_DEBUG_NONE;
5165 		}
5166 	} else
5167 		cam_dpath = NULL;
5168 
5169 	periphdriver_init(1);
5170 	xpt_hold_boot();
5171 	callout_init(&xsoftc.boot_callout, 1);
5172 	callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
5173 	    xpt_boot_delay, NULL, 0);
5174 	/* Fire up rescan thread. */
5175 	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
5176 	    "cam", "scanner")) {
5177 		printf("xpt_config: failed to create rescan thread.\n");
5178 	}
5179 }
5180 
5181 void
5182 xpt_hold_boot(void)
5183 {
5184 	xpt_lock_buses();
5185 	xsoftc.buses_to_config++;
5186 	xpt_unlock_buses();
5187 }
5188 
5189 void
5190 xpt_release_boot(void)
5191 {
5192 	xpt_lock_buses();
5193 	xsoftc.buses_to_config--;
5194 	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
5195 		struct	xpt_task *task;
5196 
5197 		xsoftc.buses_config_done = 1;
5198 		xpt_unlock_buses();
5199 		/* Call manually because we don't have any buses */
5200 		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5201 		if (task != NULL) {
5202 			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5203 			taskqueue_enqueue(taskqueue_thread, &task->task);
5204 		}
5205 	} else
5206 		xpt_unlock_buses();
5207 }
5208 
5209 /*
5210  * If the given device only has one peripheral attached to it, and if that
5211  * peripheral is the passthrough driver, announce it.  This insures that the
5212  * user sees some sort of announcement for every peripheral in their system.
5213  */
5214 static int
5215 xptpassannouncefunc(struct cam_ed *device, void *arg)
5216 {
5217 	struct cam_periph *periph;
5218 	int i;
5219 
5220 	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5221 	     periph = SLIST_NEXT(periph, periph_links), i++);
5222 
5223 	periph = SLIST_FIRST(&device->periphs);
5224 	if ((i == 1)
5225 	 && (strncmp(periph->periph_name, "pass", 4) == 0))
5226 		xpt_announce_periph(periph, NULL);
5227 
5228 	return(1);
5229 }
5230 
5231 static void
5232 xpt_finishconfig_task(void *context, int pending)
5233 {
5234 
5235 	periphdriver_init(2);
5236 	/*
5237 	 * Check for devices with no "standard" peripheral driver
5238 	 * attached.  For any devices like that, announce the
5239 	 * passthrough driver so the user will see something.
5240 	 */
5241 	if (!bootverbose)
5242 		xpt_for_all_devices(xptpassannouncefunc, NULL);
5243 
5244 	/* Release our hook so that the boot can continue. */
5245 	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5246 	free(xsoftc.xpt_config_hook, M_CAMXPT);
5247 	xsoftc.xpt_config_hook = NULL;
5248 
5249 	free(context, M_CAMXPT);
5250 }
5251 
5252 cam_status
5253 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5254 		   struct cam_path *path)
5255 {
5256 	struct ccb_setasync csa;
5257 	cam_status status;
5258 	int xptpath = 0;
5259 
5260 	if (path == NULL) {
5261 		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5262 					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5263 		if (status != CAM_REQ_CMP)
5264 			return (status);
5265 		xpt_path_lock(path);
5266 		xptpath = 1;
5267 	}
5268 
5269 	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5270 	csa.ccb_h.func_code = XPT_SASYNC_CB;
5271 	csa.event_enable = event;
5272 	csa.callback = cbfunc;
5273 	csa.callback_arg = cbarg;
5274 	xpt_action((union ccb *)&csa);
5275 	status = csa.ccb_h.status;
5276 
5277 	CAM_DEBUG(csa.ccb_h.path, CAM_DEBUG_TRACE,
5278 	    ("xpt_register_async: func %p\n", cbfunc));
5279 
5280 	if (xptpath) {
5281 		xpt_path_unlock(path);
5282 		xpt_free_path(path);
5283 	}
5284 
5285 	if ((status == CAM_REQ_CMP) &&
5286 	    (csa.event_enable & AC_FOUND_DEVICE)) {
5287 		/*
5288 		 * Get this peripheral up to date with all
5289 		 * the currently existing devices.
5290 		 */
5291 		xpt_for_all_devices(xptsetasyncfunc, &csa);
5292 	}
5293 	if ((status == CAM_REQ_CMP) &&
5294 	    (csa.event_enable & AC_PATH_REGISTERED)) {
5295 		/*
5296 		 * Get this peripheral up to date with all
5297 		 * the currently existing buses.
5298 		 */
5299 		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5300 	}
5301 
5302 	return (status);
5303 }
5304 
5305 static void
5306 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5307 {
5308 	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5309 
5310 	switch (work_ccb->ccb_h.func_code) {
5311 	/* Common cases first */
5312 	case XPT_PATH_INQ:		/* Path routing inquiry */
5313 	{
5314 		struct ccb_pathinq *cpi;
5315 
5316 		cpi = &work_ccb->cpi;
5317 		cpi->version_num = 1; /* XXX??? */
5318 		cpi->hba_inquiry = 0;
5319 		cpi->target_sprt = 0;
5320 		cpi->hba_misc = 0;
5321 		cpi->hba_eng_cnt = 0;
5322 		cpi->max_target = 0;
5323 		cpi->max_lun = 0;
5324 		cpi->initiator_id = 0;
5325 		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5326 		strlcpy(cpi->hba_vid, "", HBA_IDLEN);
5327 		strlcpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5328 		cpi->unit_number = sim->unit_number;
5329 		cpi->bus_id = sim->bus_id;
5330 		cpi->base_transfer_speed = 0;
5331 		cpi->protocol = PROTO_UNSPECIFIED;
5332 		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5333 		cpi->transport = XPORT_UNSPECIFIED;
5334 		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5335 		cpi->ccb_h.status = CAM_REQ_CMP;
5336 		xpt_done(work_ccb);
5337 		break;
5338 	}
5339 	default:
5340 		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5341 		xpt_done(work_ccb);
5342 		break;
5343 	}
5344 }
5345 
5346 /*
5347  * The xpt as a "controller" has no interrupt sources, so polling
5348  * is a no-op.
5349  */
5350 static void
5351 xptpoll(struct cam_sim *sim)
5352 {
5353 }
5354 
5355 void
5356 xpt_lock_buses(void)
5357 {
5358 	mtx_lock(&xsoftc.xpt_topo_lock);
5359 }
5360 
5361 void
5362 xpt_unlock_buses(void)
5363 {
5364 	mtx_unlock(&xsoftc.xpt_topo_lock);
5365 }
5366 
5367 struct mtx *
5368 xpt_path_mtx(struct cam_path *path)
5369 {
5370 
5371 	return (&path->device->device_mtx);
5372 }
5373 
5374 static void
5375 xpt_done_process(struct ccb_hdr *ccb_h)
5376 {
5377 	struct cam_sim *sim = NULL;
5378 	struct cam_devq *devq = NULL;
5379 	struct mtx *mtx = NULL;
5380 
5381 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
5382 	struct ccb_scsiio *csio;
5383 
5384 	if (ccb_h->func_code == XPT_SCSI_IO) {
5385 		csio = &((union ccb *)ccb_h)->csio;
5386 		if (csio->bio != NULL)
5387 			biotrack(csio->bio, __func__);
5388 	}
5389 #endif
5390 
5391 	if (ccb_h->flags & CAM_HIGH_POWER) {
5392 		struct highpowerlist	*hphead;
5393 		struct cam_ed		*device;
5394 
5395 		mtx_lock(&xsoftc.xpt_highpower_lock);
5396 		hphead = &xsoftc.highpowerq;
5397 
5398 		device = STAILQ_FIRST(hphead);
5399 
5400 		/*
5401 		 * Increment the count since this command is done.
5402 		 */
5403 		xsoftc.num_highpower++;
5404 
5405 		/*
5406 		 * Any high powered commands queued up?
5407 		 */
5408 		if (device != NULL) {
5409 
5410 			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5411 			mtx_unlock(&xsoftc.xpt_highpower_lock);
5412 
5413 			mtx_lock(&device->sim->devq->send_mtx);
5414 			xpt_release_devq_device(device,
5415 					 /*count*/1, /*runqueue*/TRUE);
5416 			mtx_unlock(&device->sim->devq->send_mtx);
5417 		} else
5418 			mtx_unlock(&xsoftc.xpt_highpower_lock);
5419 	}
5420 
5421 	/*
5422 	 * Insulate against a race where the periph is destroyed
5423 	 * but CCBs are still not all processed.
5424 	 */
5425 	if (ccb_h->path->bus)
5426 		sim = ccb_h->path->bus->sim;
5427 
5428 	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5429 		KASSERT(sim, ("sim missing for CAM_RELEASE_SIMQ request"));
5430 		xpt_release_simq(sim, /*run_queue*/FALSE);
5431 		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5432 	}
5433 
5434 	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5435 	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5436 		xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5437 		ccb_h->status &= ~CAM_DEV_QFRZN;
5438 	}
5439 
5440 	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5441 		struct cam_ed *dev = ccb_h->path->device;
5442 
5443 		if (sim)
5444 			devq = sim->devq;
5445 		KASSERT(devq, ("sim missing for XPT_FC_USER_CCB request"));
5446 
5447 		mtx_lock(&devq->send_mtx);
5448 		devq->send_active--;
5449 		devq->send_openings++;
5450 		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5451 
5452 		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5453 		  && (dev->ccbq.dev_active == 0))) {
5454 			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5455 			xpt_release_devq_device(dev, /*count*/1,
5456 					 /*run_queue*/FALSE);
5457 		}
5458 
5459 		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5460 		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5461 			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5462 			xpt_release_devq_device(dev, /*count*/1,
5463 					 /*run_queue*/FALSE);
5464 		}
5465 
5466 		if (!device_is_queued(dev))
5467 			(void)xpt_schedule_devq(devq, dev);
5468 		xpt_run_devq(devq);
5469 		mtx_unlock(&devq->send_mtx);
5470 
5471 		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5472 			mtx = xpt_path_mtx(ccb_h->path);
5473 			mtx_lock(mtx);
5474 
5475 			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5476 			 && (--dev->tag_delay_count == 0))
5477 				xpt_start_tags(ccb_h->path);
5478 		}
5479 	}
5480 
5481 	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5482 		if (mtx == NULL) {
5483 			mtx = xpt_path_mtx(ccb_h->path);
5484 			mtx_lock(mtx);
5485 		}
5486 	} else {
5487 		if (mtx != NULL) {
5488 			mtx_unlock(mtx);
5489 			mtx = NULL;
5490 		}
5491 	}
5492 
5493 	/* Call the peripheral driver's callback */
5494 	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5495 	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5496 	if (mtx != NULL)
5497 		mtx_unlock(mtx);
5498 }
5499 
5500 void
5501 xpt_done_td(void *arg)
5502 {
5503 	struct cam_doneq *queue = arg;
5504 	struct ccb_hdr *ccb_h;
5505 	STAILQ_HEAD(, ccb_hdr)	doneq;
5506 
5507 	STAILQ_INIT(&doneq);
5508 	mtx_lock(&queue->cam_doneq_mtx);
5509 	while (1) {
5510 		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5511 			queue->cam_doneq_sleep = 1;
5512 			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5513 			    PRIBIO, "-", 0);
5514 			queue->cam_doneq_sleep = 0;
5515 		}
5516 		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5517 		mtx_unlock(&queue->cam_doneq_mtx);
5518 
5519 		THREAD_NO_SLEEPING();
5520 		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5521 			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5522 			xpt_done_process(ccb_h);
5523 		}
5524 		THREAD_SLEEPING_OK();
5525 
5526 		mtx_lock(&queue->cam_doneq_mtx);
5527 	}
5528 }
5529 
5530 static void
5531 camisr_runqueue(void)
5532 {
5533 	struct	ccb_hdr *ccb_h;
5534 	struct cam_doneq *queue;
5535 	int i;
5536 
5537 	/* Process global queues. */
5538 	for (i = 0; i < cam_num_doneqs; i++) {
5539 		queue = &cam_doneqs[i];
5540 		mtx_lock(&queue->cam_doneq_mtx);
5541 		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5542 			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5543 			mtx_unlock(&queue->cam_doneq_mtx);
5544 			xpt_done_process(ccb_h);
5545 			mtx_lock(&queue->cam_doneq_mtx);
5546 		}
5547 		mtx_unlock(&queue->cam_doneq_mtx);
5548 	}
5549 }
5550 
5551 struct kv
5552 {
5553 	uint32_t v;
5554 	const char *name;
5555 };
5556 
5557 static struct kv map[] = {
5558 	{ XPT_NOOP, "XPT_NOOP" },
5559 	{ XPT_SCSI_IO, "XPT_SCSI_IO" },
5560 	{ XPT_GDEV_TYPE, "XPT_GDEV_TYPE" },
5561 	{ XPT_GDEVLIST, "XPT_GDEVLIST" },
5562 	{ XPT_PATH_INQ, "XPT_PATH_INQ" },
5563 	{ XPT_REL_SIMQ, "XPT_REL_SIMQ" },
5564 	{ XPT_SASYNC_CB, "XPT_SASYNC_CB" },
5565 	{ XPT_SDEV_TYPE, "XPT_SDEV_TYPE" },
5566 	{ XPT_SCAN_BUS, "XPT_SCAN_BUS" },
5567 	{ XPT_DEV_MATCH, "XPT_DEV_MATCH" },
5568 	{ XPT_DEBUG, "XPT_DEBUG" },
5569 	{ XPT_PATH_STATS, "XPT_PATH_STATS" },
5570 	{ XPT_GDEV_STATS, "XPT_GDEV_STATS" },
5571 	{ XPT_DEV_ADVINFO, "XPT_DEV_ADVINFO" },
5572 	{ XPT_ASYNC, "XPT_ASYNC" },
5573 	{ XPT_ABORT, "XPT_ABORT" },
5574 	{ XPT_RESET_BUS, "XPT_RESET_BUS" },
5575 	{ XPT_RESET_DEV, "XPT_RESET_DEV" },
5576 	{ XPT_TERM_IO, "XPT_TERM_IO" },
5577 	{ XPT_SCAN_LUN, "XPT_SCAN_LUN" },
5578 	{ XPT_GET_TRAN_SETTINGS, "XPT_GET_TRAN_SETTINGS" },
5579 	{ XPT_SET_TRAN_SETTINGS, "XPT_SET_TRAN_SETTINGS" },
5580 	{ XPT_CALC_GEOMETRY, "XPT_CALC_GEOMETRY" },
5581 	{ XPT_ATA_IO, "XPT_ATA_IO" },
5582 	{ XPT_GET_SIM_KNOB, "XPT_GET_SIM_KNOB" },
5583 	{ XPT_SET_SIM_KNOB, "XPT_SET_SIM_KNOB" },
5584 	{ XPT_NVME_IO, "XPT_NVME_IO" },
5585 	{ XPT_MMC_IO, "XPT_MMC_IO" },
5586 	{ XPT_SMP_IO, "XPT_SMP_IO" },
5587 	{ XPT_SCAN_TGT, "XPT_SCAN_TGT" },
5588 	{ XPT_NVME_ADMIN, "XPT_NVME_ADMIN" },
5589 	{ XPT_ENG_INQ, "XPT_ENG_INQ" },
5590 	{ XPT_ENG_EXEC, "XPT_ENG_EXEC" },
5591 	{ XPT_EN_LUN, "XPT_EN_LUN" },
5592 	{ XPT_TARGET_IO, "XPT_TARGET_IO" },
5593 	{ XPT_ACCEPT_TARGET_IO, "XPT_ACCEPT_TARGET_IO" },
5594 	{ XPT_CONT_TARGET_IO, "XPT_CONT_TARGET_IO" },
5595 	{ XPT_IMMED_NOTIFY, "XPT_IMMED_NOTIFY" },
5596 	{ XPT_NOTIFY_ACK, "XPT_NOTIFY_ACK" },
5597 	{ XPT_IMMEDIATE_NOTIFY, "XPT_IMMEDIATE_NOTIFY" },
5598 	{ XPT_NOTIFY_ACKNOWLEDGE, "XPT_NOTIFY_ACKNOWLEDGE" },
5599 	{ 0, 0 }
5600 };
5601 
5602 const char *
5603 xpt_action_name(uint32_t action)
5604 {
5605 	static char buffer[32];	/* Only for unknown messages -- racy */
5606 	struct kv *walker = map;
5607 
5608 	while (walker->name != NULL) {
5609 		if (walker->v == action)
5610 			return (walker->name);
5611 		walker++;
5612 	}
5613 
5614 	snprintf(buffer, sizeof(buffer), "%#x", action);
5615 	return (buffer);
5616 }
5617