xref: /freebsd/sys/cam/cam_xpt.c (revision bb15ca603fa442c72dde3f3cb8b46db6970e3950)
1 /*-
2  * Implementation of the Common Access Method Transport (XPT) layer.
3  *
4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification, immediately at the beginning of the file.
14  * 2. The name of the author may not be used to endorse or promote products
15  *    derived from this software without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/systm.h>
36 #include <sys/types.h>
37 #include <sys/malloc.h>
38 #include <sys/kernel.h>
39 #include <sys/time.h>
40 #include <sys/conf.h>
41 #include <sys/fcntl.h>
42 #include <sys/interrupt.h>
43 #include <sys/sbuf.h>
44 #include <sys/taskqueue.h>
45 
46 #include <sys/lock.h>
47 #include <sys/mutex.h>
48 #include <sys/sysctl.h>
49 #include <sys/kthread.h>
50 
51 #include <cam/cam.h>
52 #include <cam/cam_ccb.h>
53 #include <cam/cam_periph.h>
54 #include <cam/cam_queue.h>
55 #include <cam/cam_sim.h>
56 #include <cam/cam_xpt.h>
57 #include <cam/cam_xpt_sim.h>
58 #include <cam/cam_xpt_periph.h>
59 #include <cam/cam_xpt_internal.h>
60 #include <cam/cam_debug.h>
61 
62 #include <cam/scsi/scsi_all.h>
63 #include <cam/scsi/scsi_message.h>
64 #include <cam/scsi/scsi_pass.h>
65 
66 #include <machine/md_var.h>	/* geometry translation */
67 #include <machine/stdarg.h>	/* for xpt_print below */
68 
69 #include "opt_cam.h"
70 
71 /*
72  * This is the maximum number of high powered commands (e.g. start unit)
73  * that can be outstanding at a particular time.
74  */
75 #ifndef CAM_MAX_HIGHPOWER
76 #define CAM_MAX_HIGHPOWER  4
77 #endif
78 
79 /* Datastructures internal to the xpt layer */
80 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
81 
82 /* Object for defering XPT actions to a taskqueue */
83 struct xpt_task {
84 	struct task	task;
85 	void		*data1;
86 	uintptr_t	data2;
87 };
88 
89 typedef enum {
90 	XPT_FLAG_OPEN		= 0x01
91 } xpt_flags;
92 
93 struct xpt_softc {
94 	xpt_flags		flags;
95 	u_int32_t		xpt_generation;
96 
97 	/* number of high powered commands that can go through right now */
98 	STAILQ_HEAD(highpowerlist, ccb_hdr)	highpowerq;
99 	int			num_highpower;
100 
101 	/* queue for handling async rescan requests. */
102 	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
103 	int buses_to_config;
104 	int buses_config_done;
105 
106 	/* Registered busses */
107 	TAILQ_HEAD(,cam_eb)	xpt_busses;
108 	u_int			bus_generation;
109 
110 	struct intr_config_hook	*xpt_config_hook;
111 
112 	int			boot_delay;
113 	struct callout 		boot_callout;
114 
115 	struct mtx		xpt_topo_lock;
116 	struct mtx		xpt_lock;
117 };
118 
119 typedef enum {
120 	DM_RET_COPY		= 0x01,
121 	DM_RET_FLAG_MASK	= 0x0f,
122 	DM_RET_NONE		= 0x00,
123 	DM_RET_STOP		= 0x10,
124 	DM_RET_DESCEND		= 0x20,
125 	DM_RET_ERROR		= 0x30,
126 	DM_RET_ACTION_MASK	= 0xf0
127 } dev_match_ret;
128 
129 typedef enum {
130 	XPT_DEPTH_BUS,
131 	XPT_DEPTH_TARGET,
132 	XPT_DEPTH_DEVICE,
133 	XPT_DEPTH_PERIPH
134 } xpt_traverse_depth;
135 
136 struct xpt_traverse_config {
137 	xpt_traverse_depth	depth;
138 	void			*tr_func;
139 	void			*tr_arg;
140 };
141 
142 typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
143 typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
144 typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
145 typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
146 typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
147 
148 /* Transport layer configuration information */
149 static struct xpt_softc xsoftc;
150 
151 TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
152 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
153            &xsoftc.boot_delay, 0, "Bus registration wait time");
154 
155 /* Queues for our software interrupt handler */
156 typedef TAILQ_HEAD(cam_isrq, ccb_hdr) cam_isrq_t;
157 typedef TAILQ_HEAD(cam_simq, cam_sim) cam_simq_t;
158 static cam_simq_t cam_simq;
159 static struct mtx cam_simq_lock;
160 
161 /* Pointers to software interrupt handlers */
162 static void *cambio_ih;
163 
164 struct cam_periph *xpt_periph;
165 
166 static periph_init_t xpt_periph_init;
167 
168 static struct periph_driver xpt_driver =
169 {
170 	xpt_periph_init, "xpt",
171 	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
172 	CAM_PERIPH_DRV_EARLY
173 };
174 
175 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
176 
177 static d_open_t xptopen;
178 static d_close_t xptclose;
179 static d_ioctl_t xptioctl;
180 
181 static struct cdevsw xpt_cdevsw = {
182 	.d_version =	D_VERSION,
183 	.d_flags =	0,
184 	.d_open =	xptopen,
185 	.d_close =	xptclose,
186 	.d_ioctl =	xptioctl,
187 	.d_name =	"xpt",
188 };
189 
190 /* Storage for debugging datastructures */
191 #ifdef	CAMDEBUG
192 struct cam_path *cam_dpath;
193 #ifdef	CAM_DEBUG_FLAGS
194 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
195 #else
196 u_int32_t cam_dflags = CAM_DEBUG_NONE;
197 #endif
198 TUNABLE_INT("kern.cam.dflags", &cam_dflags);
199 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
200 	&cam_dflags, 0, "Cam Debug Flags");
201 u_int32_t cam_debug_delay;
202 TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
203 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
204 	&cam_debug_delay, 0, "Cam Debug Flags");
205 #endif
206 
207 /* Our boot-time initialization hook */
208 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
209 
210 static moduledata_t cam_moduledata = {
211 	"cam",
212 	cam_module_event_handler,
213 	NULL
214 };
215 
216 static int	xpt_init(void *);
217 
218 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
219 MODULE_VERSION(cam, 1);
220 
221 
222 static void		xpt_async_bcast(struct async_list *async_head,
223 					u_int32_t async_code,
224 					struct cam_path *path,
225 					void *async_arg);
226 static path_id_t xptnextfreepathid(void);
227 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
228 static union ccb *xpt_get_ccb(struct cam_ed *device);
229 static void	 xpt_run_dev_allocq(struct cam_eb *bus);
230 static void	 xpt_run_dev_sendq(struct cam_eb *bus);
231 static timeout_t xpt_release_devq_timeout;
232 static void	 xpt_release_simq_timeout(void *arg) __unused;
233 static void	 xpt_release_bus(struct cam_eb *bus);
234 static void	 xpt_release_devq_device(struct cam_ed *dev, cam_rl rl,
235 		    u_int count, int run_queue);
236 static struct cam_et*
237 		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
238 static void	 xpt_release_target(struct cam_et *target);
239 static struct cam_eb*
240 		 xpt_find_bus(path_id_t path_id);
241 static struct cam_et*
242 		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
243 static struct cam_ed*
244 		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
245 static void	 xpt_config(void *arg);
246 static xpt_devicefunc_t xptpassannouncefunc;
247 static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
248 static void	 xptpoll(struct cam_sim *sim);
249 static void	 camisr(void *);
250 static void	 camisr_runqueue(void *);
251 static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
252 				    u_int num_patterns, struct cam_eb *bus);
253 static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
254 				       u_int num_patterns,
255 				       struct cam_ed *device);
256 static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
257 				       u_int num_patterns,
258 				       struct cam_periph *periph);
259 static xpt_busfunc_t	xptedtbusfunc;
260 static xpt_targetfunc_t	xptedttargetfunc;
261 static xpt_devicefunc_t	xptedtdevicefunc;
262 static xpt_periphfunc_t	xptedtperiphfunc;
263 static xpt_pdrvfunc_t	xptplistpdrvfunc;
264 static xpt_periphfunc_t	xptplistperiphfunc;
265 static int		xptedtmatch(struct ccb_dev_match *cdm);
266 static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
267 static int		xptbustraverse(struct cam_eb *start_bus,
268 				       xpt_busfunc_t *tr_func, void *arg);
269 static int		xpttargettraverse(struct cam_eb *bus,
270 					  struct cam_et *start_target,
271 					  xpt_targetfunc_t *tr_func, void *arg);
272 static int		xptdevicetraverse(struct cam_et *target,
273 					  struct cam_ed *start_device,
274 					  xpt_devicefunc_t *tr_func, void *arg);
275 static int		xptperiphtraverse(struct cam_ed *device,
276 					  struct cam_periph *start_periph,
277 					  xpt_periphfunc_t *tr_func, void *arg);
278 static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
279 					xpt_pdrvfunc_t *tr_func, void *arg);
280 static int		xptpdperiphtraverse(struct periph_driver **pdrv,
281 					    struct cam_periph *start_periph,
282 					    xpt_periphfunc_t *tr_func,
283 					    void *arg);
284 static xpt_busfunc_t	xptdefbusfunc;
285 static xpt_targetfunc_t	xptdeftargetfunc;
286 static xpt_devicefunc_t	xptdefdevicefunc;
287 static xpt_periphfunc_t	xptdefperiphfunc;
288 static void		xpt_finishconfig_task(void *context, int pending);
289 static void		xpt_dev_async_default(u_int32_t async_code,
290 					      struct cam_eb *bus,
291 					      struct cam_et *target,
292 					      struct cam_ed *device,
293 					      void *async_arg);
294 static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
295 						 struct cam_et *target,
296 						 lun_id_t lun_id);
297 static xpt_devicefunc_t	xptsetasyncfunc;
298 static xpt_busfunc_t	xptsetasyncbusfunc;
299 static cam_status	xptregister(struct cam_periph *periph,
300 				    void *arg);
301 static __inline int periph_is_queued(struct cam_periph *periph);
302 static __inline int device_is_alloc_queued(struct cam_ed *device);
303 static __inline int device_is_send_queued(struct cam_ed *device);
304 
305 static __inline int
306 xpt_schedule_dev_allocq(struct cam_eb *bus, struct cam_ed *dev)
307 {
308 	int retval;
309 
310 	if ((dev->drvq.entries > 0) &&
311 	    (dev->ccbq.devq_openings > 0) &&
312 	    (cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
313 		CAMQ_GET_PRIO(&dev->drvq))) == 0)) {
314 		/*
315 		 * The priority of a device waiting for CCB resources
316 		 * is that of the highest priority peripheral driver
317 		 * enqueued.
318 		 */
319 		retval = xpt_schedule_dev(&bus->sim->devq->alloc_queue,
320 					  &dev->alloc_ccb_entry.pinfo,
321 					  CAMQ_GET_PRIO(&dev->drvq));
322 	} else {
323 		retval = 0;
324 	}
325 
326 	return (retval);
327 }
328 
329 static __inline int
330 xpt_schedule_dev_sendq(struct cam_eb *bus, struct cam_ed *dev)
331 {
332 	int	retval;
333 
334 	if ((dev->ccbq.queue.entries > 0) &&
335 	    (dev->ccbq.dev_openings > 0) &&
336 	    (cam_ccbq_frozen_top(&dev->ccbq) == 0)) {
337 		/*
338 		 * The priority of a device waiting for controller
339 		 * resources is that of the highest priority CCB
340 		 * enqueued.
341 		 */
342 		retval =
343 		    xpt_schedule_dev(&bus->sim->devq->send_queue,
344 				     &dev->send_ccb_entry.pinfo,
345 				     CAMQ_GET_PRIO(&dev->ccbq.queue));
346 	} else {
347 		retval = 0;
348 	}
349 	return (retval);
350 }
351 
352 static __inline int
353 periph_is_queued(struct cam_periph *periph)
354 {
355 	return (periph->pinfo.index != CAM_UNQUEUED_INDEX);
356 }
357 
358 static __inline int
359 device_is_alloc_queued(struct cam_ed *device)
360 {
361 	return (device->alloc_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX);
362 }
363 
364 static __inline int
365 device_is_send_queued(struct cam_ed *device)
366 {
367 	return (device->send_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX);
368 }
369 
370 static void
371 xpt_periph_init()
372 {
373 	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
374 }
375 
376 static void
377 xptdone(struct cam_periph *periph, union ccb *done_ccb)
378 {
379 	/* Caller will release the CCB */
380 	wakeup(&done_ccb->ccb_h.cbfcnp);
381 }
382 
383 static int
384 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
385 {
386 
387 	/*
388 	 * Only allow read-write access.
389 	 */
390 	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
391 		return(EPERM);
392 
393 	/*
394 	 * We don't allow nonblocking access.
395 	 */
396 	if ((flags & O_NONBLOCK) != 0) {
397 		printf("%s: can't do nonblocking access\n", devtoname(dev));
398 		return(ENODEV);
399 	}
400 
401 	/* Mark ourselves open */
402 	mtx_lock(&xsoftc.xpt_lock);
403 	xsoftc.flags |= XPT_FLAG_OPEN;
404 	mtx_unlock(&xsoftc.xpt_lock);
405 
406 	return(0);
407 }
408 
409 static int
410 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
411 {
412 
413 	/* Mark ourselves closed */
414 	mtx_lock(&xsoftc.xpt_lock);
415 	xsoftc.flags &= ~XPT_FLAG_OPEN;
416 	mtx_unlock(&xsoftc.xpt_lock);
417 
418 	return(0);
419 }
420 
421 /*
422  * Don't automatically grab the xpt softc lock here even though this is going
423  * through the xpt device.  The xpt device is really just a back door for
424  * accessing other devices and SIMs, so the right thing to do is to grab
425  * the appropriate SIM lock once the bus/SIM is located.
426  */
427 static int
428 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
429 {
430 	int error;
431 
432 	error = 0;
433 
434 	switch(cmd) {
435 	/*
436 	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
437 	 * to accept CCB types that don't quite make sense to send through a
438 	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
439 	 * in the CAM spec.
440 	 */
441 	case CAMIOCOMMAND: {
442 		union ccb *ccb;
443 		union ccb *inccb;
444 		struct cam_eb *bus;
445 
446 		inccb = (union ccb *)addr;
447 
448 		bus = xpt_find_bus(inccb->ccb_h.path_id);
449 		if (bus == NULL)
450 			return (EINVAL);
451 
452 		switch (inccb->ccb_h.func_code) {
453 		case XPT_SCAN_BUS:
454 		case XPT_RESET_BUS:
455 			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
456 			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
457 				xpt_release_bus(bus);
458 				return (EINVAL);
459 			}
460 			break;
461 		case XPT_SCAN_TGT:
462 			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
463 			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
464 				xpt_release_bus(bus);
465 				return (EINVAL);
466 			}
467 			break;
468 		default:
469 			break;
470 		}
471 
472 		switch(inccb->ccb_h.func_code) {
473 		case XPT_SCAN_BUS:
474 		case XPT_RESET_BUS:
475 		case XPT_PATH_INQ:
476 		case XPT_ENG_INQ:
477 		case XPT_SCAN_LUN:
478 		case XPT_SCAN_TGT:
479 
480 			ccb = xpt_alloc_ccb();
481 
482 			CAM_SIM_LOCK(bus->sim);
483 
484 			/*
485 			 * Create a path using the bus, target, and lun the
486 			 * user passed in.
487 			 */
488 			if (xpt_create_path(&ccb->ccb_h.path, xpt_periph,
489 					    inccb->ccb_h.path_id,
490 					    inccb->ccb_h.target_id,
491 					    inccb->ccb_h.target_lun) !=
492 					    CAM_REQ_CMP){
493 				error = EINVAL;
494 				CAM_SIM_UNLOCK(bus->sim);
495 				xpt_free_ccb(ccb);
496 				break;
497 			}
498 			/* Ensure all of our fields are correct */
499 			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
500 				      inccb->ccb_h.pinfo.priority);
501 			xpt_merge_ccb(ccb, inccb);
502 			ccb->ccb_h.cbfcnp = xptdone;
503 			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
504 			bcopy(ccb, inccb, sizeof(union ccb));
505 			xpt_free_path(ccb->ccb_h.path);
506 			xpt_free_ccb(ccb);
507 			CAM_SIM_UNLOCK(bus->sim);
508 			break;
509 
510 		case XPT_DEBUG: {
511 			union ccb ccb;
512 
513 			/*
514 			 * This is an immediate CCB, so it's okay to
515 			 * allocate it on the stack.
516 			 */
517 
518 			CAM_SIM_LOCK(bus->sim);
519 
520 			/*
521 			 * Create a path using the bus, target, and lun the
522 			 * user passed in.
523 			 */
524 			if (xpt_create_path(&ccb.ccb_h.path, xpt_periph,
525 					    inccb->ccb_h.path_id,
526 					    inccb->ccb_h.target_id,
527 					    inccb->ccb_h.target_lun) !=
528 					    CAM_REQ_CMP){
529 				error = EINVAL;
530 				CAM_SIM_UNLOCK(bus->sim);
531 				break;
532 			}
533 			/* Ensure all of our fields are correct */
534 			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
535 				      inccb->ccb_h.pinfo.priority);
536 			xpt_merge_ccb(&ccb, inccb);
537 			ccb.ccb_h.cbfcnp = xptdone;
538 			xpt_action(&ccb);
539 			CAM_SIM_UNLOCK(bus->sim);
540 			bcopy(&ccb, inccb, sizeof(union ccb));
541 			xpt_free_path(ccb.ccb_h.path);
542 			break;
543 
544 		}
545 		case XPT_DEV_MATCH: {
546 			struct cam_periph_map_info mapinfo;
547 			struct cam_path *old_path;
548 
549 			/*
550 			 * We can't deal with physical addresses for this
551 			 * type of transaction.
552 			 */
553 			if (inccb->ccb_h.flags & CAM_DATA_PHYS) {
554 				error = EINVAL;
555 				break;
556 			}
557 
558 			/*
559 			 * Save this in case the caller had it set to
560 			 * something in particular.
561 			 */
562 			old_path = inccb->ccb_h.path;
563 
564 			/*
565 			 * We really don't need a path for the matching
566 			 * code.  The path is needed because of the
567 			 * debugging statements in xpt_action().  They
568 			 * assume that the CCB has a valid path.
569 			 */
570 			inccb->ccb_h.path = xpt_periph->path;
571 
572 			bzero(&mapinfo, sizeof(mapinfo));
573 
574 			/*
575 			 * Map the pattern and match buffers into kernel
576 			 * virtual address space.
577 			 */
578 			error = cam_periph_mapmem(inccb, &mapinfo);
579 
580 			if (error) {
581 				inccb->ccb_h.path = old_path;
582 				break;
583 			}
584 
585 			/*
586 			 * This is an immediate CCB, we can send it on directly.
587 			 */
588 			xpt_action(inccb);
589 
590 			/*
591 			 * Map the buffers back into user space.
592 			 */
593 			cam_periph_unmapmem(inccb, &mapinfo);
594 
595 			inccb->ccb_h.path = old_path;
596 
597 			error = 0;
598 			break;
599 		}
600 		default:
601 			error = ENOTSUP;
602 			break;
603 		}
604 		xpt_release_bus(bus);
605 		break;
606 	}
607 	/*
608 	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
609 	 * with the periphal driver name and unit name filled in.  The other
610 	 * fields don't really matter as input.  The passthrough driver name
611 	 * ("pass"), and unit number are passed back in the ccb.  The current
612 	 * device generation number, and the index into the device peripheral
613 	 * driver list, and the status are also passed back.  Note that
614 	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
615 	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
616 	 * (or rather should be) impossible for the device peripheral driver
617 	 * list to change since we look at the whole thing in one pass, and
618 	 * we do it with lock protection.
619 	 *
620 	 */
621 	case CAMGETPASSTHRU: {
622 		union ccb *ccb;
623 		struct cam_periph *periph;
624 		struct periph_driver **p_drv;
625 		char   *name;
626 		u_int unit;
627 		u_int cur_generation;
628 		int base_periph_found;
629 		int splbreaknum;
630 
631 		ccb = (union ccb *)addr;
632 		unit = ccb->cgdl.unit_number;
633 		name = ccb->cgdl.periph_name;
634 		/*
635 		 * Every 100 devices, we want to drop our lock protection to
636 		 * give the software interrupt handler a chance to run.
637 		 * Most systems won't run into this check, but this should
638 		 * avoid starvation in the software interrupt handler in
639 		 * large systems.
640 		 */
641 		splbreaknum = 100;
642 
643 		ccb = (union ccb *)addr;
644 
645 		base_periph_found = 0;
646 
647 		/*
648 		 * Sanity check -- make sure we don't get a null peripheral
649 		 * driver name.
650 		 */
651 		if (*ccb->cgdl.periph_name == '\0') {
652 			error = EINVAL;
653 			break;
654 		}
655 
656 		/* Keep the list from changing while we traverse it */
657 		mtx_lock(&xsoftc.xpt_topo_lock);
658 ptstartover:
659 		cur_generation = xsoftc.xpt_generation;
660 
661 		/* first find our driver in the list of drivers */
662 		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
663 			if (strcmp((*p_drv)->driver_name, name) == 0)
664 				break;
665 
666 		if (*p_drv == NULL) {
667 			mtx_unlock(&xsoftc.xpt_topo_lock);
668 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
669 			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
670 			*ccb->cgdl.periph_name = '\0';
671 			ccb->cgdl.unit_number = 0;
672 			error = ENOENT;
673 			break;
674 		}
675 
676 		/*
677 		 * Run through every peripheral instance of this driver
678 		 * and check to see whether it matches the unit passed
679 		 * in by the user.  If it does, get out of the loops and
680 		 * find the passthrough driver associated with that
681 		 * peripheral driver.
682 		 */
683 		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
684 		     periph = TAILQ_NEXT(periph, unit_links)) {
685 
686 			if (periph->unit_number == unit) {
687 				break;
688 			} else if (--splbreaknum == 0) {
689 				mtx_unlock(&xsoftc.xpt_topo_lock);
690 				mtx_lock(&xsoftc.xpt_topo_lock);
691 				splbreaknum = 100;
692 				if (cur_generation != xsoftc.xpt_generation)
693 				       goto ptstartover;
694 			}
695 		}
696 		/*
697 		 * If we found the peripheral driver that the user passed
698 		 * in, go through all of the peripheral drivers for that
699 		 * particular device and look for a passthrough driver.
700 		 */
701 		if (periph != NULL) {
702 			struct cam_ed *device;
703 			int i;
704 
705 			base_periph_found = 1;
706 			device = periph->path->device;
707 			for (i = 0, periph = SLIST_FIRST(&device->periphs);
708 			     periph != NULL;
709 			     periph = SLIST_NEXT(periph, periph_links), i++) {
710 				/*
711 				 * Check to see whether we have a
712 				 * passthrough device or not.
713 				 */
714 				if (strcmp(periph->periph_name, "pass") == 0) {
715 					/*
716 					 * Fill in the getdevlist fields.
717 					 */
718 					strcpy(ccb->cgdl.periph_name,
719 					       periph->periph_name);
720 					ccb->cgdl.unit_number =
721 						periph->unit_number;
722 					if (SLIST_NEXT(periph, periph_links))
723 						ccb->cgdl.status =
724 							CAM_GDEVLIST_MORE_DEVS;
725 					else
726 						ccb->cgdl.status =
727 						       CAM_GDEVLIST_LAST_DEVICE;
728 					ccb->cgdl.generation =
729 						device->generation;
730 					ccb->cgdl.index = i;
731 					/*
732 					 * Fill in some CCB header fields
733 					 * that the user may want.
734 					 */
735 					ccb->ccb_h.path_id =
736 						periph->path->bus->path_id;
737 					ccb->ccb_h.target_id =
738 						periph->path->target->target_id;
739 					ccb->ccb_h.target_lun =
740 						periph->path->device->lun_id;
741 					ccb->ccb_h.status = CAM_REQ_CMP;
742 					break;
743 				}
744 			}
745 		}
746 
747 		/*
748 		 * If the periph is null here, one of two things has
749 		 * happened.  The first possibility is that we couldn't
750 		 * find the unit number of the particular peripheral driver
751 		 * that the user is asking about.  e.g. the user asks for
752 		 * the passthrough driver for "da11".  We find the list of
753 		 * "da" peripherals all right, but there is no unit 11.
754 		 * The other possibility is that we went through the list
755 		 * of peripheral drivers attached to the device structure,
756 		 * but didn't find one with the name "pass".  Either way,
757 		 * we return ENOENT, since we couldn't find something.
758 		 */
759 		if (periph == NULL) {
760 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
761 			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
762 			*ccb->cgdl.periph_name = '\0';
763 			ccb->cgdl.unit_number = 0;
764 			error = ENOENT;
765 			/*
766 			 * It is unfortunate that this is even necessary,
767 			 * but there are many, many clueless users out there.
768 			 * If this is true, the user is looking for the
769 			 * passthrough driver, but doesn't have one in his
770 			 * kernel.
771 			 */
772 			if (base_periph_found == 1) {
773 				printf("xptioctl: pass driver is not in the "
774 				       "kernel\n");
775 				printf("xptioctl: put \"device pass\" in "
776 				       "your kernel config file\n");
777 			}
778 		}
779 		mtx_unlock(&xsoftc.xpt_topo_lock);
780 		break;
781 		}
782 	default:
783 		error = ENOTTY;
784 		break;
785 	}
786 
787 	return(error);
788 }
789 
790 static int
791 cam_module_event_handler(module_t mod, int what, void *arg)
792 {
793 	int error;
794 
795 	switch (what) {
796 	case MOD_LOAD:
797 		if ((error = xpt_init(NULL)) != 0)
798 			return (error);
799 		break;
800 	case MOD_UNLOAD:
801 		return EBUSY;
802 	default:
803 		return EOPNOTSUPP;
804 	}
805 
806 	return 0;
807 }
808 
809 static void
810 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
811 {
812 
813 	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
814 		xpt_free_path(done_ccb->ccb_h.path);
815 		xpt_free_ccb(done_ccb);
816 	} else {
817 		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
818 		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
819 	}
820 	xpt_release_boot();
821 }
822 
823 /* thread to handle bus rescans */
824 static void
825 xpt_scanner_thread(void *dummy)
826 {
827 	union ccb	*ccb;
828 	struct cam_sim	*sim;
829 
830 	xpt_lock_buses();
831 	for (;;) {
832 		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
833 			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
834 			       "ccb_scanq", 0);
835 		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
836 			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
837 			xpt_unlock_buses();
838 
839 			sim = ccb->ccb_h.path->bus->sim;
840 			CAM_SIM_LOCK(sim);
841 			xpt_action(ccb);
842 			CAM_SIM_UNLOCK(sim);
843 
844 			xpt_lock_buses();
845 		}
846 	}
847 }
848 
849 void
850 xpt_rescan(union ccb *ccb)
851 {
852 	struct ccb_hdr *hdr;
853 
854 	/* Prepare request */
855 	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
856 	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
857 		ccb->ccb_h.func_code = XPT_SCAN_BUS;
858 	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
859 	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
860 		ccb->ccb_h.func_code = XPT_SCAN_TGT;
861 	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
862 	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
863 		ccb->ccb_h.func_code = XPT_SCAN_LUN;
864 	else {
865 		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
866 		xpt_free_path(ccb->ccb_h.path);
867 		xpt_free_ccb(ccb);
868 		return;
869 	}
870 	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
871 	ccb->ccb_h.cbfcnp = xpt_rescan_done;
872 	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
873 	/* Don't make duplicate entries for the same paths. */
874 	xpt_lock_buses();
875 	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
876 		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
877 			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
878 				wakeup(&xsoftc.ccb_scanq);
879 				xpt_unlock_buses();
880 				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
881 				xpt_free_path(ccb->ccb_h.path);
882 				xpt_free_ccb(ccb);
883 				return;
884 			}
885 		}
886 	}
887 	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
888 	xsoftc.buses_to_config++;
889 	wakeup(&xsoftc.ccb_scanq);
890 	xpt_unlock_buses();
891 }
892 
893 /* Functions accessed by the peripheral drivers */
894 static int
895 xpt_init(void *dummy)
896 {
897 	struct cam_sim *xpt_sim;
898 	struct cam_path *path;
899 	struct cam_devq *devq;
900 	cam_status status;
901 
902 	TAILQ_INIT(&xsoftc.xpt_busses);
903 	TAILQ_INIT(&cam_simq);
904 	TAILQ_INIT(&xsoftc.ccb_scanq);
905 	STAILQ_INIT(&xsoftc.highpowerq);
906 	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
907 
908 	mtx_init(&cam_simq_lock, "CAM SIMQ lock", NULL, MTX_DEF);
909 	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
910 	mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF);
911 
912 	/*
913 	 * The xpt layer is, itself, the equivelent of a SIM.
914 	 * Allow 16 ccbs in the ccb pool for it.  This should
915 	 * give decent parallelism when we probe busses and
916 	 * perform other XPT functions.
917 	 */
918 	devq = cam_simq_alloc(16);
919 	xpt_sim = cam_sim_alloc(xptaction,
920 				xptpoll,
921 				"xpt",
922 				/*softc*/NULL,
923 				/*unit*/0,
924 				/*mtx*/&xsoftc.xpt_lock,
925 				/*max_dev_transactions*/0,
926 				/*max_tagged_dev_transactions*/0,
927 				devq);
928 	if (xpt_sim == NULL)
929 		return (ENOMEM);
930 
931 	mtx_lock(&xsoftc.xpt_lock);
932 	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
933 		mtx_unlock(&xsoftc.xpt_lock);
934 		printf("xpt_init: xpt_bus_register failed with status %#x,"
935 		       " failing attach\n", status);
936 		return (EINVAL);
937 	}
938 
939 	/*
940 	 * Looking at the XPT from the SIM layer, the XPT is
941 	 * the equivelent of a peripheral driver.  Allocate
942 	 * a peripheral driver entry for us.
943 	 */
944 	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
945 				      CAM_TARGET_WILDCARD,
946 				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
947 		mtx_unlock(&xsoftc.xpt_lock);
948 		printf("xpt_init: xpt_create_path failed with status %#x,"
949 		       " failing attach\n", status);
950 		return (EINVAL);
951 	}
952 
953 	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
954 			 path, NULL, 0, xpt_sim);
955 	xpt_free_path(path);
956 	mtx_unlock(&xsoftc.xpt_lock);
957 	/* Install our software interrupt handlers */
958 	swi_add(NULL, "cambio", camisr, NULL, SWI_CAMBIO, INTR_MPSAFE, &cambio_ih);
959 	/*
960 	 * Register a callback for when interrupts are enabled.
961 	 */
962 	xsoftc.xpt_config_hook =
963 	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
964 					      M_CAMXPT, M_NOWAIT | M_ZERO);
965 	if (xsoftc.xpt_config_hook == NULL) {
966 		printf("xpt_init: Cannot malloc config hook "
967 		       "- failing attach\n");
968 		return (ENOMEM);
969 	}
970 	xsoftc.xpt_config_hook->ich_func = xpt_config;
971 	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
972 		free (xsoftc.xpt_config_hook, M_CAMXPT);
973 		printf("xpt_init: config_intrhook_establish failed "
974 		       "- failing attach\n");
975 	}
976 
977 	return (0);
978 }
979 
980 static cam_status
981 xptregister(struct cam_periph *periph, void *arg)
982 {
983 	struct cam_sim *xpt_sim;
984 
985 	if (periph == NULL) {
986 		printf("xptregister: periph was NULL!!\n");
987 		return(CAM_REQ_CMP_ERR);
988 	}
989 
990 	xpt_sim = (struct cam_sim *)arg;
991 	xpt_sim->softc = periph;
992 	xpt_periph = periph;
993 	periph->softc = NULL;
994 
995 	return(CAM_REQ_CMP);
996 }
997 
998 int32_t
999 xpt_add_periph(struct cam_periph *periph)
1000 {
1001 	struct cam_ed *device;
1002 	int32_t	 status;
1003 	struct periph_list *periph_head;
1004 
1005 	mtx_assert(periph->sim->mtx, MA_OWNED);
1006 
1007 	device = periph->path->device;
1008 
1009 	periph_head = &device->periphs;
1010 
1011 	status = CAM_REQ_CMP;
1012 
1013 	if (device != NULL) {
1014 		/*
1015 		 * Make room for this peripheral
1016 		 * so it will fit in the queue
1017 		 * when it's scheduled to run
1018 		 */
1019 		status = camq_resize(&device->drvq,
1020 				     device->drvq.array_size + 1);
1021 
1022 		device->generation++;
1023 
1024 		SLIST_INSERT_HEAD(periph_head, periph, periph_links);
1025 	}
1026 
1027 	mtx_lock(&xsoftc.xpt_topo_lock);
1028 	xsoftc.xpt_generation++;
1029 	mtx_unlock(&xsoftc.xpt_topo_lock);
1030 
1031 	return (status);
1032 }
1033 
1034 void
1035 xpt_remove_periph(struct cam_periph *periph)
1036 {
1037 	struct cam_ed *device;
1038 
1039 	mtx_assert(periph->sim->mtx, MA_OWNED);
1040 
1041 	device = periph->path->device;
1042 
1043 	if (device != NULL) {
1044 		struct periph_list *periph_head;
1045 
1046 		periph_head = &device->periphs;
1047 
1048 		/* Release the slot for this peripheral */
1049 		camq_resize(&device->drvq, device->drvq.array_size - 1);
1050 
1051 		device->generation++;
1052 
1053 		SLIST_REMOVE(periph_head, periph, cam_periph, periph_links);
1054 	}
1055 
1056 	mtx_lock(&xsoftc.xpt_topo_lock);
1057 	xsoftc.xpt_generation++;
1058 	mtx_unlock(&xsoftc.xpt_topo_lock);
1059 }
1060 
1061 
1062 void
1063 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1064 {
1065 	struct	cam_path *path = periph->path;
1066 
1067 	mtx_assert(periph->sim->mtx, MA_OWNED);
1068 
1069 	printf("%s%d at %s%d bus %d scbus%d target %d lun %d\n",
1070 	       periph->periph_name, periph->unit_number,
1071 	       path->bus->sim->sim_name,
1072 	       path->bus->sim->unit_number,
1073 	       path->bus->sim->bus_id,
1074 	       path->bus->path_id,
1075 	       path->target->target_id,
1076 	       path->device->lun_id);
1077 	printf("%s%d: ", periph->periph_name, periph->unit_number);
1078 	if (path->device->protocol == PROTO_SCSI)
1079 		scsi_print_inquiry(&path->device->inq_data);
1080 	else if (path->device->protocol == PROTO_ATA ||
1081 	    path->device->protocol == PROTO_SATAPM)
1082 		ata_print_ident(&path->device->ident_data);
1083 	else
1084 		printf("Unknown protocol device\n");
1085 	if (bootverbose && path->device->serial_num_len > 0) {
1086 		/* Don't wrap the screen  - print only the first 60 chars */
1087 		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1088 		       periph->unit_number, path->device->serial_num);
1089 	}
1090 	/* Announce transport details. */
1091 	(*(path->bus->xport->announce))(periph);
1092 	/* Announce command queueing. */
1093 	if (path->device->inq_flags & SID_CmdQue
1094 	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1095 		printf("%s%d: Command Queueing enabled\n",
1096 		       periph->periph_name, periph->unit_number);
1097 	}
1098 	/* Announce caller's details if they've passed in. */
1099 	if (announce_string != NULL)
1100 		printf("%s%d: %s\n", periph->periph_name,
1101 		       periph->unit_number, announce_string);
1102 }
1103 
1104 int
1105 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1106 {
1107 	int ret = -1;
1108 	struct ccb_dev_advinfo cdai;
1109 
1110 	memset(&cdai, 0, sizeof(cdai));
1111 	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1112 	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1113 	cdai.bufsiz = len;
1114 
1115 	if (!strcmp(attr, "GEOM::ident"))
1116 		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1117 	else if (!strcmp(attr, "GEOM::physpath"))
1118 		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1119 	else
1120 		goto out;
1121 
1122 	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1123 	if (cdai.buf == NULL) {
1124 		ret = ENOMEM;
1125 		goto out;
1126 	}
1127 	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1128 	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1129 		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1130 	if (cdai.provsiz == 0)
1131 		goto out;
1132 	ret = 0;
1133 	if (strlcpy(buf, cdai.buf, len) >= len)
1134 		ret = EFAULT;
1135 
1136 out:
1137 	if (cdai.buf != NULL)
1138 		free(cdai.buf, M_CAMXPT);
1139 	return ret;
1140 }
1141 
1142 static dev_match_ret
1143 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1144 	    struct cam_eb *bus)
1145 {
1146 	dev_match_ret retval;
1147 	int i;
1148 
1149 	retval = DM_RET_NONE;
1150 
1151 	/*
1152 	 * If we aren't given something to match against, that's an error.
1153 	 */
1154 	if (bus == NULL)
1155 		return(DM_RET_ERROR);
1156 
1157 	/*
1158 	 * If there are no match entries, then this bus matches no
1159 	 * matter what.
1160 	 */
1161 	if ((patterns == NULL) || (num_patterns == 0))
1162 		return(DM_RET_DESCEND | DM_RET_COPY);
1163 
1164 	for (i = 0; i < num_patterns; i++) {
1165 		struct bus_match_pattern *cur_pattern;
1166 
1167 		/*
1168 		 * If the pattern in question isn't for a bus node, we
1169 		 * aren't interested.  However, we do indicate to the
1170 		 * calling routine that we should continue descending the
1171 		 * tree, since the user wants to match against lower-level
1172 		 * EDT elements.
1173 		 */
1174 		if (patterns[i].type != DEV_MATCH_BUS) {
1175 			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1176 				retval |= DM_RET_DESCEND;
1177 			continue;
1178 		}
1179 
1180 		cur_pattern = &patterns[i].pattern.bus_pattern;
1181 
1182 		/*
1183 		 * If they want to match any bus node, we give them any
1184 		 * device node.
1185 		 */
1186 		if (cur_pattern->flags == BUS_MATCH_ANY) {
1187 			/* set the copy flag */
1188 			retval |= DM_RET_COPY;
1189 
1190 			/*
1191 			 * If we've already decided on an action, go ahead
1192 			 * and return.
1193 			 */
1194 			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1195 				return(retval);
1196 		}
1197 
1198 		/*
1199 		 * Not sure why someone would do this...
1200 		 */
1201 		if (cur_pattern->flags == BUS_MATCH_NONE)
1202 			continue;
1203 
1204 		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1205 		 && (cur_pattern->path_id != bus->path_id))
1206 			continue;
1207 
1208 		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1209 		 && (cur_pattern->bus_id != bus->sim->bus_id))
1210 			continue;
1211 
1212 		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1213 		 && (cur_pattern->unit_number != bus->sim->unit_number))
1214 			continue;
1215 
1216 		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1217 		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1218 			     DEV_IDLEN) != 0))
1219 			continue;
1220 
1221 		/*
1222 		 * If we get to this point, the user definitely wants
1223 		 * information on this bus.  So tell the caller to copy the
1224 		 * data out.
1225 		 */
1226 		retval |= DM_RET_COPY;
1227 
1228 		/*
1229 		 * If the return action has been set to descend, then we
1230 		 * know that we've already seen a non-bus matching
1231 		 * expression, therefore we need to further descend the tree.
1232 		 * This won't change by continuing around the loop, so we
1233 		 * go ahead and return.  If we haven't seen a non-bus
1234 		 * matching expression, we keep going around the loop until
1235 		 * we exhaust the matching expressions.  We'll set the stop
1236 		 * flag once we fall out of the loop.
1237 		 */
1238 		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1239 			return(retval);
1240 	}
1241 
1242 	/*
1243 	 * If the return action hasn't been set to descend yet, that means
1244 	 * we haven't seen anything other than bus matching patterns.  So
1245 	 * tell the caller to stop descending the tree -- the user doesn't
1246 	 * want to match against lower level tree elements.
1247 	 */
1248 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1249 		retval |= DM_RET_STOP;
1250 
1251 	return(retval);
1252 }
1253 
1254 static dev_match_ret
1255 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1256 	       struct cam_ed *device)
1257 {
1258 	dev_match_ret retval;
1259 	int i;
1260 
1261 	retval = DM_RET_NONE;
1262 
1263 	/*
1264 	 * If we aren't given something to match against, that's an error.
1265 	 */
1266 	if (device == NULL)
1267 		return(DM_RET_ERROR);
1268 
1269 	/*
1270 	 * If there are no match entries, then this device matches no
1271 	 * matter what.
1272 	 */
1273 	if ((patterns == NULL) || (num_patterns == 0))
1274 		return(DM_RET_DESCEND | DM_RET_COPY);
1275 
1276 	for (i = 0; i < num_patterns; i++) {
1277 		struct device_match_pattern *cur_pattern;
1278 		struct scsi_vpd_device_id *device_id_page;
1279 
1280 		/*
1281 		 * If the pattern in question isn't for a device node, we
1282 		 * aren't interested.
1283 		 */
1284 		if (patterns[i].type != DEV_MATCH_DEVICE) {
1285 			if ((patterns[i].type == DEV_MATCH_PERIPH)
1286 			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1287 				retval |= DM_RET_DESCEND;
1288 			continue;
1289 		}
1290 
1291 		cur_pattern = &patterns[i].pattern.device_pattern;
1292 
1293 		/* Error out if mutually exclusive options are specified. */
1294 		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1295 		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1296 			return(DM_RET_ERROR);
1297 
1298 		/*
1299 		 * If they want to match any device node, we give them any
1300 		 * device node.
1301 		 */
1302 		if (cur_pattern->flags == DEV_MATCH_ANY)
1303 			goto copy_dev_node;
1304 
1305 		/*
1306 		 * Not sure why someone would do this...
1307 		 */
1308 		if (cur_pattern->flags == DEV_MATCH_NONE)
1309 			continue;
1310 
1311 		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1312 		 && (cur_pattern->path_id != device->target->bus->path_id))
1313 			continue;
1314 
1315 		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1316 		 && (cur_pattern->target_id != device->target->target_id))
1317 			continue;
1318 
1319 		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1320 		 && (cur_pattern->target_lun != device->lun_id))
1321 			continue;
1322 
1323 		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1324 		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1325 				    (caddr_t)&cur_pattern->data.inq_pat,
1326 				    1, sizeof(cur_pattern->data.inq_pat),
1327 				    scsi_static_inquiry_match) == NULL))
1328 			continue;
1329 
1330 		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1331 		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1332 		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1333 		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1334 				      device->device_id_len
1335 				    - SVPD_DEVICE_ID_HDR_LEN,
1336 				      cur_pattern->data.devid_pat.id,
1337 				      cur_pattern->data.devid_pat.id_len) != 0))
1338 			continue;
1339 
1340 copy_dev_node:
1341 		/*
1342 		 * If we get to this point, the user definitely wants
1343 		 * information on this device.  So tell the caller to copy
1344 		 * the data out.
1345 		 */
1346 		retval |= DM_RET_COPY;
1347 
1348 		/*
1349 		 * If the return action has been set to descend, then we
1350 		 * know that we've already seen a peripheral matching
1351 		 * expression, therefore we need to further descend the tree.
1352 		 * This won't change by continuing around the loop, so we
1353 		 * go ahead and return.  If we haven't seen a peripheral
1354 		 * matching expression, we keep going around the loop until
1355 		 * we exhaust the matching expressions.  We'll set the stop
1356 		 * flag once we fall out of the loop.
1357 		 */
1358 		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1359 			return(retval);
1360 	}
1361 
1362 	/*
1363 	 * If the return action hasn't been set to descend yet, that means
1364 	 * we haven't seen any peripheral matching patterns.  So tell the
1365 	 * caller to stop descending the tree -- the user doesn't want to
1366 	 * match against lower level tree elements.
1367 	 */
1368 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1369 		retval |= DM_RET_STOP;
1370 
1371 	return(retval);
1372 }
1373 
1374 /*
1375  * Match a single peripheral against any number of match patterns.
1376  */
1377 static dev_match_ret
1378 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1379 	       struct cam_periph *periph)
1380 {
1381 	dev_match_ret retval;
1382 	int i;
1383 
1384 	/*
1385 	 * If we aren't given something to match against, that's an error.
1386 	 */
1387 	if (periph == NULL)
1388 		return(DM_RET_ERROR);
1389 
1390 	/*
1391 	 * If there are no match entries, then this peripheral matches no
1392 	 * matter what.
1393 	 */
1394 	if ((patterns == NULL) || (num_patterns == 0))
1395 		return(DM_RET_STOP | DM_RET_COPY);
1396 
1397 	/*
1398 	 * There aren't any nodes below a peripheral node, so there's no
1399 	 * reason to descend the tree any further.
1400 	 */
1401 	retval = DM_RET_STOP;
1402 
1403 	for (i = 0; i < num_patterns; i++) {
1404 		struct periph_match_pattern *cur_pattern;
1405 
1406 		/*
1407 		 * If the pattern in question isn't for a peripheral, we
1408 		 * aren't interested.
1409 		 */
1410 		if (patterns[i].type != DEV_MATCH_PERIPH)
1411 			continue;
1412 
1413 		cur_pattern = &patterns[i].pattern.periph_pattern;
1414 
1415 		/*
1416 		 * If they want to match on anything, then we will do so.
1417 		 */
1418 		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1419 			/* set the copy flag */
1420 			retval |= DM_RET_COPY;
1421 
1422 			/*
1423 			 * We've already set the return action to stop,
1424 			 * since there are no nodes below peripherals in
1425 			 * the tree.
1426 			 */
1427 			return(retval);
1428 		}
1429 
1430 		/*
1431 		 * Not sure why someone would do this...
1432 		 */
1433 		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1434 			continue;
1435 
1436 		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1437 		 && (cur_pattern->path_id != periph->path->bus->path_id))
1438 			continue;
1439 
1440 		/*
1441 		 * For the target and lun id's, we have to make sure the
1442 		 * target and lun pointers aren't NULL.  The xpt peripheral
1443 		 * has a wildcard target and device.
1444 		 */
1445 		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1446 		 && ((periph->path->target == NULL)
1447 		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1448 			continue;
1449 
1450 		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1451 		 && ((periph->path->device == NULL)
1452 		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1453 			continue;
1454 
1455 		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1456 		 && (cur_pattern->unit_number != periph->unit_number))
1457 			continue;
1458 
1459 		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1460 		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1461 			     DEV_IDLEN) != 0))
1462 			continue;
1463 
1464 		/*
1465 		 * If we get to this point, the user definitely wants
1466 		 * information on this peripheral.  So tell the caller to
1467 		 * copy the data out.
1468 		 */
1469 		retval |= DM_RET_COPY;
1470 
1471 		/*
1472 		 * The return action has already been set to stop, since
1473 		 * peripherals don't have any nodes below them in the EDT.
1474 		 */
1475 		return(retval);
1476 	}
1477 
1478 	/*
1479 	 * If we get to this point, the peripheral that was passed in
1480 	 * doesn't match any of the patterns.
1481 	 */
1482 	return(retval);
1483 }
1484 
1485 static int
1486 xptedtbusfunc(struct cam_eb *bus, void *arg)
1487 {
1488 	struct ccb_dev_match *cdm;
1489 	dev_match_ret retval;
1490 
1491 	cdm = (struct ccb_dev_match *)arg;
1492 
1493 	/*
1494 	 * If our position is for something deeper in the tree, that means
1495 	 * that we've already seen this node.  So, we keep going down.
1496 	 */
1497 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1498 	 && (cdm->pos.cookie.bus == bus)
1499 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1500 	 && (cdm->pos.cookie.target != NULL))
1501 		retval = DM_RET_DESCEND;
1502 	else
1503 		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1504 
1505 	/*
1506 	 * If we got an error, bail out of the search.
1507 	 */
1508 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1509 		cdm->status = CAM_DEV_MATCH_ERROR;
1510 		return(0);
1511 	}
1512 
1513 	/*
1514 	 * If the copy flag is set, copy this bus out.
1515 	 */
1516 	if (retval & DM_RET_COPY) {
1517 		int spaceleft, j;
1518 
1519 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1520 			sizeof(struct dev_match_result));
1521 
1522 		/*
1523 		 * If we don't have enough space to put in another
1524 		 * match result, save our position and tell the
1525 		 * user there are more devices to check.
1526 		 */
1527 		if (spaceleft < sizeof(struct dev_match_result)) {
1528 			bzero(&cdm->pos, sizeof(cdm->pos));
1529 			cdm->pos.position_type =
1530 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1531 
1532 			cdm->pos.cookie.bus = bus;
1533 			cdm->pos.generations[CAM_BUS_GENERATION]=
1534 				xsoftc.bus_generation;
1535 			cdm->status = CAM_DEV_MATCH_MORE;
1536 			return(0);
1537 		}
1538 		j = cdm->num_matches;
1539 		cdm->num_matches++;
1540 		cdm->matches[j].type = DEV_MATCH_BUS;
1541 		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1542 		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1543 		cdm->matches[j].result.bus_result.unit_number =
1544 			bus->sim->unit_number;
1545 		strncpy(cdm->matches[j].result.bus_result.dev_name,
1546 			bus->sim->sim_name, DEV_IDLEN);
1547 	}
1548 
1549 	/*
1550 	 * If the user is only interested in busses, there's no
1551 	 * reason to descend to the next level in the tree.
1552 	 */
1553 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1554 		return(1);
1555 
1556 	/*
1557 	 * If there is a target generation recorded, check it to
1558 	 * make sure the target list hasn't changed.
1559 	 */
1560 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1561 	 && (bus == cdm->pos.cookie.bus)
1562 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1563 	 && (cdm->pos.generations[CAM_TARGET_GENERATION] != 0)
1564 	 && (cdm->pos.generations[CAM_TARGET_GENERATION] !=
1565 	     bus->generation)) {
1566 		cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1567 		return(0);
1568 	}
1569 
1570 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1571 	 && (cdm->pos.cookie.bus == bus)
1572 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1573 	 && (cdm->pos.cookie.target != NULL))
1574 		return(xpttargettraverse(bus,
1575 					(struct cam_et *)cdm->pos.cookie.target,
1576 					 xptedttargetfunc, arg));
1577 	else
1578 		return(xpttargettraverse(bus, NULL, xptedttargetfunc, arg));
1579 }
1580 
1581 static int
1582 xptedttargetfunc(struct cam_et *target, void *arg)
1583 {
1584 	struct ccb_dev_match *cdm;
1585 
1586 	cdm = (struct ccb_dev_match *)arg;
1587 
1588 	/*
1589 	 * If there is a device list generation recorded, check it to
1590 	 * make sure the device list hasn't changed.
1591 	 */
1592 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1593 	 && (cdm->pos.cookie.bus == target->bus)
1594 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1595 	 && (cdm->pos.cookie.target == target)
1596 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1597 	 && (cdm->pos.generations[CAM_DEV_GENERATION] != 0)
1598 	 && (cdm->pos.generations[CAM_DEV_GENERATION] !=
1599 	     target->generation)) {
1600 		cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1601 		return(0);
1602 	}
1603 
1604 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1605 	 && (cdm->pos.cookie.bus == target->bus)
1606 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1607 	 && (cdm->pos.cookie.target == target)
1608 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1609 	 && (cdm->pos.cookie.device != NULL))
1610 		return(xptdevicetraverse(target,
1611 					(struct cam_ed *)cdm->pos.cookie.device,
1612 					 xptedtdevicefunc, arg));
1613 	else
1614 		return(xptdevicetraverse(target, NULL, xptedtdevicefunc, arg));
1615 }
1616 
1617 static int
1618 xptedtdevicefunc(struct cam_ed *device, void *arg)
1619 {
1620 
1621 	struct ccb_dev_match *cdm;
1622 	dev_match_ret retval;
1623 
1624 	cdm = (struct ccb_dev_match *)arg;
1625 
1626 	/*
1627 	 * If our position is for something deeper in the tree, that means
1628 	 * that we've already seen this node.  So, we keep going down.
1629 	 */
1630 	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1631 	 && (cdm->pos.cookie.device == device)
1632 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1633 	 && (cdm->pos.cookie.periph != NULL))
1634 		retval = DM_RET_DESCEND;
1635 	else
1636 		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1637 					device);
1638 
1639 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1640 		cdm->status = CAM_DEV_MATCH_ERROR;
1641 		return(0);
1642 	}
1643 
1644 	/*
1645 	 * If the copy flag is set, copy this device out.
1646 	 */
1647 	if (retval & DM_RET_COPY) {
1648 		int spaceleft, j;
1649 
1650 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1651 			sizeof(struct dev_match_result));
1652 
1653 		/*
1654 		 * If we don't have enough space to put in another
1655 		 * match result, save our position and tell the
1656 		 * user there are more devices to check.
1657 		 */
1658 		if (spaceleft < sizeof(struct dev_match_result)) {
1659 			bzero(&cdm->pos, sizeof(cdm->pos));
1660 			cdm->pos.position_type =
1661 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1662 				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1663 
1664 			cdm->pos.cookie.bus = device->target->bus;
1665 			cdm->pos.generations[CAM_BUS_GENERATION]=
1666 				xsoftc.bus_generation;
1667 			cdm->pos.cookie.target = device->target;
1668 			cdm->pos.generations[CAM_TARGET_GENERATION] =
1669 				device->target->bus->generation;
1670 			cdm->pos.cookie.device = device;
1671 			cdm->pos.generations[CAM_DEV_GENERATION] =
1672 				device->target->generation;
1673 			cdm->status = CAM_DEV_MATCH_MORE;
1674 			return(0);
1675 		}
1676 		j = cdm->num_matches;
1677 		cdm->num_matches++;
1678 		cdm->matches[j].type = DEV_MATCH_DEVICE;
1679 		cdm->matches[j].result.device_result.path_id =
1680 			device->target->bus->path_id;
1681 		cdm->matches[j].result.device_result.target_id =
1682 			device->target->target_id;
1683 		cdm->matches[j].result.device_result.target_lun =
1684 			device->lun_id;
1685 		cdm->matches[j].result.device_result.protocol =
1686 			device->protocol;
1687 		bcopy(&device->inq_data,
1688 		      &cdm->matches[j].result.device_result.inq_data,
1689 		      sizeof(struct scsi_inquiry_data));
1690 		bcopy(&device->ident_data,
1691 		      &cdm->matches[j].result.device_result.ident_data,
1692 		      sizeof(struct ata_params));
1693 
1694 		/* Let the user know whether this device is unconfigured */
1695 		if (device->flags & CAM_DEV_UNCONFIGURED)
1696 			cdm->matches[j].result.device_result.flags =
1697 				DEV_RESULT_UNCONFIGURED;
1698 		else
1699 			cdm->matches[j].result.device_result.flags =
1700 				DEV_RESULT_NOFLAG;
1701 	}
1702 
1703 	/*
1704 	 * If the user isn't interested in peripherals, don't descend
1705 	 * the tree any further.
1706 	 */
1707 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1708 		return(1);
1709 
1710 	/*
1711 	 * If there is a peripheral list generation recorded, make sure
1712 	 * it hasn't changed.
1713 	 */
1714 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1715 	 && (device->target->bus == cdm->pos.cookie.bus)
1716 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1717 	 && (device->target == cdm->pos.cookie.target)
1718 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1719 	 && (device == cdm->pos.cookie.device)
1720 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1721 	 && (cdm->pos.generations[CAM_PERIPH_GENERATION] != 0)
1722 	 && (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1723 	     device->generation)){
1724 		cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1725 		return(0);
1726 	}
1727 
1728 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1729 	 && (cdm->pos.cookie.bus == device->target->bus)
1730 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1731 	 && (cdm->pos.cookie.target == device->target)
1732 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1733 	 && (cdm->pos.cookie.device == device)
1734 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1735 	 && (cdm->pos.cookie.periph != NULL))
1736 		return(xptperiphtraverse(device,
1737 				(struct cam_periph *)cdm->pos.cookie.periph,
1738 				xptedtperiphfunc, arg));
1739 	else
1740 		return(xptperiphtraverse(device, NULL, xptedtperiphfunc, arg));
1741 }
1742 
1743 static int
1744 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1745 {
1746 	struct ccb_dev_match *cdm;
1747 	dev_match_ret retval;
1748 
1749 	cdm = (struct ccb_dev_match *)arg;
1750 
1751 	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1752 
1753 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1754 		cdm->status = CAM_DEV_MATCH_ERROR;
1755 		return(0);
1756 	}
1757 
1758 	/*
1759 	 * If the copy flag is set, copy this peripheral out.
1760 	 */
1761 	if (retval & DM_RET_COPY) {
1762 		int spaceleft, j;
1763 
1764 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1765 			sizeof(struct dev_match_result));
1766 
1767 		/*
1768 		 * If we don't have enough space to put in another
1769 		 * match result, save our position and tell the
1770 		 * user there are more devices to check.
1771 		 */
1772 		if (spaceleft < sizeof(struct dev_match_result)) {
1773 			bzero(&cdm->pos, sizeof(cdm->pos));
1774 			cdm->pos.position_type =
1775 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1776 				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1777 				CAM_DEV_POS_PERIPH;
1778 
1779 			cdm->pos.cookie.bus = periph->path->bus;
1780 			cdm->pos.generations[CAM_BUS_GENERATION]=
1781 				xsoftc.bus_generation;
1782 			cdm->pos.cookie.target = periph->path->target;
1783 			cdm->pos.generations[CAM_TARGET_GENERATION] =
1784 				periph->path->bus->generation;
1785 			cdm->pos.cookie.device = periph->path->device;
1786 			cdm->pos.generations[CAM_DEV_GENERATION] =
1787 				periph->path->target->generation;
1788 			cdm->pos.cookie.periph = periph;
1789 			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1790 				periph->path->device->generation;
1791 			cdm->status = CAM_DEV_MATCH_MORE;
1792 			return(0);
1793 		}
1794 
1795 		j = cdm->num_matches;
1796 		cdm->num_matches++;
1797 		cdm->matches[j].type = DEV_MATCH_PERIPH;
1798 		cdm->matches[j].result.periph_result.path_id =
1799 			periph->path->bus->path_id;
1800 		cdm->matches[j].result.periph_result.target_id =
1801 			periph->path->target->target_id;
1802 		cdm->matches[j].result.periph_result.target_lun =
1803 			periph->path->device->lun_id;
1804 		cdm->matches[j].result.periph_result.unit_number =
1805 			periph->unit_number;
1806 		strncpy(cdm->matches[j].result.periph_result.periph_name,
1807 			periph->periph_name, DEV_IDLEN);
1808 	}
1809 
1810 	return(1);
1811 }
1812 
1813 static int
1814 xptedtmatch(struct ccb_dev_match *cdm)
1815 {
1816 	int ret;
1817 
1818 	cdm->num_matches = 0;
1819 
1820 	/*
1821 	 * Check the bus list generation.  If it has changed, the user
1822 	 * needs to reset everything and start over.
1823 	 */
1824 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1825 	 && (cdm->pos.generations[CAM_BUS_GENERATION] != 0)
1826 	 && (cdm->pos.generations[CAM_BUS_GENERATION] != xsoftc.bus_generation)) {
1827 		cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1828 		return(0);
1829 	}
1830 
1831 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1832 	 && (cdm->pos.cookie.bus != NULL))
1833 		ret = xptbustraverse((struct cam_eb *)cdm->pos.cookie.bus,
1834 				     xptedtbusfunc, cdm);
1835 	else
1836 		ret = xptbustraverse(NULL, xptedtbusfunc, cdm);
1837 
1838 	/*
1839 	 * If we get back 0, that means that we had to stop before fully
1840 	 * traversing the EDT.  It also means that one of the subroutines
1841 	 * has set the status field to the proper value.  If we get back 1,
1842 	 * we've fully traversed the EDT and copied out any matching entries.
1843 	 */
1844 	if (ret == 1)
1845 		cdm->status = CAM_DEV_MATCH_LAST;
1846 
1847 	return(ret);
1848 }
1849 
1850 static int
1851 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1852 {
1853 	struct ccb_dev_match *cdm;
1854 
1855 	cdm = (struct ccb_dev_match *)arg;
1856 
1857 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1858 	 && (cdm->pos.cookie.pdrv == pdrv)
1859 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1860 	 && (cdm->pos.generations[CAM_PERIPH_GENERATION] != 0)
1861 	 && (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1862 	     (*pdrv)->generation)) {
1863 		cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1864 		return(0);
1865 	}
1866 
1867 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1868 	 && (cdm->pos.cookie.pdrv == pdrv)
1869 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1870 	 && (cdm->pos.cookie.periph != NULL))
1871 		return(xptpdperiphtraverse(pdrv,
1872 				(struct cam_periph *)cdm->pos.cookie.periph,
1873 				xptplistperiphfunc, arg));
1874 	else
1875 		return(xptpdperiphtraverse(pdrv, NULL,xptplistperiphfunc, arg));
1876 }
1877 
1878 static int
1879 xptplistperiphfunc(struct cam_periph *periph, void *arg)
1880 {
1881 	struct ccb_dev_match *cdm;
1882 	dev_match_ret retval;
1883 
1884 	cdm = (struct ccb_dev_match *)arg;
1885 
1886 	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1887 
1888 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1889 		cdm->status = CAM_DEV_MATCH_ERROR;
1890 		return(0);
1891 	}
1892 
1893 	/*
1894 	 * If the copy flag is set, copy this peripheral out.
1895 	 */
1896 	if (retval & DM_RET_COPY) {
1897 		int spaceleft, j;
1898 
1899 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1900 			sizeof(struct dev_match_result));
1901 
1902 		/*
1903 		 * If we don't have enough space to put in another
1904 		 * match result, save our position and tell the
1905 		 * user there are more devices to check.
1906 		 */
1907 		if (spaceleft < sizeof(struct dev_match_result)) {
1908 			struct periph_driver **pdrv;
1909 
1910 			pdrv = NULL;
1911 			bzero(&cdm->pos, sizeof(cdm->pos));
1912 			cdm->pos.position_type =
1913 				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1914 				CAM_DEV_POS_PERIPH;
1915 
1916 			/*
1917 			 * This may look a bit non-sensical, but it is
1918 			 * actually quite logical.  There are very few
1919 			 * peripheral drivers, and bloating every peripheral
1920 			 * structure with a pointer back to its parent
1921 			 * peripheral driver linker set entry would cost
1922 			 * more in the long run than doing this quick lookup.
1923 			 */
1924 			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1925 				if (strcmp((*pdrv)->driver_name,
1926 				    periph->periph_name) == 0)
1927 					break;
1928 			}
1929 
1930 			if (*pdrv == NULL) {
1931 				cdm->status = CAM_DEV_MATCH_ERROR;
1932 				return(0);
1933 			}
1934 
1935 			cdm->pos.cookie.pdrv = pdrv;
1936 			/*
1937 			 * The periph generation slot does double duty, as
1938 			 * does the periph pointer slot.  They are used for
1939 			 * both edt and pdrv lookups and positioning.
1940 			 */
1941 			cdm->pos.cookie.periph = periph;
1942 			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1943 				(*pdrv)->generation;
1944 			cdm->status = CAM_DEV_MATCH_MORE;
1945 			return(0);
1946 		}
1947 
1948 		j = cdm->num_matches;
1949 		cdm->num_matches++;
1950 		cdm->matches[j].type = DEV_MATCH_PERIPH;
1951 		cdm->matches[j].result.periph_result.path_id =
1952 			periph->path->bus->path_id;
1953 
1954 		/*
1955 		 * The transport layer peripheral doesn't have a target or
1956 		 * lun.
1957 		 */
1958 		if (periph->path->target)
1959 			cdm->matches[j].result.periph_result.target_id =
1960 				periph->path->target->target_id;
1961 		else
1962 			cdm->matches[j].result.periph_result.target_id = -1;
1963 
1964 		if (periph->path->device)
1965 			cdm->matches[j].result.periph_result.target_lun =
1966 				periph->path->device->lun_id;
1967 		else
1968 			cdm->matches[j].result.periph_result.target_lun = -1;
1969 
1970 		cdm->matches[j].result.periph_result.unit_number =
1971 			periph->unit_number;
1972 		strncpy(cdm->matches[j].result.periph_result.periph_name,
1973 			periph->periph_name, DEV_IDLEN);
1974 	}
1975 
1976 	return(1);
1977 }
1978 
1979 static int
1980 xptperiphlistmatch(struct ccb_dev_match *cdm)
1981 {
1982 	int ret;
1983 
1984 	cdm->num_matches = 0;
1985 
1986 	/*
1987 	 * At this point in the edt traversal function, we check the bus
1988 	 * list generation to make sure that no busses have been added or
1989 	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
1990 	 * For the peripheral driver list traversal function, however, we
1991 	 * don't have to worry about new peripheral driver types coming or
1992 	 * going; they're in a linker set, and therefore can't change
1993 	 * without a recompile.
1994 	 */
1995 
1996 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1997 	 && (cdm->pos.cookie.pdrv != NULL))
1998 		ret = xptpdrvtraverse(
1999 				(struct periph_driver **)cdm->pos.cookie.pdrv,
2000 				xptplistpdrvfunc, cdm);
2001 	else
2002 		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2003 
2004 	/*
2005 	 * If we get back 0, that means that we had to stop before fully
2006 	 * traversing the peripheral driver tree.  It also means that one of
2007 	 * the subroutines has set the status field to the proper value.  If
2008 	 * we get back 1, we've fully traversed the EDT and copied out any
2009 	 * matching entries.
2010 	 */
2011 	if (ret == 1)
2012 		cdm->status = CAM_DEV_MATCH_LAST;
2013 
2014 	return(ret);
2015 }
2016 
2017 static int
2018 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2019 {
2020 	struct cam_eb *bus, *next_bus;
2021 	int retval;
2022 
2023 	retval = 1;
2024 
2025 	mtx_lock(&xsoftc.xpt_topo_lock);
2026 	for (bus = (start_bus ? start_bus : TAILQ_FIRST(&xsoftc.xpt_busses));
2027 	     bus != NULL;
2028 	     bus = next_bus) {
2029 		next_bus = TAILQ_NEXT(bus, links);
2030 
2031 		mtx_unlock(&xsoftc.xpt_topo_lock);
2032 		CAM_SIM_LOCK(bus->sim);
2033 		retval = tr_func(bus, arg);
2034 		CAM_SIM_UNLOCK(bus->sim);
2035 		if (retval == 0)
2036 			return(retval);
2037 		mtx_lock(&xsoftc.xpt_topo_lock);
2038 	}
2039 	mtx_unlock(&xsoftc.xpt_topo_lock);
2040 
2041 	return(retval);
2042 }
2043 
2044 int
2045 xpt_sim_opened(struct cam_sim *sim)
2046 {
2047 	struct cam_eb *bus;
2048 	struct cam_et *target;
2049 	struct cam_ed *device;
2050 	struct cam_periph *periph;
2051 
2052 	KASSERT(sim->refcount >= 1, ("sim->refcount >= 1"));
2053 	mtx_assert(sim->mtx, MA_OWNED);
2054 
2055 	mtx_lock(&xsoftc.xpt_topo_lock);
2056 	TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
2057 		if (bus->sim != sim)
2058 			continue;
2059 
2060 		TAILQ_FOREACH(target, &bus->et_entries, links) {
2061 			TAILQ_FOREACH(device, &target->ed_entries, links) {
2062 				SLIST_FOREACH(periph, &device->periphs,
2063 				    periph_links) {
2064 					if (periph->refcount > 0) {
2065 						mtx_unlock(&xsoftc.xpt_topo_lock);
2066 						return (1);
2067 					}
2068 				}
2069 			}
2070 		}
2071 	}
2072 
2073 	mtx_unlock(&xsoftc.xpt_topo_lock);
2074 	return (0);
2075 }
2076 
2077 static int
2078 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2079 		  xpt_targetfunc_t *tr_func, void *arg)
2080 {
2081 	struct cam_et *target, *next_target;
2082 	int retval;
2083 
2084 	retval = 1;
2085 	for (target = (start_target ? start_target :
2086 		       TAILQ_FIRST(&bus->et_entries));
2087 	     target != NULL; target = next_target) {
2088 
2089 		next_target = TAILQ_NEXT(target, links);
2090 
2091 		retval = tr_func(target, arg);
2092 
2093 		if (retval == 0)
2094 			return(retval);
2095 	}
2096 
2097 	return(retval);
2098 }
2099 
2100 static int
2101 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2102 		  xpt_devicefunc_t *tr_func, void *arg)
2103 {
2104 	struct cam_ed *device, *next_device;
2105 	int retval;
2106 
2107 	retval = 1;
2108 	for (device = (start_device ? start_device :
2109 		       TAILQ_FIRST(&target->ed_entries));
2110 	     device != NULL;
2111 	     device = next_device) {
2112 
2113 		next_device = TAILQ_NEXT(device, links);
2114 
2115 		retval = tr_func(device, arg);
2116 
2117 		if (retval == 0)
2118 			return(retval);
2119 	}
2120 
2121 	return(retval);
2122 }
2123 
2124 static int
2125 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2126 		  xpt_periphfunc_t *tr_func, void *arg)
2127 {
2128 	struct cam_periph *periph, *next_periph;
2129 	int retval;
2130 
2131 	retval = 1;
2132 
2133 	for (periph = (start_periph ? start_periph :
2134 		       SLIST_FIRST(&device->periphs));
2135 	     periph != NULL;
2136 	     periph = next_periph) {
2137 
2138 		next_periph = SLIST_NEXT(periph, periph_links);
2139 
2140 		retval = tr_func(periph, arg);
2141 		if (retval == 0)
2142 			return(retval);
2143 	}
2144 
2145 	return(retval);
2146 }
2147 
2148 static int
2149 xptpdrvtraverse(struct periph_driver **start_pdrv,
2150 		xpt_pdrvfunc_t *tr_func, void *arg)
2151 {
2152 	struct periph_driver **pdrv;
2153 	int retval;
2154 
2155 	retval = 1;
2156 
2157 	/*
2158 	 * We don't traverse the peripheral driver list like we do the
2159 	 * other lists, because it is a linker set, and therefore cannot be
2160 	 * changed during runtime.  If the peripheral driver list is ever
2161 	 * re-done to be something other than a linker set (i.e. it can
2162 	 * change while the system is running), the list traversal should
2163 	 * be modified to work like the other traversal functions.
2164 	 */
2165 	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2166 	     *pdrv != NULL; pdrv++) {
2167 		retval = tr_func(pdrv, arg);
2168 
2169 		if (retval == 0)
2170 			return(retval);
2171 	}
2172 
2173 	return(retval);
2174 }
2175 
2176 static int
2177 xptpdperiphtraverse(struct periph_driver **pdrv,
2178 		    struct cam_periph *start_periph,
2179 		    xpt_periphfunc_t *tr_func, void *arg)
2180 {
2181 	struct cam_periph *periph, *next_periph;
2182 	int retval;
2183 
2184 	retval = 1;
2185 
2186 	xpt_lock_buses();
2187 	for (periph = (start_periph ? start_periph :
2188 	     TAILQ_FIRST(&(*pdrv)->units)); periph != NULL;
2189 	     periph = next_periph) {
2190 
2191 		next_periph = TAILQ_NEXT(periph, unit_links);
2192 
2193 		retval = tr_func(periph, arg);
2194 		if (retval == 0) {
2195 			xpt_unlock_buses();
2196 			return(retval);
2197 		}
2198 	}
2199 	xpt_unlock_buses();
2200 	return(retval);
2201 }
2202 
2203 static int
2204 xptdefbusfunc(struct cam_eb *bus, void *arg)
2205 {
2206 	struct xpt_traverse_config *tr_config;
2207 
2208 	tr_config = (struct xpt_traverse_config *)arg;
2209 
2210 	if (tr_config->depth == XPT_DEPTH_BUS) {
2211 		xpt_busfunc_t *tr_func;
2212 
2213 		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2214 
2215 		return(tr_func(bus, tr_config->tr_arg));
2216 	} else
2217 		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2218 }
2219 
2220 static int
2221 xptdeftargetfunc(struct cam_et *target, void *arg)
2222 {
2223 	struct xpt_traverse_config *tr_config;
2224 
2225 	tr_config = (struct xpt_traverse_config *)arg;
2226 
2227 	if (tr_config->depth == XPT_DEPTH_TARGET) {
2228 		xpt_targetfunc_t *tr_func;
2229 
2230 		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2231 
2232 		return(tr_func(target, tr_config->tr_arg));
2233 	} else
2234 		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2235 }
2236 
2237 static int
2238 xptdefdevicefunc(struct cam_ed *device, void *arg)
2239 {
2240 	struct xpt_traverse_config *tr_config;
2241 
2242 	tr_config = (struct xpt_traverse_config *)arg;
2243 
2244 	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2245 		xpt_devicefunc_t *tr_func;
2246 
2247 		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2248 
2249 		return(tr_func(device, tr_config->tr_arg));
2250 	} else
2251 		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2252 }
2253 
2254 static int
2255 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2256 {
2257 	struct xpt_traverse_config *tr_config;
2258 	xpt_periphfunc_t *tr_func;
2259 
2260 	tr_config = (struct xpt_traverse_config *)arg;
2261 
2262 	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2263 
2264 	/*
2265 	 * Unlike the other default functions, we don't check for depth
2266 	 * here.  The peripheral driver level is the last level in the EDT,
2267 	 * so if we're here, we should execute the function in question.
2268 	 */
2269 	return(tr_func(periph, tr_config->tr_arg));
2270 }
2271 
2272 /*
2273  * Execute the given function for every bus in the EDT.
2274  */
2275 static int
2276 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2277 {
2278 	struct xpt_traverse_config tr_config;
2279 
2280 	tr_config.depth = XPT_DEPTH_BUS;
2281 	tr_config.tr_func = tr_func;
2282 	tr_config.tr_arg = arg;
2283 
2284 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2285 }
2286 
2287 /*
2288  * Execute the given function for every device in the EDT.
2289  */
2290 static int
2291 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2292 {
2293 	struct xpt_traverse_config tr_config;
2294 
2295 	tr_config.depth = XPT_DEPTH_DEVICE;
2296 	tr_config.tr_func = tr_func;
2297 	tr_config.tr_arg = arg;
2298 
2299 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2300 }
2301 
2302 static int
2303 xptsetasyncfunc(struct cam_ed *device, void *arg)
2304 {
2305 	struct cam_path path;
2306 	struct ccb_getdev cgd;
2307 	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2308 
2309 	/*
2310 	 * Don't report unconfigured devices (Wildcard devs,
2311 	 * devices only for target mode, device instances
2312 	 * that have been invalidated but are waiting for
2313 	 * their last reference count to be released).
2314 	 */
2315 	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2316 		return (1);
2317 
2318 	xpt_compile_path(&path,
2319 			 NULL,
2320 			 device->target->bus->path_id,
2321 			 device->target->target_id,
2322 			 device->lun_id);
2323 	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2324 	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2325 	xpt_action((union ccb *)&cgd);
2326 	csa->callback(csa->callback_arg,
2327 			    AC_FOUND_DEVICE,
2328 			    &path, &cgd);
2329 	xpt_release_path(&path);
2330 
2331 	return(1);
2332 }
2333 
2334 static int
2335 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2336 {
2337 	struct cam_path path;
2338 	struct ccb_pathinq cpi;
2339 	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2340 
2341 	xpt_compile_path(&path, /*periph*/NULL,
2342 			 bus->sim->path_id,
2343 			 CAM_TARGET_WILDCARD,
2344 			 CAM_LUN_WILDCARD);
2345 	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2346 	cpi.ccb_h.func_code = XPT_PATH_INQ;
2347 	xpt_action((union ccb *)&cpi);
2348 	csa->callback(csa->callback_arg,
2349 			    AC_PATH_REGISTERED,
2350 			    &path, &cpi);
2351 	xpt_release_path(&path);
2352 
2353 	return(1);
2354 }
2355 
2356 void
2357 xpt_action(union ccb *start_ccb)
2358 {
2359 
2360 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2361 
2362 	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2363 	/* Compatibility for RL-unaware code. */
2364 	if (CAM_PRIORITY_TO_RL(start_ccb->ccb_h.pinfo.priority) == 0)
2365 	    start_ccb->ccb_h.pinfo.priority += CAM_PRIORITY_NORMAL - 1;
2366 	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2367 }
2368 
2369 void
2370 xpt_action_default(union ccb *start_ccb)
2371 {
2372 #ifdef CAMDEBUG
2373 	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
2374 #endif
2375 	struct cam_path *path;
2376 
2377 	path = start_ccb->ccb_h.path;
2378 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2379 
2380 	switch (start_ccb->ccb_h.func_code) {
2381 	case XPT_SCSI_IO:
2382 	{
2383 		struct cam_ed *device;
2384 
2385 		/*
2386 		 * For the sake of compatibility with SCSI-1
2387 		 * devices that may not understand the identify
2388 		 * message, we include lun information in the
2389 		 * second byte of all commands.  SCSI-1 specifies
2390 		 * that luns are a 3 bit value and reserves only 3
2391 		 * bits for lun information in the CDB.  Later
2392 		 * revisions of the SCSI spec allow for more than 8
2393 		 * luns, but have deprecated lun information in the
2394 		 * CDB.  So, if the lun won't fit, we must omit.
2395 		 *
2396 		 * Also be aware that during initial probing for devices,
2397 		 * the inquiry information is unknown but initialized to 0.
2398 		 * This means that this code will be exercised while probing
2399 		 * devices with an ANSI revision greater than 2.
2400 		 */
2401 		device = path->device;
2402 		if (device->protocol_version <= SCSI_REV_2
2403 		 && start_ccb->ccb_h.target_lun < 8
2404 		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2405 
2406 			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2407 			    start_ccb->ccb_h.target_lun << 5;
2408 		}
2409 		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2410 		CAM_DEBUG(path, CAM_DEBUG_CDB,("%s. CDB: %s\n",
2411 			  scsi_op_desc(start_ccb->csio.cdb_io.cdb_bytes[0],
2412 			  	       &path->device->inq_data),
2413 			  scsi_cdb_string(start_ccb->csio.cdb_io.cdb_bytes,
2414 					  cdb_str, sizeof(cdb_str))));
2415 	}
2416 	/* FALLTHROUGH */
2417 	case XPT_TARGET_IO:
2418 	case XPT_CONT_TARGET_IO:
2419 		start_ccb->csio.sense_resid = 0;
2420 		start_ccb->csio.resid = 0;
2421 		/* FALLTHROUGH */
2422 	case XPT_ATA_IO:
2423 		if (start_ccb->ccb_h.func_code == XPT_ATA_IO) {
2424 			start_ccb->ataio.resid = 0;
2425 			CAM_DEBUG(path, CAM_DEBUG_CDB,("%s. ACB: %s\n",
2426 			    ata_op_string(&start_ccb->ataio.cmd),
2427 			    ata_cmd_string(&start_ccb->ataio.cmd,
2428 					  cdb_str, sizeof(cdb_str))));
2429 		}
2430 		/* FALLTHROUGH */
2431 	case XPT_RESET_DEV:
2432 	case XPT_ENG_EXEC:
2433 	case XPT_SMP_IO:
2434 	{
2435 		int frozen;
2436 
2437 		frozen = cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2438 		path->device->sim->devq->alloc_openings += frozen;
2439 		if (frozen > 0)
2440 			xpt_run_dev_allocq(path->bus);
2441 		if (xpt_schedule_dev_sendq(path->bus, path->device))
2442 			xpt_run_dev_sendq(path->bus);
2443 		break;
2444 	}
2445 	case XPT_CALC_GEOMETRY:
2446 	{
2447 		struct cam_sim *sim;
2448 
2449 		/* Filter out garbage */
2450 		if (start_ccb->ccg.block_size == 0
2451 		 || start_ccb->ccg.volume_size == 0) {
2452 			start_ccb->ccg.cylinders = 0;
2453 			start_ccb->ccg.heads = 0;
2454 			start_ccb->ccg.secs_per_track = 0;
2455 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2456 			break;
2457 		}
2458 #if defined(PC98) || defined(__sparc64__)
2459 		/*
2460 		 * In a PC-98 system, geometry translation depens on
2461 		 * the "real" device geometry obtained from mode page 4.
2462 		 * SCSI geometry translation is performed in the
2463 		 * initialization routine of the SCSI BIOS and the result
2464 		 * stored in host memory.  If the translation is available
2465 		 * in host memory, use it.  If not, rely on the default
2466 		 * translation the device driver performs.
2467 		 * For sparc64, we may need adjust the geometry of large
2468 		 * disks in order to fit the limitations of the 16-bit
2469 		 * fields of the VTOC8 disk label.
2470 		 */
2471 		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2472 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2473 			break;
2474 		}
2475 #endif
2476 		sim = path->bus->sim;
2477 		(*(sim->sim_action))(sim, start_ccb);
2478 		break;
2479 	}
2480 	case XPT_ABORT:
2481 	{
2482 		union ccb* abort_ccb;
2483 
2484 		abort_ccb = start_ccb->cab.abort_ccb;
2485 		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2486 
2487 			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2488 				struct cam_ccbq *ccbq;
2489 				struct cam_ed *device;
2490 
2491 				device = abort_ccb->ccb_h.path->device;
2492 				ccbq = &device->ccbq;
2493 				device->sim->devq->alloc_openings -=
2494 				    cam_ccbq_remove_ccb(ccbq, abort_ccb);
2495 				abort_ccb->ccb_h.status =
2496 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2497 				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2498 				xpt_done(abort_ccb);
2499 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2500 				break;
2501 			}
2502 			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2503 			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2504 				/*
2505 				 * We've caught this ccb en route to
2506 				 * the SIM.  Flag it for abort and the
2507 				 * SIM will do so just before starting
2508 				 * real work on the CCB.
2509 				 */
2510 				abort_ccb->ccb_h.status =
2511 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2512 				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2513 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2514 				break;
2515 			}
2516 		}
2517 		if (XPT_FC_IS_QUEUED(abort_ccb)
2518 		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2519 			/*
2520 			 * It's already completed but waiting
2521 			 * for our SWI to get to it.
2522 			 */
2523 			start_ccb->ccb_h.status = CAM_UA_ABORT;
2524 			break;
2525 		}
2526 		/*
2527 		 * If we weren't able to take care of the abort request
2528 		 * in the XPT, pass the request down to the SIM for processing.
2529 		 */
2530 	}
2531 	/* FALLTHROUGH */
2532 	case XPT_ACCEPT_TARGET_IO:
2533 	case XPT_EN_LUN:
2534 	case XPT_IMMED_NOTIFY:
2535 	case XPT_NOTIFY_ACK:
2536 	case XPT_RESET_BUS:
2537 	case XPT_IMMEDIATE_NOTIFY:
2538 	case XPT_NOTIFY_ACKNOWLEDGE:
2539 	case XPT_GET_SIM_KNOB:
2540 	case XPT_SET_SIM_KNOB:
2541 	{
2542 		struct cam_sim *sim;
2543 
2544 		sim = path->bus->sim;
2545 		(*(sim->sim_action))(sim, start_ccb);
2546 		break;
2547 	}
2548 	case XPT_PATH_INQ:
2549 	{
2550 		struct cam_sim *sim;
2551 
2552 		sim = path->bus->sim;
2553 		(*(sim->sim_action))(sim, start_ccb);
2554 		break;
2555 	}
2556 	case XPT_PATH_STATS:
2557 		start_ccb->cpis.last_reset = path->bus->last_reset;
2558 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2559 		break;
2560 	case XPT_GDEV_TYPE:
2561 	{
2562 		struct cam_ed *dev;
2563 
2564 		dev = path->device;
2565 		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2566 			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2567 		} else {
2568 			struct ccb_getdev *cgd;
2569 
2570 			cgd = &start_ccb->cgd;
2571 			cgd->protocol = dev->protocol;
2572 			cgd->inq_data = dev->inq_data;
2573 			cgd->ident_data = dev->ident_data;
2574 			cgd->inq_flags = dev->inq_flags;
2575 			cgd->ccb_h.status = CAM_REQ_CMP;
2576 			cgd->serial_num_len = dev->serial_num_len;
2577 			if ((dev->serial_num_len > 0)
2578 			 && (dev->serial_num != NULL))
2579 				bcopy(dev->serial_num, cgd->serial_num,
2580 				      dev->serial_num_len);
2581 		}
2582 		break;
2583 	}
2584 	case XPT_GDEV_STATS:
2585 	{
2586 		struct cam_ed *dev;
2587 
2588 		dev = path->device;
2589 		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2590 			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2591 		} else {
2592 			struct ccb_getdevstats *cgds;
2593 			struct cam_eb *bus;
2594 			struct cam_et *tar;
2595 
2596 			cgds = &start_ccb->cgds;
2597 			bus = path->bus;
2598 			tar = path->target;
2599 			cgds->dev_openings = dev->ccbq.dev_openings;
2600 			cgds->dev_active = dev->ccbq.dev_active;
2601 			cgds->devq_openings = dev->ccbq.devq_openings;
2602 			cgds->devq_queued = dev->ccbq.queue.entries;
2603 			cgds->held = dev->ccbq.held;
2604 			cgds->last_reset = tar->last_reset;
2605 			cgds->maxtags = dev->maxtags;
2606 			cgds->mintags = dev->mintags;
2607 			if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2608 				cgds->last_reset = bus->last_reset;
2609 			cgds->ccb_h.status = CAM_REQ_CMP;
2610 		}
2611 		break;
2612 	}
2613 	case XPT_GDEVLIST:
2614 	{
2615 		struct cam_periph	*nperiph;
2616 		struct periph_list	*periph_head;
2617 		struct ccb_getdevlist	*cgdl;
2618 		u_int			i;
2619 		struct cam_ed		*device;
2620 		int			found;
2621 
2622 
2623 		found = 0;
2624 
2625 		/*
2626 		 * Don't want anyone mucking with our data.
2627 		 */
2628 		device = path->device;
2629 		periph_head = &device->periphs;
2630 		cgdl = &start_ccb->cgdl;
2631 
2632 		/*
2633 		 * Check and see if the list has changed since the user
2634 		 * last requested a list member.  If so, tell them that the
2635 		 * list has changed, and therefore they need to start over
2636 		 * from the beginning.
2637 		 */
2638 		if ((cgdl->index != 0) &&
2639 		    (cgdl->generation != device->generation)) {
2640 			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2641 			break;
2642 		}
2643 
2644 		/*
2645 		 * Traverse the list of peripherals and attempt to find
2646 		 * the requested peripheral.
2647 		 */
2648 		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2649 		     (nperiph != NULL) && (i <= cgdl->index);
2650 		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2651 			if (i == cgdl->index) {
2652 				strncpy(cgdl->periph_name,
2653 					nperiph->periph_name,
2654 					DEV_IDLEN);
2655 				cgdl->unit_number = nperiph->unit_number;
2656 				found = 1;
2657 			}
2658 		}
2659 		if (found == 0) {
2660 			cgdl->status = CAM_GDEVLIST_ERROR;
2661 			break;
2662 		}
2663 
2664 		if (nperiph == NULL)
2665 			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2666 		else
2667 			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2668 
2669 		cgdl->index++;
2670 		cgdl->generation = device->generation;
2671 
2672 		cgdl->ccb_h.status = CAM_REQ_CMP;
2673 		break;
2674 	}
2675 	case XPT_DEV_MATCH:
2676 	{
2677 		dev_pos_type position_type;
2678 		struct ccb_dev_match *cdm;
2679 
2680 		cdm = &start_ccb->cdm;
2681 
2682 		/*
2683 		 * There are two ways of getting at information in the EDT.
2684 		 * The first way is via the primary EDT tree.  It starts
2685 		 * with a list of busses, then a list of targets on a bus,
2686 		 * then devices/luns on a target, and then peripherals on a
2687 		 * device/lun.  The "other" way is by the peripheral driver
2688 		 * lists.  The peripheral driver lists are organized by
2689 		 * peripheral driver.  (obviously)  So it makes sense to
2690 		 * use the peripheral driver list if the user is looking
2691 		 * for something like "da1", or all "da" devices.  If the
2692 		 * user is looking for something on a particular bus/target
2693 		 * or lun, it's generally better to go through the EDT tree.
2694 		 */
2695 
2696 		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2697 			position_type = cdm->pos.position_type;
2698 		else {
2699 			u_int i;
2700 
2701 			position_type = CAM_DEV_POS_NONE;
2702 
2703 			for (i = 0; i < cdm->num_patterns; i++) {
2704 				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2705 				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2706 					position_type = CAM_DEV_POS_EDT;
2707 					break;
2708 				}
2709 			}
2710 
2711 			if (cdm->num_patterns == 0)
2712 				position_type = CAM_DEV_POS_EDT;
2713 			else if (position_type == CAM_DEV_POS_NONE)
2714 				position_type = CAM_DEV_POS_PDRV;
2715 		}
2716 
2717 		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2718 		case CAM_DEV_POS_EDT:
2719 			xptedtmatch(cdm);
2720 			break;
2721 		case CAM_DEV_POS_PDRV:
2722 			xptperiphlistmatch(cdm);
2723 			break;
2724 		default:
2725 			cdm->status = CAM_DEV_MATCH_ERROR;
2726 			break;
2727 		}
2728 
2729 		if (cdm->status == CAM_DEV_MATCH_ERROR)
2730 			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2731 		else
2732 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2733 
2734 		break;
2735 	}
2736 	case XPT_SASYNC_CB:
2737 	{
2738 		struct ccb_setasync *csa;
2739 		struct async_node *cur_entry;
2740 		struct async_list *async_head;
2741 		u_int32_t added;
2742 
2743 		csa = &start_ccb->csa;
2744 		added = csa->event_enable;
2745 		async_head = &path->device->asyncs;
2746 
2747 		/*
2748 		 * If there is already an entry for us, simply
2749 		 * update it.
2750 		 */
2751 		cur_entry = SLIST_FIRST(async_head);
2752 		while (cur_entry != NULL) {
2753 			if ((cur_entry->callback_arg == csa->callback_arg)
2754 			 && (cur_entry->callback == csa->callback))
2755 				break;
2756 			cur_entry = SLIST_NEXT(cur_entry, links);
2757 		}
2758 
2759 		if (cur_entry != NULL) {
2760 		 	/*
2761 			 * If the request has no flags set,
2762 			 * remove the entry.
2763 			 */
2764 			added &= ~cur_entry->event_enable;
2765 			if (csa->event_enable == 0) {
2766 				SLIST_REMOVE(async_head, cur_entry,
2767 					     async_node, links);
2768 				xpt_release_device(path->device);
2769 				free(cur_entry, M_CAMXPT);
2770 			} else {
2771 				cur_entry->event_enable = csa->event_enable;
2772 			}
2773 			csa->event_enable = added;
2774 		} else {
2775 			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2776 					   M_NOWAIT);
2777 			if (cur_entry == NULL) {
2778 				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2779 				break;
2780 			}
2781 			cur_entry->event_enable = csa->event_enable;
2782 			cur_entry->callback_arg = csa->callback_arg;
2783 			cur_entry->callback = csa->callback;
2784 			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2785 			xpt_acquire_device(path->device);
2786 		}
2787 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2788 		break;
2789 	}
2790 	case XPT_REL_SIMQ:
2791 	{
2792 		struct ccb_relsim *crs;
2793 		struct cam_ed *dev;
2794 
2795 		crs = &start_ccb->crs;
2796 		dev = path->device;
2797 		if (dev == NULL) {
2798 
2799 			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2800 			break;
2801 		}
2802 
2803 		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2804 
2805  			if (INQ_DATA_TQ_ENABLED(&dev->inq_data)) {
2806 				/* Don't ever go below one opening */
2807 				if (crs->openings > 0) {
2808 					xpt_dev_ccbq_resize(path,
2809 							    crs->openings);
2810 
2811 					if (bootverbose) {
2812 						xpt_print(path,
2813 						    "tagged openings now %d\n",
2814 						    crs->openings);
2815 					}
2816 				}
2817 			}
2818 		}
2819 
2820 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2821 
2822 			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2823 
2824 				/*
2825 				 * Just extend the old timeout and decrement
2826 				 * the freeze count so that a single timeout
2827 				 * is sufficient for releasing the queue.
2828 				 */
2829 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2830 				callout_stop(&dev->callout);
2831 			} else {
2832 
2833 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2834 			}
2835 
2836 			callout_reset(&dev->callout,
2837 			    (crs->release_timeout * hz) / 1000,
2838 			    xpt_release_devq_timeout, dev);
2839 
2840 			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2841 
2842 		}
2843 
2844 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2845 
2846 			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2847 				/*
2848 				 * Decrement the freeze count so that a single
2849 				 * completion is still sufficient to unfreeze
2850 				 * the queue.
2851 				 */
2852 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2853 			} else {
2854 
2855 				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2856 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2857 			}
2858 		}
2859 
2860 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2861 
2862 			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2863 			 || (dev->ccbq.dev_active == 0)) {
2864 
2865 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2866 			} else {
2867 
2868 				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2869 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2870 			}
2871 		}
2872 
2873 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0) {
2874 			xpt_release_devq_rl(path, /*runlevel*/
2875 			    (crs->release_flags & RELSIM_RELEASE_RUNLEVEL) ?
2876 				crs->release_timeout : 0,
2877 			    /*count*/1, /*run_queue*/TRUE);
2878 		}
2879 		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt[0];
2880 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2881 		break;
2882 	}
2883 	case XPT_DEBUG: {
2884 #ifdef CAMDEBUG
2885 #ifdef CAM_DEBUG_DELAY
2886 		cam_debug_delay = CAM_DEBUG_DELAY;
2887 #endif
2888 		cam_dflags = start_ccb->cdbg.flags;
2889 		if (cam_dpath != NULL) {
2890 			xpt_free_path(cam_dpath);
2891 			cam_dpath = NULL;
2892 		}
2893 
2894 		if (cam_dflags != CAM_DEBUG_NONE) {
2895 			if (xpt_create_path(&cam_dpath, xpt_periph,
2896 					    start_ccb->ccb_h.path_id,
2897 					    start_ccb->ccb_h.target_id,
2898 					    start_ccb->ccb_h.target_lun) !=
2899 					    CAM_REQ_CMP) {
2900 				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2901 				cam_dflags = CAM_DEBUG_NONE;
2902 			} else {
2903 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2904 				xpt_print(cam_dpath, "debugging flags now %x\n",
2905 				    cam_dflags);
2906 			}
2907 		} else {
2908 			cam_dpath = NULL;
2909 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2910 		}
2911 #else /* !CAMDEBUG */
2912 		start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2913 #endif /* CAMDEBUG */
2914 		break;
2915 	}
2916 	case XPT_FREEZE_QUEUE:
2917 	{
2918 		struct ccb_relsim *crs = &start_ccb->crs;
2919 
2920 		xpt_freeze_devq_rl(path, /*runlevel*/
2921 		    (crs->release_flags & RELSIM_RELEASE_RUNLEVEL) ?
2922 		    crs->release_timeout : 0, /*count*/1);
2923 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2924 		break;
2925 	}
2926 	case XPT_NOOP:
2927 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2928 			xpt_freeze_devq(path, 1);
2929 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2930 		break;
2931 	default:
2932 	case XPT_SDEV_TYPE:
2933 	case XPT_TERM_IO:
2934 	case XPT_ENG_INQ:
2935 		/* XXX Implement */
2936 		printf("%s: CCB type %#x not supported\n", __func__,
2937 		       start_ccb->ccb_h.func_code);
2938 		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
2939 		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
2940 			xpt_done(start_ccb);
2941 		}
2942 		break;
2943 	}
2944 }
2945 
2946 void
2947 xpt_polled_action(union ccb *start_ccb)
2948 {
2949 	u_int32_t timeout;
2950 	struct	  cam_sim *sim;
2951 	struct	  cam_devq *devq;
2952 	struct	  cam_ed *dev;
2953 
2954 
2955 	timeout = start_ccb->ccb_h.timeout * 10;
2956 	sim = start_ccb->ccb_h.path->bus->sim;
2957 	devq = sim->devq;
2958 	dev = start_ccb->ccb_h.path->device;
2959 
2960 	mtx_assert(sim->mtx, MA_OWNED);
2961 
2962 	/* Don't use ISR for this SIM while polling. */
2963 	sim->flags |= CAM_SIM_POLLED;
2964 
2965 	/*
2966 	 * Steal an opening so that no other queued requests
2967 	 * can get it before us while we simulate interrupts.
2968 	 */
2969 	dev->ccbq.devq_openings--;
2970 	dev->ccbq.dev_openings--;
2971 
2972 	while(((devq != NULL && devq->send_openings <= 0) ||
2973 	   dev->ccbq.dev_openings < 0) && (--timeout > 0)) {
2974 		DELAY(100);
2975 		(*(sim->sim_poll))(sim);
2976 		camisr_runqueue(&sim->sim_doneq);
2977 	}
2978 
2979 	dev->ccbq.devq_openings++;
2980 	dev->ccbq.dev_openings++;
2981 
2982 	if (timeout != 0) {
2983 		xpt_action(start_ccb);
2984 		while(--timeout > 0) {
2985 			(*(sim->sim_poll))(sim);
2986 			camisr_runqueue(&sim->sim_doneq);
2987 			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
2988 			    != CAM_REQ_INPROG)
2989 				break;
2990 			DELAY(100);
2991 		}
2992 		if (timeout == 0) {
2993 			/*
2994 			 * XXX Is it worth adding a sim_timeout entry
2995 			 * point so we can attempt recovery?  If
2996 			 * this is only used for dumps, I don't think
2997 			 * it is.
2998 			 */
2999 			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3000 		}
3001 	} else {
3002 		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3003 	}
3004 
3005 	/* We will use CAM ISR for this SIM again. */
3006 	sim->flags &= ~CAM_SIM_POLLED;
3007 }
3008 
3009 /*
3010  * Schedule a peripheral driver to receive a ccb when it's
3011  * target device has space for more transactions.
3012  */
3013 void
3014 xpt_schedule(struct cam_periph *perph, u_int32_t new_priority)
3015 {
3016 	struct cam_ed *device;
3017 	int runq = 0;
3018 
3019 	mtx_assert(perph->sim->mtx, MA_OWNED);
3020 
3021 	CAM_DEBUG(perph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3022 	device = perph->path->device;
3023 	if (periph_is_queued(perph)) {
3024 		/* Simply reorder based on new priority */
3025 		CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
3026 			  ("   change priority to %d\n", new_priority));
3027 		if (new_priority < perph->pinfo.priority) {
3028 			camq_change_priority(&device->drvq,
3029 					     perph->pinfo.index,
3030 					     new_priority);
3031 			runq = xpt_schedule_dev_allocq(perph->path->bus, device);
3032 		}
3033 	} else {
3034 		/* New entry on the queue */
3035 		CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
3036 			  ("   added periph to queue\n"));
3037 		perph->pinfo.priority = new_priority;
3038 		perph->pinfo.generation = ++device->drvq.generation;
3039 		camq_insert(&device->drvq, &perph->pinfo);
3040 		runq = xpt_schedule_dev_allocq(perph->path->bus, device);
3041 	}
3042 	if (runq != 0) {
3043 		CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
3044 			  ("   calling xpt_run_devq\n"));
3045 		xpt_run_dev_allocq(perph->path->bus);
3046 	}
3047 }
3048 
3049 
3050 /*
3051  * Schedule a device to run on a given queue.
3052  * If the device was inserted as a new entry on the queue,
3053  * return 1 meaning the device queue should be run. If we
3054  * were already queued, implying someone else has already
3055  * started the queue, return 0 so the caller doesn't attempt
3056  * to run the queue.
3057  */
3058 int
3059 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3060 		 u_int32_t new_priority)
3061 {
3062 	int retval;
3063 	u_int32_t old_priority;
3064 
3065 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3066 
3067 	old_priority = pinfo->priority;
3068 
3069 	/*
3070 	 * Are we already queued?
3071 	 */
3072 	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3073 		/* Simply reorder based on new priority */
3074 		if (new_priority < old_priority) {
3075 			camq_change_priority(queue, pinfo->index,
3076 					     new_priority);
3077 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3078 					("changed priority to %d\n",
3079 					 new_priority));
3080 			retval = 1;
3081 		} else
3082 			retval = 0;
3083 	} else {
3084 		/* New entry on the queue */
3085 		if (new_priority < old_priority)
3086 			pinfo->priority = new_priority;
3087 
3088 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3089 				("Inserting onto queue\n"));
3090 		pinfo->generation = ++queue->generation;
3091 		camq_insert(queue, pinfo);
3092 		retval = 1;
3093 	}
3094 	return (retval);
3095 }
3096 
3097 static void
3098 xpt_run_dev_allocq(struct cam_eb *bus)
3099 {
3100 	struct	cam_devq *devq;
3101 
3102 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_dev_allocq\n"));
3103 	devq = bus->sim->devq;
3104 
3105 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3106 			("   qfrozen_cnt == 0x%x, entries == %d, "
3107 			 "openings == %d, active == %d\n",
3108 			 devq->alloc_queue.qfrozen_cnt[0],
3109 			 devq->alloc_queue.entries,
3110 			 devq->alloc_openings,
3111 			 devq->alloc_active));
3112 
3113 	devq->alloc_queue.qfrozen_cnt[0]++;
3114 	while ((devq->alloc_queue.entries > 0)
3115 	    && (devq->alloc_openings > 0)
3116 	    && (devq->alloc_queue.qfrozen_cnt[0] <= 1)) {
3117 		struct	cam_ed_qinfo *qinfo;
3118 		struct	cam_ed *device;
3119 		union	ccb *work_ccb;
3120 		struct	cam_periph *drv;
3121 		struct	camq *drvq;
3122 
3123 		qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->alloc_queue,
3124 							   CAMQ_HEAD);
3125 		device = qinfo->device;
3126 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3127 				("running device %p\n", device));
3128 
3129 		drvq = &device->drvq;
3130 
3131 #ifdef CAMDEBUG
3132 		if (drvq->entries <= 0) {
3133 			panic("xpt_run_dev_allocq: "
3134 			      "Device on queue without any work to do");
3135 		}
3136 #endif
3137 		if ((work_ccb = xpt_get_ccb(device)) != NULL) {
3138 			devq->alloc_openings--;
3139 			devq->alloc_active++;
3140 			drv = (struct cam_periph*)camq_remove(drvq, CAMQ_HEAD);
3141 			xpt_setup_ccb(&work_ccb->ccb_h, drv->path,
3142 				      drv->pinfo.priority);
3143 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3144 					("calling periph start\n"));
3145 			drv->periph_start(drv, work_ccb);
3146 		} else {
3147 			/*
3148 			 * Malloc failure in alloc_ccb
3149 			 */
3150 			/*
3151 			 * XXX add us to a list to be run from free_ccb
3152 			 * if we don't have any ccbs active on this
3153 			 * device queue otherwise we may never get run
3154 			 * again.
3155 			 */
3156 			break;
3157 		}
3158 
3159 		/* We may have more work. Attempt to reschedule. */
3160 		xpt_schedule_dev_allocq(bus, device);
3161 	}
3162 	devq->alloc_queue.qfrozen_cnt[0]--;
3163 }
3164 
3165 static void
3166 xpt_run_dev_sendq(struct cam_eb *bus)
3167 {
3168 	struct	cam_devq *devq;
3169 
3170 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_dev_sendq\n"));
3171 
3172 	devq = bus->sim->devq;
3173 
3174 	devq->send_queue.qfrozen_cnt[0]++;
3175 	while ((devq->send_queue.entries > 0)
3176 	    && (devq->send_openings > 0)
3177 	    && (devq->send_queue.qfrozen_cnt[0] <= 1)) {
3178 		struct	cam_ed_qinfo *qinfo;
3179 		struct	cam_ed *device;
3180 		union ccb *work_ccb;
3181 		struct	cam_sim *sim;
3182 
3183 		qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->send_queue,
3184 							   CAMQ_HEAD);
3185 		device = qinfo->device;
3186 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3187 				("running device %p\n", device));
3188 
3189 		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3190 		if (work_ccb == NULL) {
3191 			printf("device on run queue with no ccbs???\n");
3192 			continue;
3193 		}
3194 
3195 		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3196 
3197 			mtx_lock(&xsoftc.xpt_lock);
3198 		 	if (xsoftc.num_highpower <= 0) {
3199 				/*
3200 				 * We got a high power command, but we
3201 				 * don't have any available slots.  Freeze
3202 				 * the device queue until we have a slot
3203 				 * available.
3204 				 */
3205 				xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3206 				STAILQ_INSERT_TAIL(&xsoftc.highpowerq,
3207 						   &work_ccb->ccb_h,
3208 						   xpt_links.stqe);
3209 
3210 				mtx_unlock(&xsoftc.xpt_lock);
3211 				continue;
3212 			} else {
3213 				/*
3214 				 * Consume a high power slot while
3215 				 * this ccb runs.
3216 				 */
3217 				xsoftc.num_highpower--;
3218 			}
3219 			mtx_unlock(&xsoftc.xpt_lock);
3220 		}
3221 		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3222 		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3223 
3224 		devq->send_openings--;
3225 		devq->send_active++;
3226 
3227 		xpt_schedule_dev_sendq(bus, device);
3228 
3229 		if (work_ccb && (work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0){
3230 			/*
3231 			 * The client wants to freeze the queue
3232 			 * after this CCB is sent.
3233 			 */
3234 			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3235 		}
3236 
3237 		/* In Target mode, the peripheral driver knows best... */
3238 		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3239 			if ((device->inq_flags & SID_CmdQue) != 0
3240 			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3241 				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3242 			else
3243 				/*
3244 				 * Clear this in case of a retried CCB that
3245 				 * failed due to a rejected tag.
3246 				 */
3247 				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3248 		}
3249 
3250 		/*
3251 		 * Device queues can be shared among multiple sim instances
3252 		 * that reside on different busses.  Use the SIM in the queue
3253 		 * CCB's path, rather than the one in the bus that was passed
3254 		 * into this function.
3255 		 */
3256 		sim = work_ccb->ccb_h.path->bus->sim;
3257 		(*(sim->sim_action))(sim, work_ccb);
3258 	}
3259 	devq->send_queue.qfrozen_cnt[0]--;
3260 }
3261 
3262 /*
3263  * This function merges stuff from the slave ccb into the master ccb, while
3264  * keeping important fields in the master ccb constant.
3265  */
3266 void
3267 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3268 {
3269 
3270 	/*
3271 	 * Pull fields that are valid for peripheral drivers to set
3272 	 * into the master CCB along with the CCB "payload".
3273 	 */
3274 	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3275 	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3276 	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3277 	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3278 	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3279 	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3280 }
3281 
3282 void
3283 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3284 {
3285 
3286 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3287 	ccb_h->pinfo.priority = priority;
3288 	ccb_h->path = path;
3289 	ccb_h->path_id = path->bus->path_id;
3290 	if (path->target)
3291 		ccb_h->target_id = path->target->target_id;
3292 	else
3293 		ccb_h->target_id = CAM_TARGET_WILDCARD;
3294 	if (path->device) {
3295 		ccb_h->target_lun = path->device->lun_id;
3296 		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3297 	} else {
3298 		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3299 	}
3300 	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3301 	ccb_h->flags = 0;
3302 }
3303 
3304 /* Path manipulation functions */
3305 cam_status
3306 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3307 		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3308 {
3309 	struct	   cam_path *path;
3310 	cam_status status;
3311 
3312 	path = (struct cam_path *)malloc(sizeof(*path), M_CAMXPT, M_NOWAIT);
3313 
3314 	if (path == NULL) {
3315 		status = CAM_RESRC_UNAVAIL;
3316 		return(status);
3317 	}
3318 	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3319 	if (status != CAM_REQ_CMP) {
3320 		free(path, M_CAMXPT);
3321 		path = NULL;
3322 	}
3323 	*new_path_ptr = path;
3324 	return (status);
3325 }
3326 
3327 cam_status
3328 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3329 			 struct cam_periph *periph, path_id_t path_id,
3330 			 target_id_t target_id, lun_id_t lun_id)
3331 {
3332 	struct	   cam_path *path;
3333 	struct	   cam_eb *bus = NULL;
3334 	cam_status status;
3335 	int	   need_unlock = 0;
3336 
3337 	path = (struct cam_path *)malloc(sizeof(*path), M_CAMXPT, M_WAITOK);
3338 
3339 	if (path_id != CAM_BUS_WILDCARD) {
3340 		bus = xpt_find_bus(path_id);
3341 		if (bus != NULL) {
3342 			need_unlock = 1;
3343 			CAM_SIM_LOCK(bus->sim);
3344 		}
3345 	}
3346 	status = xpt_compile_path(path, periph, path_id, target_id, lun_id);
3347 	if (need_unlock) {
3348 		CAM_SIM_UNLOCK(bus->sim);
3349 		xpt_release_bus(bus);
3350 	}
3351 	if (status != CAM_REQ_CMP) {
3352 		free(path, M_CAMXPT);
3353 		path = NULL;
3354 	}
3355 	*new_path_ptr = path;
3356 	return (status);
3357 }
3358 
3359 cam_status
3360 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3361 		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3362 {
3363 	struct	     cam_eb *bus;
3364 	struct	     cam_et *target;
3365 	struct	     cam_ed *device;
3366 	cam_status   status;
3367 
3368 	status = CAM_REQ_CMP;	/* Completed without error */
3369 	target = NULL;		/* Wildcarded */
3370 	device = NULL;		/* Wildcarded */
3371 
3372 	/*
3373 	 * We will potentially modify the EDT, so block interrupts
3374 	 * that may attempt to create cam paths.
3375 	 */
3376 	bus = xpt_find_bus(path_id);
3377 	if (bus == NULL) {
3378 		status = CAM_PATH_INVALID;
3379 	} else {
3380 		target = xpt_find_target(bus, target_id);
3381 		if (target == NULL) {
3382 			/* Create one */
3383 			struct cam_et *new_target;
3384 
3385 			new_target = xpt_alloc_target(bus, target_id);
3386 			if (new_target == NULL) {
3387 				status = CAM_RESRC_UNAVAIL;
3388 			} else {
3389 				target = new_target;
3390 			}
3391 		}
3392 		if (target != NULL) {
3393 			device = xpt_find_device(target, lun_id);
3394 			if (device == NULL) {
3395 				/* Create one */
3396 				struct cam_ed *new_device;
3397 
3398 				new_device =
3399 				    (*(bus->xport->alloc_device))(bus,
3400 								      target,
3401 								      lun_id);
3402 				if (new_device == NULL) {
3403 					status = CAM_RESRC_UNAVAIL;
3404 				} else {
3405 					device = new_device;
3406 				}
3407 			}
3408 		}
3409 	}
3410 
3411 	/*
3412 	 * Only touch the user's data if we are successful.
3413 	 */
3414 	if (status == CAM_REQ_CMP) {
3415 		new_path->periph = perph;
3416 		new_path->bus = bus;
3417 		new_path->target = target;
3418 		new_path->device = device;
3419 		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3420 	} else {
3421 		if (device != NULL)
3422 			xpt_release_device(device);
3423 		if (target != NULL)
3424 			xpt_release_target(target);
3425 		if (bus != NULL)
3426 			xpt_release_bus(bus);
3427 	}
3428 	return (status);
3429 }
3430 
3431 void
3432 xpt_release_path(struct cam_path *path)
3433 {
3434 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3435 	if (path->device != NULL) {
3436 		xpt_release_device(path->device);
3437 		path->device = NULL;
3438 	}
3439 	if (path->target != NULL) {
3440 		xpt_release_target(path->target);
3441 		path->target = NULL;
3442 	}
3443 	if (path->bus != NULL) {
3444 		xpt_release_bus(path->bus);
3445 		path->bus = NULL;
3446 	}
3447 }
3448 
3449 void
3450 xpt_free_path(struct cam_path *path)
3451 {
3452 
3453 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3454 	xpt_release_path(path);
3455 	free(path, M_CAMXPT);
3456 }
3457 
3458 void
3459 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3460     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3461 {
3462 
3463 	mtx_lock(&xsoftc.xpt_topo_lock);
3464 	if (bus_ref) {
3465 		if (path->bus)
3466 			*bus_ref = path->bus->refcount;
3467 		else
3468 			*bus_ref = 0;
3469 	}
3470 	mtx_unlock(&xsoftc.xpt_topo_lock);
3471 	if (periph_ref) {
3472 		if (path->periph)
3473 			*periph_ref = path->periph->refcount;
3474 		else
3475 			*periph_ref = 0;
3476 	}
3477 	if (target_ref) {
3478 		if (path->target)
3479 			*target_ref = path->target->refcount;
3480 		else
3481 			*target_ref = 0;
3482 	}
3483 	if (device_ref) {
3484 		if (path->device)
3485 			*device_ref = path->device->refcount;
3486 		else
3487 			*device_ref = 0;
3488 	}
3489 }
3490 
3491 /*
3492  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3493  * in path1, 2 for match with wildcards in path2.
3494  */
3495 int
3496 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3497 {
3498 	int retval = 0;
3499 
3500 	if (path1->bus != path2->bus) {
3501 		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3502 			retval = 1;
3503 		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3504 			retval = 2;
3505 		else
3506 			return (-1);
3507 	}
3508 	if (path1->target != path2->target) {
3509 		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3510 			if (retval == 0)
3511 				retval = 1;
3512 		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3513 			retval = 2;
3514 		else
3515 			return (-1);
3516 	}
3517 	if (path1->device != path2->device) {
3518 		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3519 			if (retval == 0)
3520 				retval = 1;
3521 		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3522 			retval = 2;
3523 		else
3524 			return (-1);
3525 	}
3526 	return (retval);
3527 }
3528 
3529 void
3530 xpt_print_path(struct cam_path *path)
3531 {
3532 
3533 	if (path == NULL)
3534 		printf("(nopath): ");
3535 	else {
3536 		if (path->periph != NULL)
3537 			printf("(%s%d:", path->periph->periph_name,
3538 			       path->periph->unit_number);
3539 		else
3540 			printf("(noperiph:");
3541 
3542 		if (path->bus != NULL)
3543 			printf("%s%d:%d:", path->bus->sim->sim_name,
3544 			       path->bus->sim->unit_number,
3545 			       path->bus->sim->bus_id);
3546 		else
3547 			printf("nobus:");
3548 
3549 		if (path->target != NULL)
3550 			printf("%d:", path->target->target_id);
3551 		else
3552 			printf("X:");
3553 
3554 		if (path->device != NULL)
3555 			printf("%d): ", path->device->lun_id);
3556 		else
3557 			printf("X): ");
3558 	}
3559 }
3560 
3561 void
3562 xpt_print(struct cam_path *path, const char *fmt, ...)
3563 {
3564 	va_list ap;
3565 	xpt_print_path(path);
3566 	va_start(ap, fmt);
3567 	vprintf(fmt, ap);
3568 	va_end(ap);
3569 }
3570 
3571 int
3572 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3573 {
3574 	struct sbuf sb;
3575 
3576 #ifdef INVARIANTS
3577 	if (path != NULL && path->bus != NULL)
3578 		mtx_assert(path->bus->sim->mtx, MA_OWNED);
3579 #endif
3580 
3581 	sbuf_new(&sb, str, str_len, 0);
3582 
3583 	if (path == NULL)
3584 		sbuf_printf(&sb, "(nopath): ");
3585 	else {
3586 		if (path->periph != NULL)
3587 			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3588 				    path->periph->unit_number);
3589 		else
3590 			sbuf_printf(&sb, "(noperiph:");
3591 
3592 		if (path->bus != NULL)
3593 			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3594 				    path->bus->sim->unit_number,
3595 				    path->bus->sim->bus_id);
3596 		else
3597 			sbuf_printf(&sb, "nobus:");
3598 
3599 		if (path->target != NULL)
3600 			sbuf_printf(&sb, "%d:", path->target->target_id);
3601 		else
3602 			sbuf_printf(&sb, "X:");
3603 
3604 		if (path->device != NULL)
3605 			sbuf_printf(&sb, "%d): ", path->device->lun_id);
3606 		else
3607 			sbuf_printf(&sb, "X): ");
3608 	}
3609 	sbuf_finish(&sb);
3610 
3611 	return(sbuf_len(&sb));
3612 }
3613 
3614 path_id_t
3615 xpt_path_path_id(struct cam_path *path)
3616 {
3617 	return(path->bus->path_id);
3618 }
3619 
3620 target_id_t
3621 xpt_path_target_id(struct cam_path *path)
3622 {
3623 	if (path->target != NULL)
3624 		return (path->target->target_id);
3625 	else
3626 		return (CAM_TARGET_WILDCARD);
3627 }
3628 
3629 lun_id_t
3630 xpt_path_lun_id(struct cam_path *path)
3631 {
3632 	if (path->device != NULL)
3633 		return (path->device->lun_id);
3634 	else
3635 		return (CAM_LUN_WILDCARD);
3636 }
3637 
3638 struct cam_sim *
3639 xpt_path_sim(struct cam_path *path)
3640 {
3641 
3642 	return (path->bus->sim);
3643 }
3644 
3645 struct cam_periph*
3646 xpt_path_periph(struct cam_path *path)
3647 {
3648 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
3649 
3650 	return (path->periph);
3651 }
3652 
3653 int
3654 xpt_path_legacy_ata_id(struct cam_path *path)
3655 {
3656 	struct cam_eb *bus;
3657 	int bus_id;
3658 
3659 	if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3660 	    strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3661 	    strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3662 	    strcmp(path->bus->sim->sim_name, "siisch") != 0)
3663 		return (-1);
3664 
3665 	if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3666 	    path->bus->sim->unit_number < 2) {
3667 		bus_id = path->bus->sim->unit_number;
3668 	} else {
3669 		bus_id = 2;
3670 		xpt_lock_buses();
3671 		TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3672 			if (bus == path->bus)
3673 				break;
3674 			if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3675 			     bus->sim->unit_number >= 2) ||
3676 			    strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3677 			    strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3678 			    strcmp(bus->sim->sim_name, "siisch") == 0)
3679 				bus_id++;
3680 		}
3681 		xpt_unlock_buses();
3682 	}
3683 	if (path->target != NULL) {
3684 		if (path->target->target_id < 2)
3685 			return (bus_id * 2 + path->target->target_id);
3686 		else
3687 			return (-1);
3688 	} else
3689 		return (bus_id * 2);
3690 }
3691 
3692 /*
3693  * Release a CAM control block for the caller.  Remit the cost of the structure
3694  * to the device referenced by the path.  If the this device had no 'credits'
3695  * and peripheral drivers have registered async callbacks for this notification
3696  * call them now.
3697  */
3698 void
3699 xpt_release_ccb(union ccb *free_ccb)
3700 {
3701 	struct	 cam_path *path;
3702 	struct	 cam_ed *device;
3703 	struct	 cam_eb *bus;
3704 	struct   cam_sim *sim;
3705 
3706 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3707 	path = free_ccb->ccb_h.path;
3708 	device = path->device;
3709 	bus = path->bus;
3710 	sim = bus->sim;
3711 
3712 	mtx_assert(sim->mtx, MA_OWNED);
3713 
3714 	cam_ccbq_release_opening(&device->ccbq);
3715 	if (device->flags & CAM_DEV_RESIZE_QUEUE_NEEDED) {
3716 		device->flags &= ~CAM_DEV_RESIZE_QUEUE_NEEDED;
3717 		cam_ccbq_resize(&device->ccbq,
3718 		    device->ccbq.dev_openings + device->ccbq.dev_active);
3719 	}
3720 	if (sim->ccb_count > sim->max_ccbs) {
3721 		xpt_free_ccb(free_ccb);
3722 		sim->ccb_count--;
3723 	} else {
3724 		SLIST_INSERT_HEAD(&sim->ccb_freeq, &free_ccb->ccb_h,
3725 		    xpt_links.sle);
3726 	}
3727 	if (sim->devq == NULL) {
3728 		return;
3729 	}
3730 	sim->devq->alloc_openings++;
3731 	sim->devq->alloc_active--;
3732 	if (device_is_alloc_queued(device) == 0)
3733 		xpt_schedule_dev_allocq(bus, device);
3734 	xpt_run_dev_allocq(bus);
3735 }
3736 
3737 /* Functions accessed by SIM drivers */
3738 
3739 static struct xpt_xport xport_default = {
3740 	.alloc_device = xpt_alloc_device_default,
3741 	.action = xpt_action_default,
3742 	.async = xpt_dev_async_default,
3743 };
3744 
3745 /*
3746  * A sim structure, listing the SIM entry points and instance
3747  * identification info is passed to xpt_bus_register to hook the SIM
3748  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3749  * for this new bus and places it in the array of busses and assigns
3750  * it a path_id.  The path_id may be influenced by "hard wiring"
3751  * information specified by the user.  Once interrupt services are
3752  * available, the bus will be probed.
3753  */
3754 int32_t
3755 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3756 {
3757 	struct cam_eb *new_bus;
3758 	struct cam_eb *old_bus;
3759 	struct ccb_pathinq cpi;
3760 	struct cam_path *path;
3761 	cam_status status;
3762 
3763 	mtx_assert(sim->mtx, MA_OWNED);
3764 
3765 	sim->bus_id = bus;
3766 	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3767 					  M_CAMXPT, M_NOWAIT);
3768 	if (new_bus == NULL) {
3769 		/* Couldn't satisfy request */
3770 		return (CAM_RESRC_UNAVAIL);
3771 	}
3772 	path = (struct cam_path *)malloc(sizeof(*path), M_CAMXPT, M_NOWAIT);
3773 	if (path == NULL) {
3774 		free(new_bus, M_CAMXPT);
3775 		return (CAM_RESRC_UNAVAIL);
3776 	}
3777 
3778 	if (strcmp(sim->sim_name, "xpt") != 0) {
3779 		sim->path_id =
3780 		    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3781 	}
3782 
3783 	TAILQ_INIT(&new_bus->et_entries);
3784 	new_bus->path_id = sim->path_id;
3785 	cam_sim_hold(sim);
3786 	new_bus->sim = sim;
3787 	timevalclear(&new_bus->last_reset);
3788 	new_bus->flags = 0;
3789 	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3790 	new_bus->generation = 0;
3791 
3792 	mtx_lock(&xsoftc.xpt_topo_lock);
3793 	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3794 	while (old_bus != NULL
3795 	    && old_bus->path_id < new_bus->path_id)
3796 		old_bus = TAILQ_NEXT(old_bus, links);
3797 	if (old_bus != NULL)
3798 		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3799 	else
3800 		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3801 	xsoftc.bus_generation++;
3802 	mtx_unlock(&xsoftc.xpt_topo_lock);
3803 
3804 	/*
3805 	 * Set a default transport so that a PATH_INQ can be issued to
3806 	 * the SIM.  This will then allow for probing and attaching of
3807 	 * a more appropriate transport.
3808 	 */
3809 	new_bus->xport = &xport_default;
3810 
3811 	status = xpt_compile_path(path, /*periph*/NULL, sim->path_id,
3812 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3813 	if (status != CAM_REQ_CMP)
3814 		printf("xpt_compile_path returned %d\n", status);
3815 
3816 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3817 	cpi.ccb_h.func_code = XPT_PATH_INQ;
3818 	xpt_action((union ccb *)&cpi);
3819 
3820 	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3821 		switch (cpi.transport) {
3822 		case XPORT_SPI:
3823 		case XPORT_SAS:
3824 		case XPORT_FC:
3825 		case XPORT_USB:
3826 		case XPORT_ISCSI:
3827 		case XPORT_PPB:
3828 			new_bus->xport = scsi_get_xport();
3829 			break;
3830 		case XPORT_ATA:
3831 		case XPORT_SATA:
3832 			new_bus->xport = ata_get_xport();
3833 			break;
3834 		default:
3835 			new_bus->xport = &xport_default;
3836 			break;
3837 		}
3838 	}
3839 
3840 	/* Notify interested parties */
3841 	if (sim->path_id != CAM_XPT_PATH_ID) {
3842 		union	ccb *scan_ccb;
3843 
3844 		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3845 		/* Initiate bus rescan. */
3846 		scan_ccb = xpt_alloc_ccb_nowait();
3847 		scan_ccb->ccb_h.path = path;
3848 		scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3849 		scan_ccb->crcn.flags = 0;
3850 		xpt_rescan(scan_ccb);
3851 	} else
3852 		xpt_free_path(path);
3853 	return (CAM_SUCCESS);
3854 }
3855 
3856 int32_t
3857 xpt_bus_deregister(path_id_t pathid)
3858 {
3859 	struct cam_path bus_path;
3860 	cam_status status;
3861 
3862 	status = xpt_compile_path(&bus_path, NULL, pathid,
3863 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3864 	if (status != CAM_REQ_CMP)
3865 		return (status);
3866 
3867 	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3868 	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3869 
3870 	/* Release the reference count held while registered. */
3871 	xpt_release_bus(bus_path.bus);
3872 	xpt_release_path(&bus_path);
3873 
3874 	return (CAM_REQ_CMP);
3875 }
3876 
3877 static path_id_t
3878 xptnextfreepathid(void)
3879 {
3880 	struct cam_eb *bus;
3881 	path_id_t pathid;
3882 	const char *strval;
3883 
3884 	pathid = 0;
3885 	mtx_lock(&xsoftc.xpt_topo_lock);
3886 	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3887 retry:
3888 	/* Find an unoccupied pathid */
3889 	while (bus != NULL && bus->path_id <= pathid) {
3890 		if (bus->path_id == pathid)
3891 			pathid++;
3892 		bus = TAILQ_NEXT(bus, links);
3893 	}
3894 	mtx_unlock(&xsoftc.xpt_topo_lock);
3895 
3896 	/*
3897 	 * Ensure that this pathid is not reserved for
3898 	 * a bus that may be registered in the future.
3899 	 */
3900 	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
3901 		++pathid;
3902 		/* Start the search over */
3903 		mtx_lock(&xsoftc.xpt_topo_lock);
3904 		goto retry;
3905 	}
3906 	return (pathid);
3907 }
3908 
3909 static path_id_t
3910 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
3911 {
3912 	path_id_t pathid;
3913 	int i, dunit, val;
3914 	char buf[32];
3915 	const char *dname;
3916 
3917 	pathid = CAM_XPT_PATH_ID;
3918 	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
3919 	i = 0;
3920 	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
3921 		if (strcmp(dname, "scbus")) {
3922 			/* Avoid a bit of foot shooting. */
3923 			continue;
3924 		}
3925 		if (dunit < 0)		/* unwired?! */
3926 			continue;
3927 		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
3928 			if (sim_bus == val) {
3929 				pathid = dunit;
3930 				break;
3931 			}
3932 		} else if (sim_bus == 0) {
3933 			/* Unspecified matches bus 0 */
3934 			pathid = dunit;
3935 			break;
3936 		} else {
3937 			printf("Ambiguous scbus configuration for %s%d "
3938 			       "bus %d, cannot wire down.  The kernel "
3939 			       "config entry for scbus%d should "
3940 			       "specify a controller bus.\n"
3941 			       "Scbus will be assigned dynamically.\n",
3942 			       sim_name, sim_unit, sim_bus, dunit);
3943 			break;
3944 		}
3945 	}
3946 
3947 	if (pathid == CAM_XPT_PATH_ID)
3948 		pathid = xptnextfreepathid();
3949 	return (pathid);
3950 }
3951 
3952 void
3953 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
3954 {
3955 	struct cam_eb *bus;
3956 	struct cam_et *target, *next_target;
3957 	struct cam_ed *device, *next_device;
3958 
3959 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
3960 
3961 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_async\n"));
3962 
3963 	/*
3964 	 * Most async events come from a CAM interrupt context.  In
3965 	 * a few cases, the error recovery code at the peripheral layer,
3966 	 * which may run from our SWI or a process context, may signal
3967 	 * deferred events with a call to xpt_async.
3968 	 */
3969 
3970 	bus = path->bus;
3971 
3972 	if (async_code == AC_BUS_RESET) {
3973 		/* Update our notion of when the last reset occurred */
3974 		microtime(&bus->last_reset);
3975 	}
3976 
3977 	for (target = TAILQ_FIRST(&bus->et_entries);
3978 	     target != NULL;
3979 	     target = next_target) {
3980 
3981 		next_target = TAILQ_NEXT(target, links);
3982 
3983 		if (path->target != target
3984 		 && path->target->target_id != CAM_TARGET_WILDCARD
3985 		 && target->target_id != CAM_TARGET_WILDCARD)
3986 			continue;
3987 
3988 		if (async_code == AC_SENT_BDR) {
3989 			/* Update our notion of when the last reset occurred */
3990 			microtime(&path->target->last_reset);
3991 		}
3992 
3993 		for (device = TAILQ_FIRST(&target->ed_entries);
3994 		     device != NULL;
3995 		     device = next_device) {
3996 
3997 			next_device = TAILQ_NEXT(device, links);
3998 
3999 			if (path->device != device
4000 			 && path->device->lun_id != CAM_LUN_WILDCARD
4001 			 && device->lun_id != CAM_LUN_WILDCARD)
4002 				continue;
4003 			/*
4004 			 * The async callback could free the device.
4005 			 * If it is a broadcast async, it doesn't hold
4006 			 * device reference, so take our own reference.
4007 			 */
4008 			xpt_acquire_device(device);
4009 			(*(bus->xport->async))(async_code, bus,
4010 					       target, device,
4011 					       async_arg);
4012 
4013 			xpt_async_bcast(&device->asyncs, async_code,
4014 					path, async_arg);
4015 			xpt_release_device(device);
4016 		}
4017 	}
4018 
4019 	/*
4020 	 * If this wasn't a fully wildcarded async, tell all
4021 	 * clients that want all async events.
4022 	 */
4023 	if (bus != xpt_periph->path->bus)
4024 		xpt_async_bcast(&xpt_periph->path->device->asyncs, async_code,
4025 				path, async_arg);
4026 }
4027 
4028 static void
4029 xpt_async_bcast(struct async_list *async_head,
4030 		u_int32_t async_code,
4031 		struct cam_path *path, void *async_arg)
4032 {
4033 	struct async_node *cur_entry;
4034 
4035 	cur_entry = SLIST_FIRST(async_head);
4036 	while (cur_entry != NULL) {
4037 		struct async_node *next_entry;
4038 		/*
4039 		 * Grab the next list entry before we call the current
4040 		 * entry's callback.  This is because the callback function
4041 		 * can delete its async callback entry.
4042 		 */
4043 		next_entry = SLIST_NEXT(cur_entry, links);
4044 		if ((cur_entry->event_enable & async_code) != 0)
4045 			cur_entry->callback(cur_entry->callback_arg,
4046 					    async_code, path,
4047 					    async_arg);
4048 		cur_entry = next_entry;
4049 	}
4050 }
4051 
4052 static void
4053 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4054 		      struct cam_et *target, struct cam_ed *device,
4055 		      void *async_arg)
4056 {
4057 	printf("%s called\n", __func__);
4058 }
4059 
4060 u_int32_t
4061 xpt_freeze_devq_rl(struct cam_path *path, cam_rl rl, u_int count)
4062 {
4063 	struct cam_ed *dev = path->device;
4064 
4065 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
4066 	dev->sim->devq->alloc_openings +=
4067 	    cam_ccbq_freeze(&dev->ccbq, rl, count);
4068 	/* Remove frozen device from allocq. */
4069 	if (device_is_alloc_queued(dev) &&
4070 	    cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
4071 	     CAMQ_GET_PRIO(&dev->drvq)))) {
4072 		camq_remove(&dev->sim->devq->alloc_queue,
4073 		    dev->alloc_ccb_entry.pinfo.index);
4074 	}
4075 	/* Remove frozen device from sendq. */
4076 	if (device_is_send_queued(dev) &&
4077 	    cam_ccbq_frozen_top(&dev->ccbq)) {
4078 		camq_remove(&dev->sim->devq->send_queue,
4079 		    dev->send_ccb_entry.pinfo.index);
4080 	}
4081 	return (dev->ccbq.queue.qfrozen_cnt[rl]);
4082 }
4083 
4084 u_int32_t
4085 xpt_freeze_devq(struct cam_path *path, u_int count)
4086 {
4087 
4088 	return (xpt_freeze_devq_rl(path, 0, count));
4089 }
4090 
4091 u_int32_t
4092 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4093 {
4094 
4095 	mtx_assert(sim->mtx, MA_OWNED);
4096 	sim->devq->send_queue.qfrozen_cnt[0] += count;
4097 	return (sim->devq->send_queue.qfrozen_cnt[0]);
4098 }
4099 
4100 static void
4101 xpt_release_devq_timeout(void *arg)
4102 {
4103 	struct cam_ed *device;
4104 
4105 	device = (struct cam_ed *)arg;
4106 
4107 	xpt_release_devq_device(device, /*rl*/0, /*count*/1, /*run_queue*/TRUE);
4108 }
4109 
4110 void
4111 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4112 {
4113 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
4114 
4115 	xpt_release_devq_device(path->device, /*rl*/0, count, run_queue);
4116 }
4117 
4118 void
4119 xpt_release_devq_rl(struct cam_path *path, cam_rl rl, u_int count, int run_queue)
4120 {
4121 	mtx_assert(path->bus->sim->mtx, MA_OWNED);
4122 
4123 	xpt_release_devq_device(path->device, rl, count, run_queue);
4124 }
4125 
4126 static void
4127 xpt_release_devq_device(struct cam_ed *dev, cam_rl rl, u_int count, int run_queue)
4128 {
4129 
4130 	if (count > dev->ccbq.queue.qfrozen_cnt[rl]) {
4131 #ifdef INVARIANTS
4132 		printf("xpt_release_devq(%d): requested %u > present %u\n",
4133 		    rl, count, dev->ccbq.queue.qfrozen_cnt[rl]);
4134 #endif
4135 		count = dev->ccbq.queue.qfrozen_cnt[rl];
4136 	}
4137 	dev->sim->devq->alloc_openings -=
4138 	    cam_ccbq_release(&dev->ccbq, rl, count);
4139 	if (cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
4140 	    CAMQ_GET_PRIO(&dev->drvq))) == 0) {
4141 		if (xpt_schedule_dev_allocq(dev->target->bus, dev))
4142 			xpt_run_dev_allocq(dev->target->bus);
4143 	}
4144 	if (cam_ccbq_frozen_top(&dev->ccbq) == 0) {
4145 		/*
4146 		 * No longer need to wait for a successful
4147 		 * command completion.
4148 		 */
4149 		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4150 		/*
4151 		 * Remove any timeouts that might be scheduled
4152 		 * to release this queue.
4153 		 */
4154 		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4155 			callout_stop(&dev->callout);
4156 			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4157 		}
4158 		if (run_queue == 0)
4159 			return;
4160 		/*
4161 		 * Now that we are unfrozen schedule the
4162 		 * device so any pending transactions are
4163 		 * run.
4164 		 */
4165 		if (xpt_schedule_dev_sendq(dev->target->bus, dev))
4166 			xpt_run_dev_sendq(dev->target->bus);
4167 	}
4168 }
4169 
4170 void
4171 xpt_release_simq(struct cam_sim *sim, int run_queue)
4172 {
4173 	struct	camq *sendq;
4174 
4175 	mtx_assert(sim->mtx, MA_OWNED);
4176 	sendq = &(sim->devq->send_queue);
4177 	if (sendq->qfrozen_cnt[0] <= 0) {
4178 #ifdef INVARIANTS
4179 		printf("xpt_release_simq: requested 1 > present %u\n",
4180 		    sendq->qfrozen_cnt[0]);
4181 #endif
4182 	} else
4183 		sendq->qfrozen_cnt[0]--;
4184 	if (sendq->qfrozen_cnt[0] == 0) {
4185 		/*
4186 		 * If there is a timeout scheduled to release this
4187 		 * sim queue, remove it.  The queue frozen count is
4188 		 * already at 0.
4189 		 */
4190 		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4191 			callout_stop(&sim->callout);
4192 			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4193 		}
4194 		if (run_queue) {
4195 			struct cam_eb *bus;
4196 
4197 			/*
4198 			 * Now that we are unfrozen run the send queue.
4199 			 */
4200 			bus = xpt_find_bus(sim->path_id);
4201 			xpt_run_dev_sendq(bus);
4202 			xpt_release_bus(bus);
4203 		}
4204 	}
4205 }
4206 
4207 /*
4208  * XXX Appears to be unused.
4209  */
4210 static void
4211 xpt_release_simq_timeout(void *arg)
4212 {
4213 	struct cam_sim *sim;
4214 
4215 	sim = (struct cam_sim *)arg;
4216 	xpt_release_simq(sim, /* run_queue */ TRUE);
4217 }
4218 
4219 void
4220 xpt_done(union ccb *done_ccb)
4221 {
4222 	struct cam_sim *sim;
4223 	int	first;
4224 
4225 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4226 	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) {
4227 		/*
4228 		 * Queue up the request for handling by our SWI handler
4229 		 * any of the "non-immediate" type of ccbs.
4230 		 */
4231 		sim = done_ccb->ccb_h.path->bus->sim;
4232 		TAILQ_INSERT_TAIL(&sim->sim_doneq, &done_ccb->ccb_h,
4233 		    sim_links.tqe);
4234 		done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4235 		if ((sim->flags & (CAM_SIM_ON_DONEQ | CAM_SIM_POLLED)) == 0) {
4236 			mtx_lock(&cam_simq_lock);
4237 			first = TAILQ_EMPTY(&cam_simq);
4238 			TAILQ_INSERT_TAIL(&cam_simq, sim, links);
4239 			mtx_unlock(&cam_simq_lock);
4240 			sim->flags |= CAM_SIM_ON_DONEQ;
4241 			if (first)
4242 				swi_sched(cambio_ih, 0);
4243 		}
4244 	}
4245 }
4246 
4247 union ccb *
4248 xpt_alloc_ccb()
4249 {
4250 	union ccb *new_ccb;
4251 
4252 	new_ccb = malloc(sizeof(*new_ccb), M_CAMXPT, M_ZERO|M_WAITOK);
4253 	return (new_ccb);
4254 }
4255 
4256 union ccb *
4257 xpt_alloc_ccb_nowait()
4258 {
4259 	union ccb *new_ccb;
4260 
4261 	new_ccb = malloc(sizeof(*new_ccb), M_CAMXPT, M_ZERO|M_NOWAIT);
4262 	return (new_ccb);
4263 }
4264 
4265 void
4266 xpt_free_ccb(union ccb *free_ccb)
4267 {
4268 	free(free_ccb, M_CAMXPT);
4269 }
4270 
4271 
4272 
4273 /* Private XPT functions */
4274 
4275 /*
4276  * Get a CAM control block for the caller. Charge the structure to the device
4277  * referenced by the path.  If the this device has no 'credits' then the
4278  * device already has the maximum number of outstanding operations under way
4279  * and we return NULL. If we don't have sufficient resources to allocate more
4280  * ccbs, we also return NULL.
4281  */
4282 static union ccb *
4283 xpt_get_ccb(struct cam_ed *device)
4284 {
4285 	union ccb *new_ccb;
4286 	struct cam_sim *sim;
4287 
4288 	sim = device->sim;
4289 	if ((new_ccb = (union ccb *)SLIST_FIRST(&sim->ccb_freeq)) == NULL) {
4290 		new_ccb = xpt_alloc_ccb_nowait();
4291                 if (new_ccb == NULL) {
4292 			return (NULL);
4293 		}
4294 		if ((sim->flags & CAM_SIM_MPSAFE) == 0)
4295 			callout_handle_init(&new_ccb->ccb_h.timeout_ch);
4296 		SLIST_INSERT_HEAD(&sim->ccb_freeq, &new_ccb->ccb_h,
4297 				  xpt_links.sle);
4298 		sim->ccb_count++;
4299 	}
4300 	cam_ccbq_take_opening(&device->ccbq);
4301 	SLIST_REMOVE_HEAD(&sim->ccb_freeq, xpt_links.sle);
4302 	return (new_ccb);
4303 }
4304 
4305 static void
4306 xpt_release_bus(struct cam_eb *bus)
4307 {
4308 
4309 	mtx_lock(&xsoftc.xpt_topo_lock);
4310 	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4311 	if ((--bus->refcount == 0)
4312 	 && (TAILQ_FIRST(&bus->et_entries) == NULL)) {
4313 		TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4314 		xsoftc.bus_generation++;
4315 		mtx_unlock(&xsoftc.xpt_topo_lock);
4316 		cam_sim_release(bus->sim);
4317 		free(bus, M_CAMXPT);
4318 	} else
4319 		mtx_unlock(&xsoftc.xpt_topo_lock);
4320 }
4321 
4322 static struct cam_et *
4323 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4324 {
4325 	struct cam_et *target;
4326 
4327 	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4328 					 M_NOWAIT|M_ZERO);
4329 	if (target != NULL) {
4330 		struct cam_et *cur_target;
4331 
4332 		TAILQ_INIT(&target->ed_entries);
4333 		target->bus = bus;
4334 		target->target_id = target_id;
4335 		target->refcount = 1;
4336 		target->generation = 0;
4337 		target->luns = NULL;
4338 		timevalclear(&target->last_reset);
4339 		/*
4340 		 * Hold a reference to our parent bus so it
4341 		 * will not go away before we do.
4342 		 */
4343 		mtx_lock(&xsoftc.xpt_topo_lock);
4344 		bus->refcount++;
4345 		mtx_unlock(&xsoftc.xpt_topo_lock);
4346 
4347 		/* Insertion sort into our bus's target list */
4348 		cur_target = TAILQ_FIRST(&bus->et_entries);
4349 		while (cur_target != NULL && cur_target->target_id < target_id)
4350 			cur_target = TAILQ_NEXT(cur_target, links);
4351 
4352 		if (cur_target != NULL) {
4353 			TAILQ_INSERT_BEFORE(cur_target, target, links);
4354 		} else {
4355 			TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4356 		}
4357 		bus->generation++;
4358 	}
4359 	return (target);
4360 }
4361 
4362 static void
4363 xpt_release_target(struct cam_et *target)
4364 {
4365 
4366 	if (target->refcount == 1) {
4367 		if (TAILQ_FIRST(&target->ed_entries) == NULL) {
4368 			TAILQ_REMOVE(&target->bus->et_entries, target, links);
4369 			target->bus->generation++;
4370 			xpt_release_bus(target->bus);
4371 			if (target->luns)
4372 				free(target->luns, M_CAMXPT);
4373 			free(target, M_CAMXPT);
4374 		}
4375 	} else
4376 		target->refcount--;
4377 }
4378 
4379 static struct cam_ed *
4380 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4381 			 lun_id_t lun_id)
4382 {
4383 	struct cam_ed *device, *cur_device;
4384 
4385 	device = xpt_alloc_device(bus, target, lun_id);
4386 	if (device == NULL)
4387 		return (NULL);
4388 
4389 	device->mintags = 1;
4390 	device->maxtags = 1;
4391 	bus->sim->max_ccbs += device->ccbq.devq_openings;
4392 	cur_device = TAILQ_FIRST(&target->ed_entries);
4393 	while (cur_device != NULL && cur_device->lun_id < lun_id)
4394 		cur_device = TAILQ_NEXT(cur_device, links);
4395 	if (cur_device != NULL) {
4396 		TAILQ_INSERT_BEFORE(cur_device, device, links);
4397 	} else {
4398 		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4399 	}
4400 	target->generation++;
4401 
4402 	return (device);
4403 }
4404 
4405 struct cam_ed *
4406 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4407 {
4408 	struct	   cam_ed *device;
4409 	struct	   cam_devq *devq;
4410 	cam_status status;
4411 
4412 	/* Make space for us in the device queue on our bus */
4413 	devq = bus->sim->devq;
4414 	status = cam_devq_resize(devq, devq->alloc_queue.array_size + 1);
4415 
4416 	if (status != CAM_REQ_CMP) {
4417 		device = NULL;
4418 	} else {
4419 		device = (struct cam_ed *)malloc(sizeof(*device),
4420 						 M_CAMXPT, M_NOWAIT|M_ZERO);
4421 	}
4422 
4423 	if (device != NULL) {
4424 		cam_init_pinfo(&device->alloc_ccb_entry.pinfo);
4425 		device->alloc_ccb_entry.device = device;
4426 		cam_init_pinfo(&device->send_ccb_entry.pinfo);
4427 		device->send_ccb_entry.device = device;
4428 		device->target = target;
4429 		device->lun_id = lun_id;
4430 		device->sim = bus->sim;
4431 		/* Initialize our queues */
4432 		if (camq_init(&device->drvq, 0) != 0) {
4433 			free(device, M_CAMXPT);
4434 			return (NULL);
4435 		}
4436 		if (cam_ccbq_init(&device->ccbq,
4437 				  bus->sim->max_dev_openings) != 0) {
4438 			camq_fini(&device->drvq);
4439 			free(device, M_CAMXPT);
4440 			return (NULL);
4441 		}
4442 		SLIST_INIT(&device->asyncs);
4443 		SLIST_INIT(&device->periphs);
4444 		device->generation = 0;
4445 		device->owner = NULL;
4446 		device->flags = CAM_DEV_UNCONFIGURED;
4447 		device->tag_delay_count = 0;
4448 		device->tag_saved_openings = 0;
4449 		device->refcount = 1;
4450 		callout_init_mtx(&device->callout, bus->sim->mtx, 0);
4451 
4452 		/*
4453 		 * Hold a reference to our parent target so it
4454 		 * will not go away before we do.
4455 		 */
4456 		target->refcount++;
4457 
4458 	}
4459 	return (device);
4460 }
4461 
4462 void
4463 xpt_acquire_device(struct cam_ed *device)
4464 {
4465 
4466 	device->refcount++;
4467 }
4468 
4469 void
4470 xpt_release_device(struct cam_ed *device)
4471 {
4472 
4473 	if (device->refcount == 1) {
4474 		struct cam_devq *devq;
4475 
4476 		if (device->alloc_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX
4477 		 || device->send_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX)
4478 			panic("Removing device while still queued for ccbs");
4479 
4480 		if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4481 			callout_stop(&device->callout);
4482 
4483 		TAILQ_REMOVE(&device->target->ed_entries, device,links);
4484 		device->target->generation++;
4485 		device->target->bus->sim->max_ccbs -= device->ccbq.devq_openings;
4486 		/* Release our slot in the devq */
4487 		devq = device->target->bus->sim->devq;
4488 		cam_devq_resize(devq, devq->alloc_queue.array_size - 1);
4489 		camq_fini(&device->drvq);
4490 		cam_ccbq_fini(&device->ccbq);
4491 		xpt_release_target(device->target);
4492 		free(device, M_CAMXPT);
4493 	} else
4494 		device->refcount--;
4495 }
4496 
4497 u_int32_t
4498 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4499 {
4500 	int	diff;
4501 	int	result;
4502 	struct	cam_ed *dev;
4503 
4504 	dev = path->device;
4505 
4506 	diff = newopenings - (dev->ccbq.dev_active + dev->ccbq.dev_openings);
4507 	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4508 	if (result == CAM_REQ_CMP && (diff < 0)) {
4509 		dev->flags |= CAM_DEV_RESIZE_QUEUE_NEEDED;
4510 	}
4511 	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4512 	 || (dev->inq_flags & SID_CmdQue) != 0)
4513 		dev->tag_saved_openings = newopenings;
4514 	/* Adjust the global limit */
4515 	dev->sim->max_ccbs += diff;
4516 	return (result);
4517 }
4518 
4519 static struct cam_eb *
4520 xpt_find_bus(path_id_t path_id)
4521 {
4522 	struct cam_eb *bus;
4523 
4524 	mtx_lock(&xsoftc.xpt_topo_lock);
4525 	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4526 	     bus != NULL;
4527 	     bus = TAILQ_NEXT(bus, links)) {
4528 		if (bus->path_id == path_id) {
4529 			bus->refcount++;
4530 			break;
4531 		}
4532 	}
4533 	mtx_unlock(&xsoftc.xpt_topo_lock);
4534 	return (bus);
4535 }
4536 
4537 static struct cam_et *
4538 xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4539 {
4540 	struct cam_et *target;
4541 
4542 	for (target = TAILQ_FIRST(&bus->et_entries);
4543 	     target != NULL;
4544 	     target = TAILQ_NEXT(target, links)) {
4545 		if (target->target_id == target_id) {
4546 			target->refcount++;
4547 			break;
4548 		}
4549 	}
4550 	return (target);
4551 }
4552 
4553 static struct cam_ed *
4554 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4555 {
4556 	struct cam_ed *device;
4557 
4558 	for (device = TAILQ_FIRST(&target->ed_entries);
4559 	     device != NULL;
4560 	     device = TAILQ_NEXT(device, links)) {
4561 		if (device->lun_id == lun_id) {
4562 			device->refcount++;
4563 			break;
4564 		}
4565 	}
4566 	return (device);
4567 }
4568 
4569 void
4570 xpt_start_tags(struct cam_path *path)
4571 {
4572 	struct ccb_relsim crs;
4573 	struct cam_ed *device;
4574 	struct cam_sim *sim;
4575 	int    newopenings;
4576 
4577 	device = path->device;
4578 	sim = path->bus->sim;
4579 	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4580 	xpt_freeze_devq(path, /*count*/1);
4581 	device->inq_flags |= SID_CmdQue;
4582 	if (device->tag_saved_openings != 0)
4583 		newopenings = device->tag_saved_openings;
4584 	else
4585 		newopenings = min(device->maxtags,
4586 				  sim->max_tagged_dev_openings);
4587 	xpt_dev_ccbq_resize(path, newopenings);
4588 	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4589 	crs.ccb_h.func_code = XPT_REL_SIMQ;
4590 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4591 	crs.openings
4592 	    = crs.release_timeout
4593 	    = crs.qfrozen_cnt
4594 	    = 0;
4595 	xpt_action((union ccb *)&crs);
4596 }
4597 
4598 void
4599 xpt_stop_tags(struct cam_path *path)
4600 {
4601 	struct ccb_relsim crs;
4602 	struct cam_ed *device;
4603 	struct cam_sim *sim;
4604 
4605 	device = path->device;
4606 	sim = path->bus->sim;
4607 	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4608 	device->tag_delay_count = 0;
4609 	xpt_freeze_devq(path, /*count*/1);
4610 	device->inq_flags &= ~SID_CmdQue;
4611 	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4612 	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4613 	crs.ccb_h.func_code = XPT_REL_SIMQ;
4614 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4615 	crs.openings
4616 	    = crs.release_timeout
4617 	    = crs.qfrozen_cnt
4618 	    = 0;
4619 	xpt_action((union ccb *)&crs);
4620 }
4621 
4622 static void
4623 xpt_boot_delay(void *arg)
4624 {
4625 
4626 	xpt_release_boot();
4627 }
4628 
4629 static void
4630 xpt_config(void *arg)
4631 {
4632 	/*
4633 	 * Now that interrupts are enabled, go find our devices
4634 	 */
4635 
4636 #ifdef CAMDEBUG
4637 	/* Setup debugging flags and path */
4638 #ifdef CAM_DEBUG_BUS
4639 	if (cam_dflags != CAM_DEBUG_NONE) {
4640 		/*
4641 		 * Locking is specifically omitted here.  No SIMs have
4642 		 * registered yet, so xpt_create_path will only be searching
4643 		 * empty lists of targets and devices.
4644 		 */
4645 		if (xpt_create_path(&cam_dpath, xpt_periph,
4646 				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4647 				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4648 			printf("xpt_config: xpt_create_path() failed for debug"
4649 			       " target %d:%d:%d, debugging disabled\n",
4650 			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4651 			cam_dflags = CAM_DEBUG_NONE;
4652 		}
4653 	} else
4654 		cam_dpath = NULL;
4655 #else /* !CAM_DEBUG_BUS */
4656 	cam_dpath = NULL;
4657 #endif /* CAM_DEBUG_BUS */
4658 #endif /* CAMDEBUG */
4659 
4660 	periphdriver_init(1);
4661 	xpt_hold_boot();
4662 	callout_init(&xsoftc.boot_callout, 1);
4663 	callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000,
4664 	    xpt_boot_delay, NULL);
4665 	/* Fire up rescan thread. */
4666 	if (kproc_create(xpt_scanner_thread, NULL, NULL, 0, 0, "xpt_thrd")) {
4667 		printf("xpt_config: failed to create rescan thread.\n");
4668 	}
4669 }
4670 
4671 void
4672 xpt_hold_boot(void)
4673 {
4674 	xpt_lock_buses();
4675 	xsoftc.buses_to_config++;
4676 	xpt_unlock_buses();
4677 }
4678 
4679 void
4680 xpt_release_boot(void)
4681 {
4682 	xpt_lock_buses();
4683 	xsoftc.buses_to_config--;
4684 	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4685 		struct	xpt_task *task;
4686 
4687 		xsoftc.buses_config_done = 1;
4688 		xpt_unlock_buses();
4689 		/* Call manually because we don't have any busses */
4690 		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
4691 		if (task != NULL) {
4692 			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
4693 			taskqueue_enqueue(taskqueue_thread, &task->task);
4694 		}
4695 	} else
4696 		xpt_unlock_buses();
4697 }
4698 
4699 /*
4700  * If the given device only has one peripheral attached to it, and if that
4701  * peripheral is the passthrough driver, announce it.  This insures that the
4702  * user sees some sort of announcement for every peripheral in their system.
4703  */
4704 static int
4705 xptpassannouncefunc(struct cam_ed *device, void *arg)
4706 {
4707 	struct cam_periph *periph;
4708 	int i;
4709 
4710 	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
4711 	     periph = SLIST_NEXT(periph, periph_links), i++);
4712 
4713 	periph = SLIST_FIRST(&device->periphs);
4714 	if ((i == 1)
4715 	 && (strncmp(periph->periph_name, "pass", 4) == 0))
4716 		xpt_announce_periph(periph, NULL);
4717 
4718 	return(1);
4719 }
4720 
4721 static void
4722 xpt_finishconfig_task(void *context, int pending)
4723 {
4724 
4725 	periphdriver_init(2);
4726 	/*
4727 	 * Check for devices with no "standard" peripheral driver
4728 	 * attached.  For any devices like that, announce the
4729 	 * passthrough driver so the user will see something.
4730 	 */
4731 	xpt_for_all_devices(xptpassannouncefunc, NULL);
4732 
4733 	/* Release our hook so that the boot can continue. */
4734 	config_intrhook_disestablish(xsoftc.xpt_config_hook);
4735 	free(xsoftc.xpt_config_hook, M_CAMXPT);
4736 	xsoftc.xpt_config_hook = NULL;
4737 
4738 	free(context, M_CAMXPT);
4739 }
4740 
4741 cam_status
4742 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
4743 		   struct cam_path *path)
4744 {
4745 	struct ccb_setasync csa;
4746 	cam_status status;
4747 	int xptpath = 0;
4748 
4749 	if (path == NULL) {
4750 		mtx_lock(&xsoftc.xpt_lock);
4751 		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
4752 					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4753 		if (status != CAM_REQ_CMP) {
4754 			mtx_unlock(&xsoftc.xpt_lock);
4755 			return (status);
4756 		}
4757 		xptpath = 1;
4758 	}
4759 
4760 	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
4761 	csa.ccb_h.func_code = XPT_SASYNC_CB;
4762 	csa.event_enable = event;
4763 	csa.callback = cbfunc;
4764 	csa.callback_arg = cbarg;
4765 	xpt_action((union ccb *)&csa);
4766 	status = csa.ccb_h.status;
4767 
4768 	if (xptpath) {
4769 		xpt_free_path(path);
4770 		mtx_unlock(&xsoftc.xpt_lock);
4771 	}
4772 
4773 	if ((status == CAM_REQ_CMP) &&
4774 	    (csa.event_enable & AC_FOUND_DEVICE)) {
4775 		/*
4776 		 * Get this peripheral up to date with all
4777 		 * the currently existing devices.
4778 		 */
4779 		xpt_for_all_devices(xptsetasyncfunc, &csa);
4780 	}
4781 	if ((status == CAM_REQ_CMP) &&
4782 	    (csa.event_enable & AC_PATH_REGISTERED)) {
4783 		/*
4784 		 * Get this peripheral up to date with all
4785 		 * the currently existing busses.
4786 		 */
4787 		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
4788 	}
4789 
4790 	return (status);
4791 }
4792 
4793 static void
4794 xptaction(struct cam_sim *sim, union ccb *work_ccb)
4795 {
4796 	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
4797 
4798 	switch (work_ccb->ccb_h.func_code) {
4799 	/* Common cases first */
4800 	case XPT_PATH_INQ:		/* Path routing inquiry */
4801 	{
4802 		struct ccb_pathinq *cpi;
4803 
4804 		cpi = &work_ccb->cpi;
4805 		cpi->version_num = 1; /* XXX??? */
4806 		cpi->hba_inquiry = 0;
4807 		cpi->target_sprt = 0;
4808 		cpi->hba_misc = 0;
4809 		cpi->hba_eng_cnt = 0;
4810 		cpi->max_target = 0;
4811 		cpi->max_lun = 0;
4812 		cpi->initiator_id = 0;
4813 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
4814 		strncpy(cpi->hba_vid, "", HBA_IDLEN);
4815 		strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
4816 		cpi->unit_number = sim->unit_number;
4817 		cpi->bus_id = sim->bus_id;
4818 		cpi->base_transfer_speed = 0;
4819 		cpi->protocol = PROTO_UNSPECIFIED;
4820 		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
4821 		cpi->transport = XPORT_UNSPECIFIED;
4822 		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
4823 		cpi->ccb_h.status = CAM_REQ_CMP;
4824 		xpt_done(work_ccb);
4825 		break;
4826 	}
4827 	default:
4828 		work_ccb->ccb_h.status = CAM_REQ_INVALID;
4829 		xpt_done(work_ccb);
4830 		break;
4831 	}
4832 }
4833 
4834 /*
4835  * The xpt as a "controller" has no interrupt sources, so polling
4836  * is a no-op.
4837  */
4838 static void
4839 xptpoll(struct cam_sim *sim)
4840 {
4841 }
4842 
4843 void
4844 xpt_lock_buses(void)
4845 {
4846 	mtx_lock(&xsoftc.xpt_topo_lock);
4847 }
4848 
4849 void
4850 xpt_unlock_buses(void)
4851 {
4852 	mtx_unlock(&xsoftc.xpt_topo_lock);
4853 }
4854 
4855 static void
4856 camisr(void *dummy)
4857 {
4858 	cam_simq_t queue;
4859 	struct cam_sim *sim;
4860 
4861 	mtx_lock(&cam_simq_lock);
4862 	TAILQ_INIT(&queue);
4863 	while (!TAILQ_EMPTY(&cam_simq)) {
4864 		TAILQ_CONCAT(&queue, &cam_simq, links);
4865 		mtx_unlock(&cam_simq_lock);
4866 
4867 		while ((sim = TAILQ_FIRST(&queue)) != NULL) {
4868 			TAILQ_REMOVE(&queue, sim, links);
4869 			CAM_SIM_LOCK(sim);
4870 			sim->flags &= ~CAM_SIM_ON_DONEQ;
4871 			camisr_runqueue(&sim->sim_doneq);
4872 			CAM_SIM_UNLOCK(sim);
4873 		}
4874 		mtx_lock(&cam_simq_lock);
4875 	}
4876 	mtx_unlock(&cam_simq_lock);
4877 }
4878 
4879 static void
4880 camisr_runqueue(void *V_queue)
4881 {
4882 	cam_isrq_t *queue = V_queue;
4883 	struct	ccb_hdr *ccb_h;
4884 
4885 	while ((ccb_h = TAILQ_FIRST(queue)) != NULL) {
4886 		int	runq;
4887 
4888 		TAILQ_REMOVE(queue, ccb_h, sim_links.tqe);
4889 		ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
4890 
4891 		CAM_DEBUG(ccb_h->path, CAM_DEBUG_TRACE,
4892 			  ("camisr\n"));
4893 
4894 		runq = FALSE;
4895 
4896 		if (ccb_h->flags & CAM_HIGH_POWER) {
4897 			struct highpowerlist	*hphead;
4898 			union ccb		*send_ccb;
4899 
4900 			mtx_lock(&xsoftc.xpt_lock);
4901 			hphead = &xsoftc.highpowerq;
4902 
4903 			send_ccb = (union ccb *)STAILQ_FIRST(hphead);
4904 
4905 			/*
4906 			 * Increment the count since this command is done.
4907 			 */
4908 			xsoftc.num_highpower++;
4909 
4910 			/*
4911 			 * Any high powered commands queued up?
4912 			 */
4913 			if (send_ccb != NULL) {
4914 
4915 				STAILQ_REMOVE_HEAD(hphead, xpt_links.stqe);
4916 				mtx_unlock(&xsoftc.xpt_lock);
4917 
4918 				xpt_release_devq(send_ccb->ccb_h.path,
4919 						 /*count*/1, /*runqueue*/TRUE);
4920 			} else
4921 				mtx_unlock(&xsoftc.xpt_lock);
4922 		}
4923 
4924 		if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
4925 			struct cam_ed *dev;
4926 
4927 			dev = ccb_h->path->device;
4928 
4929 			cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
4930 			ccb_h->path->bus->sim->devq->send_active--;
4931 			ccb_h->path->bus->sim->devq->send_openings++;
4932 			runq = TRUE;
4933 
4934 			if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
4935 			  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)
4936 			 || ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
4937 			  && (dev->ccbq.dev_active == 0))) {
4938 				xpt_release_devq(ccb_h->path, /*count*/1,
4939 						 /*run_queue*/FALSE);
4940 			}
4941 
4942 			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4943 			 && (--dev->tag_delay_count == 0))
4944 				xpt_start_tags(ccb_h->path);
4945 			if (!device_is_send_queued(dev)) {
4946 				(void)xpt_schedule_dev_sendq(ccb_h->path->bus,
4947 							     dev);
4948 			}
4949 		}
4950 
4951 		if (ccb_h->status & CAM_RELEASE_SIMQ) {
4952 			xpt_release_simq(ccb_h->path->bus->sim,
4953 					 /*run_queue*/TRUE);
4954 			ccb_h->status &= ~CAM_RELEASE_SIMQ;
4955 			runq = FALSE;
4956 		}
4957 
4958 		if ((ccb_h->flags & CAM_DEV_QFRZDIS)
4959 		 && (ccb_h->status & CAM_DEV_QFRZN)) {
4960 			xpt_release_devq(ccb_h->path, /*count*/1,
4961 					 /*run_queue*/TRUE);
4962 			ccb_h->status &= ~CAM_DEV_QFRZN;
4963 		} else if (runq) {
4964 			xpt_run_dev_sendq(ccb_h->path->bus);
4965 		}
4966 
4967 		/* Call the peripheral driver's callback */
4968 		(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
4969 	}
4970 }
4971