xref: /freebsd/sys/cam/cam_xpt.c (revision 5608fd23c27fa1e8ee595d7b678cbfd35d657fbe)
1 /*-
2  * Implementation of the Common Access Method Transport (XPT) layer.
3  *
4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions, and the following disclaimer,
13  *    without modification, immediately at the beginning of the file.
14  * 2. The name of the author may not be used to endorse or promote products
15  *    derived from this software without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/systm.h>
36 #include <sys/types.h>
37 #include <sys/malloc.h>
38 #include <sys/kernel.h>
39 #include <sys/time.h>
40 #include <sys/conf.h>
41 #include <sys/fcntl.h>
42 #include <sys/interrupt.h>
43 #include <sys/proc.h>
44 #include <sys/sbuf.h>
45 #include <sys/smp.h>
46 #include <sys/taskqueue.h>
47 
48 #include <sys/lock.h>
49 #include <sys/mutex.h>
50 #include <sys/sysctl.h>
51 #include <sys/kthread.h>
52 
53 #include <cam/cam.h>
54 #include <cam/cam_ccb.h>
55 #include <cam/cam_periph.h>
56 #include <cam/cam_queue.h>
57 #include <cam/cam_sim.h>
58 #include <cam/cam_xpt.h>
59 #include <cam/cam_xpt_sim.h>
60 #include <cam/cam_xpt_periph.h>
61 #include <cam/cam_xpt_internal.h>
62 #include <cam/cam_debug.h>
63 #include <cam/cam_compat.h>
64 
65 #include <cam/scsi/scsi_all.h>
66 #include <cam/scsi/scsi_message.h>
67 #include <cam/scsi/scsi_pass.h>
68 
69 #include <machine/md_var.h>	/* geometry translation */
70 #include <machine/stdarg.h>	/* for xpt_print below */
71 
72 #include "opt_cam.h"
73 
74 /*
75  * This is the maximum number of high powered commands (e.g. start unit)
76  * that can be outstanding at a particular time.
77  */
78 #ifndef CAM_MAX_HIGHPOWER
79 #define CAM_MAX_HIGHPOWER  4
80 #endif
81 
82 /* Datastructures internal to the xpt layer */
83 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
84 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
85 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
86 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
87 
88 /* Object for defering XPT actions to a taskqueue */
89 struct xpt_task {
90 	struct task	task;
91 	void		*data1;
92 	uintptr_t	data2;
93 };
94 
95 struct xpt_softc {
96 	/* number of high powered commands that can go through right now */
97 	struct mtx		xpt_highpower_lock;
98 	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
99 	int			num_highpower;
100 
101 	/* queue for handling async rescan requests. */
102 	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
103 	int buses_to_config;
104 	int buses_config_done;
105 
106 	/* Registered busses */
107 	TAILQ_HEAD(,cam_eb)	xpt_busses;
108 	u_int			bus_generation;
109 
110 	struct intr_config_hook	*xpt_config_hook;
111 
112 	int			boot_delay;
113 	struct callout 		boot_callout;
114 
115 	struct mtx		xpt_topo_lock;
116 	struct mtx		xpt_lock;
117 	struct taskqueue	*xpt_taskq;
118 };
119 
120 typedef enum {
121 	DM_RET_COPY		= 0x01,
122 	DM_RET_FLAG_MASK	= 0x0f,
123 	DM_RET_NONE		= 0x00,
124 	DM_RET_STOP		= 0x10,
125 	DM_RET_DESCEND		= 0x20,
126 	DM_RET_ERROR		= 0x30,
127 	DM_RET_ACTION_MASK	= 0xf0
128 } dev_match_ret;
129 
130 typedef enum {
131 	XPT_DEPTH_BUS,
132 	XPT_DEPTH_TARGET,
133 	XPT_DEPTH_DEVICE,
134 	XPT_DEPTH_PERIPH
135 } xpt_traverse_depth;
136 
137 struct xpt_traverse_config {
138 	xpt_traverse_depth	depth;
139 	void			*tr_func;
140 	void			*tr_arg;
141 };
142 
143 typedef	int	xpt_busfunc_t (struct cam_eb *bus, void *arg);
144 typedef	int	xpt_targetfunc_t (struct cam_et *target, void *arg);
145 typedef	int	xpt_devicefunc_t (struct cam_ed *device, void *arg);
146 typedef	int	xpt_periphfunc_t (struct cam_periph *periph, void *arg);
147 typedef int	xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
148 
149 /* Transport layer configuration information */
150 static struct xpt_softc xsoftc;
151 
152 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
153            &xsoftc.boot_delay, 0, "Bus registration wait time");
154 
155 struct cam_doneq {
156 	struct mtx_padalign	cam_doneq_mtx;
157 	STAILQ_HEAD(, ccb_hdr)	cam_doneq;
158 	int			cam_doneq_sleep;
159 };
160 
161 static struct cam_doneq cam_doneqs[MAXCPU];
162 static int cam_num_doneqs;
163 static struct proc *cam_proc;
164 
165 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
166            &cam_num_doneqs, 0, "Number of completion queues/threads");
167 
168 struct cam_periph *xpt_periph;
169 
170 static periph_init_t xpt_periph_init;
171 
172 static struct periph_driver xpt_driver =
173 {
174 	xpt_periph_init, "xpt",
175 	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
176 	CAM_PERIPH_DRV_EARLY
177 };
178 
179 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
180 
181 static d_open_t xptopen;
182 static d_close_t xptclose;
183 static d_ioctl_t xptioctl;
184 static d_ioctl_t xptdoioctl;
185 
186 static struct cdevsw xpt_cdevsw = {
187 	.d_version =	D_VERSION,
188 	.d_flags =	0,
189 	.d_open =	xptopen,
190 	.d_close =	xptclose,
191 	.d_ioctl =	xptioctl,
192 	.d_name =	"xpt",
193 };
194 
195 /* Storage for debugging datastructures */
196 struct cam_path *cam_dpath;
197 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
198 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RWTUN,
199 	&cam_dflags, 0, "Enabled debug flags");
200 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
201 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RWTUN,
202 	&cam_debug_delay, 0, "Delay in us after each debug message");
203 
204 /* Our boot-time initialization hook */
205 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
206 
207 static moduledata_t cam_moduledata = {
208 	"cam",
209 	cam_module_event_handler,
210 	NULL
211 };
212 
213 static int	xpt_init(void *);
214 
215 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
216 MODULE_VERSION(cam, 1);
217 
218 
219 static void		xpt_async_bcast(struct async_list *async_head,
220 					u_int32_t async_code,
221 					struct cam_path *path,
222 					void *async_arg);
223 static path_id_t xptnextfreepathid(void);
224 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
225 static union ccb *xpt_get_ccb(struct cam_periph *periph);
226 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
227 static void	 xpt_run_allocq(struct cam_periph *periph, int sleep);
228 static void	 xpt_run_allocq_task(void *context, int pending);
229 static void	 xpt_run_devq(struct cam_devq *devq);
230 static timeout_t xpt_release_devq_timeout;
231 static void	 xpt_release_simq_timeout(void *arg) __unused;
232 static void	 xpt_acquire_bus(struct cam_eb *bus);
233 static void	 xpt_release_bus(struct cam_eb *bus);
234 static uint32_t	 xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
235 static int	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
236 		    int run_queue);
237 static struct cam_et*
238 		 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
239 static void	 xpt_acquire_target(struct cam_et *target);
240 static void	 xpt_release_target(struct cam_et *target);
241 static struct cam_eb*
242 		 xpt_find_bus(path_id_t path_id);
243 static struct cam_et*
244 		 xpt_find_target(struct cam_eb *bus, target_id_t target_id);
245 static struct cam_ed*
246 		 xpt_find_device(struct cam_et *target, lun_id_t lun_id);
247 static void	 xpt_config(void *arg);
248 static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
249 				 u_int32_t new_priority);
250 static xpt_devicefunc_t xptpassannouncefunc;
251 static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
252 static void	 xptpoll(struct cam_sim *sim);
253 static void	 camisr_runqueue(void);
254 static void	 xpt_done_process(struct ccb_hdr *ccb_h);
255 static void	 xpt_done_td(void *);
256 static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
257 				    u_int num_patterns, struct cam_eb *bus);
258 static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
259 				       u_int num_patterns,
260 				       struct cam_ed *device);
261 static dev_match_ret	xptperiphmatch(struct dev_match_pattern *patterns,
262 				       u_int num_patterns,
263 				       struct cam_periph *periph);
264 static xpt_busfunc_t	xptedtbusfunc;
265 static xpt_targetfunc_t	xptedttargetfunc;
266 static xpt_devicefunc_t	xptedtdevicefunc;
267 static xpt_periphfunc_t	xptedtperiphfunc;
268 static xpt_pdrvfunc_t	xptplistpdrvfunc;
269 static xpt_periphfunc_t	xptplistperiphfunc;
270 static int		xptedtmatch(struct ccb_dev_match *cdm);
271 static int		xptperiphlistmatch(struct ccb_dev_match *cdm);
272 static int		xptbustraverse(struct cam_eb *start_bus,
273 				       xpt_busfunc_t *tr_func, void *arg);
274 static int		xpttargettraverse(struct cam_eb *bus,
275 					  struct cam_et *start_target,
276 					  xpt_targetfunc_t *tr_func, void *arg);
277 static int		xptdevicetraverse(struct cam_et *target,
278 					  struct cam_ed *start_device,
279 					  xpt_devicefunc_t *tr_func, void *arg);
280 static int		xptperiphtraverse(struct cam_ed *device,
281 					  struct cam_periph *start_periph,
282 					  xpt_periphfunc_t *tr_func, void *arg);
283 static int		xptpdrvtraverse(struct periph_driver **start_pdrv,
284 					xpt_pdrvfunc_t *tr_func, void *arg);
285 static int		xptpdperiphtraverse(struct periph_driver **pdrv,
286 					    struct cam_periph *start_periph,
287 					    xpt_periphfunc_t *tr_func,
288 					    void *arg);
289 static xpt_busfunc_t	xptdefbusfunc;
290 static xpt_targetfunc_t	xptdeftargetfunc;
291 static xpt_devicefunc_t	xptdefdevicefunc;
292 static xpt_periphfunc_t	xptdefperiphfunc;
293 static void		xpt_finishconfig_task(void *context, int pending);
294 static void		xpt_dev_async_default(u_int32_t async_code,
295 					      struct cam_eb *bus,
296 					      struct cam_et *target,
297 					      struct cam_ed *device,
298 					      void *async_arg);
299 static struct cam_ed *	xpt_alloc_device_default(struct cam_eb *bus,
300 						 struct cam_et *target,
301 						 lun_id_t lun_id);
302 static xpt_devicefunc_t	xptsetasyncfunc;
303 static xpt_busfunc_t	xptsetasyncbusfunc;
304 static cam_status	xptregister(struct cam_periph *periph,
305 				    void *arg);
306 static __inline int device_is_queued(struct cam_ed *device);
307 
308 static __inline int
309 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
310 {
311 	int	retval;
312 
313 	mtx_assert(&devq->send_mtx, MA_OWNED);
314 	if ((dev->ccbq.queue.entries > 0) &&
315 	    (dev->ccbq.dev_openings > 0) &&
316 	    (dev->ccbq.queue.qfrozen_cnt == 0)) {
317 		/*
318 		 * The priority of a device waiting for controller
319 		 * resources is that of the highest priority CCB
320 		 * enqueued.
321 		 */
322 		retval =
323 		    xpt_schedule_dev(&devq->send_queue,
324 				     &dev->devq_entry,
325 				     CAMQ_GET_PRIO(&dev->ccbq.queue));
326 	} else {
327 		retval = 0;
328 	}
329 	return (retval);
330 }
331 
332 static __inline int
333 device_is_queued(struct cam_ed *device)
334 {
335 	return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
336 }
337 
338 static void
339 xpt_periph_init()
340 {
341 	make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
342 }
343 
344 static int
345 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
346 {
347 
348 	/*
349 	 * Only allow read-write access.
350 	 */
351 	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
352 		return(EPERM);
353 
354 	/*
355 	 * We don't allow nonblocking access.
356 	 */
357 	if ((flags & O_NONBLOCK) != 0) {
358 		printf("%s: can't do nonblocking access\n", devtoname(dev));
359 		return(ENODEV);
360 	}
361 
362 	return(0);
363 }
364 
365 static int
366 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
367 {
368 
369 	return(0);
370 }
371 
372 /*
373  * Don't automatically grab the xpt softc lock here even though this is going
374  * through the xpt device.  The xpt device is really just a back door for
375  * accessing other devices and SIMs, so the right thing to do is to grab
376  * the appropriate SIM lock once the bus/SIM is located.
377  */
378 static int
379 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
380 {
381 	int error;
382 
383 	if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
384 		error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
385 	}
386 	return (error);
387 }
388 
389 static int
390 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
391 {
392 	int error;
393 
394 	error = 0;
395 
396 	switch(cmd) {
397 	/*
398 	 * For the transport layer CAMIOCOMMAND ioctl, we really only want
399 	 * to accept CCB types that don't quite make sense to send through a
400 	 * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
401 	 * in the CAM spec.
402 	 */
403 	case CAMIOCOMMAND: {
404 		union ccb *ccb;
405 		union ccb *inccb;
406 		struct cam_eb *bus;
407 
408 		inccb = (union ccb *)addr;
409 
410 		bus = xpt_find_bus(inccb->ccb_h.path_id);
411 		if (bus == NULL)
412 			return (EINVAL);
413 
414 		switch (inccb->ccb_h.func_code) {
415 		case XPT_SCAN_BUS:
416 		case XPT_RESET_BUS:
417 			if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
418 			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
419 				xpt_release_bus(bus);
420 				return (EINVAL);
421 			}
422 			break;
423 		case XPT_SCAN_TGT:
424 			if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
425 			    inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
426 				xpt_release_bus(bus);
427 				return (EINVAL);
428 			}
429 			break;
430 		default:
431 			break;
432 		}
433 
434 		switch(inccb->ccb_h.func_code) {
435 		case XPT_SCAN_BUS:
436 		case XPT_RESET_BUS:
437 		case XPT_PATH_INQ:
438 		case XPT_ENG_INQ:
439 		case XPT_SCAN_LUN:
440 		case XPT_SCAN_TGT:
441 
442 			ccb = xpt_alloc_ccb();
443 
444 			/*
445 			 * Create a path using the bus, target, and lun the
446 			 * user passed in.
447 			 */
448 			if (xpt_create_path(&ccb->ccb_h.path, NULL,
449 					    inccb->ccb_h.path_id,
450 					    inccb->ccb_h.target_id,
451 					    inccb->ccb_h.target_lun) !=
452 					    CAM_REQ_CMP){
453 				error = EINVAL;
454 				xpt_free_ccb(ccb);
455 				break;
456 			}
457 			/* Ensure all of our fields are correct */
458 			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
459 				      inccb->ccb_h.pinfo.priority);
460 			xpt_merge_ccb(ccb, inccb);
461 			xpt_path_lock(ccb->ccb_h.path);
462 			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
463 			xpt_path_unlock(ccb->ccb_h.path);
464 			bcopy(ccb, inccb, sizeof(union ccb));
465 			xpt_free_path(ccb->ccb_h.path);
466 			xpt_free_ccb(ccb);
467 			break;
468 
469 		case XPT_DEBUG: {
470 			union ccb ccb;
471 
472 			/*
473 			 * This is an immediate CCB, so it's okay to
474 			 * allocate it on the stack.
475 			 */
476 
477 			/*
478 			 * Create a path using the bus, target, and lun the
479 			 * user passed in.
480 			 */
481 			if (xpt_create_path(&ccb.ccb_h.path, NULL,
482 					    inccb->ccb_h.path_id,
483 					    inccb->ccb_h.target_id,
484 					    inccb->ccb_h.target_lun) !=
485 					    CAM_REQ_CMP){
486 				error = EINVAL;
487 				break;
488 			}
489 			/* Ensure all of our fields are correct */
490 			xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
491 				      inccb->ccb_h.pinfo.priority);
492 			xpt_merge_ccb(&ccb, inccb);
493 			xpt_action(&ccb);
494 			bcopy(&ccb, inccb, sizeof(union ccb));
495 			xpt_free_path(ccb.ccb_h.path);
496 			break;
497 
498 		}
499 		case XPT_DEV_MATCH: {
500 			struct cam_periph_map_info mapinfo;
501 			struct cam_path *old_path;
502 
503 			/*
504 			 * We can't deal with physical addresses for this
505 			 * type of transaction.
506 			 */
507 			if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
508 			    CAM_DATA_VADDR) {
509 				error = EINVAL;
510 				break;
511 			}
512 
513 			/*
514 			 * Save this in case the caller had it set to
515 			 * something in particular.
516 			 */
517 			old_path = inccb->ccb_h.path;
518 
519 			/*
520 			 * We really don't need a path for the matching
521 			 * code.  The path is needed because of the
522 			 * debugging statements in xpt_action().  They
523 			 * assume that the CCB has a valid path.
524 			 */
525 			inccb->ccb_h.path = xpt_periph->path;
526 
527 			bzero(&mapinfo, sizeof(mapinfo));
528 
529 			/*
530 			 * Map the pattern and match buffers into kernel
531 			 * virtual address space.
532 			 */
533 			error = cam_periph_mapmem(inccb, &mapinfo);
534 
535 			if (error) {
536 				inccb->ccb_h.path = old_path;
537 				break;
538 			}
539 
540 			/*
541 			 * This is an immediate CCB, we can send it on directly.
542 			 */
543 			xpt_action(inccb);
544 
545 			/*
546 			 * Map the buffers back into user space.
547 			 */
548 			cam_periph_unmapmem(inccb, &mapinfo);
549 
550 			inccb->ccb_h.path = old_path;
551 
552 			error = 0;
553 			break;
554 		}
555 		default:
556 			error = ENOTSUP;
557 			break;
558 		}
559 		xpt_release_bus(bus);
560 		break;
561 	}
562 	/*
563 	 * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
564 	 * with the periphal driver name and unit name filled in.  The other
565 	 * fields don't really matter as input.  The passthrough driver name
566 	 * ("pass"), and unit number are passed back in the ccb.  The current
567 	 * device generation number, and the index into the device peripheral
568 	 * driver list, and the status are also passed back.  Note that
569 	 * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
570 	 * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
571 	 * (or rather should be) impossible for the device peripheral driver
572 	 * list to change since we look at the whole thing in one pass, and
573 	 * we do it with lock protection.
574 	 *
575 	 */
576 	case CAMGETPASSTHRU: {
577 		union ccb *ccb;
578 		struct cam_periph *periph;
579 		struct periph_driver **p_drv;
580 		char   *name;
581 		u_int unit;
582 		int base_periph_found;
583 
584 		ccb = (union ccb *)addr;
585 		unit = ccb->cgdl.unit_number;
586 		name = ccb->cgdl.periph_name;
587 		base_periph_found = 0;
588 
589 		/*
590 		 * Sanity check -- make sure we don't get a null peripheral
591 		 * driver name.
592 		 */
593 		if (*ccb->cgdl.periph_name == '\0') {
594 			error = EINVAL;
595 			break;
596 		}
597 
598 		/* Keep the list from changing while we traverse it */
599 		xpt_lock_buses();
600 
601 		/* first find our driver in the list of drivers */
602 		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
603 			if (strcmp((*p_drv)->driver_name, name) == 0)
604 				break;
605 
606 		if (*p_drv == NULL) {
607 			xpt_unlock_buses();
608 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
609 			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
610 			*ccb->cgdl.periph_name = '\0';
611 			ccb->cgdl.unit_number = 0;
612 			error = ENOENT;
613 			break;
614 		}
615 
616 		/*
617 		 * Run through every peripheral instance of this driver
618 		 * and check to see whether it matches the unit passed
619 		 * in by the user.  If it does, get out of the loops and
620 		 * find the passthrough driver associated with that
621 		 * peripheral driver.
622 		 */
623 		for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
624 		     periph = TAILQ_NEXT(periph, unit_links)) {
625 
626 			if (periph->unit_number == unit)
627 				break;
628 		}
629 		/*
630 		 * If we found the peripheral driver that the user passed
631 		 * in, go through all of the peripheral drivers for that
632 		 * particular device and look for a passthrough driver.
633 		 */
634 		if (periph != NULL) {
635 			struct cam_ed *device;
636 			int i;
637 
638 			base_periph_found = 1;
639 			device = periph->path->device;
640 			for (i = 0, periph = SLIST_FIRST(&device->periphs);
641 			     periph != NULL;
642 			     periph = SLIST_NEXT(periph, periph_links), i++) {
643 				/*
644 				 * Check to see whether we have a
645 				 * passthrough device or not.
646 				 */
647 				if (strcmp(periph->periph_name, "pass") == 0) {
648 					/*
649 					 * Fill in the getdevlist fields.
650 					 */
651 					strcpy(ccb->cgdl.periph_name,
652 					       periph->periph_name);
653 					ccb->cgdl.unit_number =
654 						periph->unit_number;
655 					if (SLIST_NEXT(periph, periph_links))
656 						ccb->cgdl.status =
657 							CAM_GDEVLIST_MORE_DEVS;
658 					else
659 						ccb->cgdl.status =
660 						       CAM_GDEVLIST_LAST_DEVICE;
661 					ccb->cgdl.generation =
662 						device->generation;
663 					ccb->cgdl.index = i;
664 					/*
665 					 * Fill in some CCB header fields
666 					 * that the user may want.
667 					 */
668 					ccb->ccb_h.path_id =
669 						periph->path->bus->path_id;
670 					ccb->ccb_h.target_id =
671 						periph->path->target->target_id;
672 					ccb->ccb_h.target_lun =
673 						periph->path->device->lun_id;
674 					ccb->ccb_h.status = CAM_REQ_CMP;
675 					break;
676 				}
677 			}
678 		}
679 
680 		/*
681 		 * If the periph is null here, one of two things has
682 		 * happened.  The first possibility is that we couldn't
683 		 * find the unit number of the particular peripheral driver
684 		 * that the user is asking about.  e.g. the user asks for
685 		 * the passthrough driver for "da11".  We find the list of
686 		 * "da" peripherals all right, but there is no unit 11.
687 		 * The other possibility is that we went through the list
688 		 * of peripheral drivers attached to the device structure,
689 		 * but didn't find one with the name "pass".  Either way,
690 		 * we return ENOENT, since we couldn't find something.
691 		 */
692 		if (periph == NULL) {
693 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
694 			ccb->cgdl.status = CAM_GDEVLIST_ERROR;
695 			*ccb->cgdl.periph_name = '\0';
696 			ccb->cgdl.unit_number = 0;
697 			error = ENOENT;
698 			/*
699 			 * It is unfortunate that this is even necessary,
700 			 * but there are many, many clueless users out there.
701 			 * If this is true, the user is looking for the
702 			 * passthrough driver, but doesn't have one in his
703 			 * kernel.
704 			 */
705 			if (base_periph_found == 1) {
706 				printf("xptioctl: pass driver is not in the "
707 				       "kernel\n");
708 				printf("xptioctl: put \"device pass\" in "
709 				       "your kernel config file\n");
710 			}
711 		}
712 		xpt_unlock_buses();
713 		break;
714 		}
715 	default:
716 		error = ENOTTY;
717 		break;
718 	}
719 
720 	return(error);
721 }
722 
723 static int
724 cam_module_event_handler(module_t mod, int what, void *arg)
725 {
726 	int error;
727 
728 	switch (what) {
729 	case MOD_LOAD:
730 		if ((error = xpt_init(NULL)) != 0)
731 			return (error);
732 		break;
733 	case MOD_UNLOAD:
734 		return EBUSY;
735 	default:
736 		return EOPNOTSUPP;
737 	}
738 
739 	return 0;
740 }
741 
742 static void
743 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
744 {
745 
746 	if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
747 		xpt_free_path(done_ccb->ccb_h.path);
748 		xpt_free_ccb(done_ccb);
749 	} else {
750 		done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
751 		(*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
752 	}
753 	xpt_release_boot();
754 }
755 
756 /* thread to handle bus rescans */
757 static void
758 xpt_scanner_thread(void *dummy)
759 {
760 	union ccb	*ccb;
761 	struct cam_path	 path;
762 
763 	xpt_lock_buses();
764 	for (;;) {
765 		if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
766 			msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
767 			       "-", 0);
768 		if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
769 			TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
770 			xpt_unlock_buses();
771 
772 			/*
773 			 * Since lock can be dropped inside and path freed
774 			 * by completion callback even before return here,
775 			 * take our own path copy for reference.
776 			 */
777 			xpt_copy_path(&path, ccb->ccb_h.path);
778 			xpt_path_lock(&path);
779 			xpt_action(ccb);
780 			xpt_path_unlock(&path);
781 			xpt_release_path(&path);
782 
783 			xpt_lock_buses();
784 		}
785 	}
786 }
787 
788 void
789 xpt_rescan(union ccb *ccb)
790 {
791 	struct ccb_hdr *hdr;
792 
793 	/* Prepare request */
794 	if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
795 	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
796 		ccb->ccb_h.func_code = XPT_SCAN_BUS;
797 	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
798 	    ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
799 		ccb->ccb_h.func_code = XPT_SCAN_TGT;
800 	else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
801 	    ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
802 		ccb->ccb_h.func_code = XPT_SCAN_LUN;
803 	else {
804 		xpt_print(ccb->ccb_h.path, "illegal scan path\n");
805 		xpt_free_path(ccb->ccb_h.path);
806 		xpt_free_ccb(ccb);
807 		return;
808 	}
809 	ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
810 	ccb->ccb_h.cbfcnp = xpt_rescan_done;
811 	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
812 	/* Don't make duplicate entries for the same paths. */
813 	xpt_lock_buses();
814 	if (ccb->ccb_h.ppriv_ptr1 == NULL) {
815 		TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
816 			if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
817 				wakeup(&xsoftc.ccb_scanq);
818 				xpt_unlock_buses();
819 				xpt_print(ccb->ccb_h.path, "rescan already queued\n");
820 				xpt_free_path(ccb->ccb_h.path);
821 				xpt_free_ccb(ccb);
822 				return;
823 			}
824 		}
825 	}
826 	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
827 	xsoftc.buses_to_config++;
828 	wakeup(&xsoftc.ccb_scanq);
829 	xpt_unlock_buses();
830 }
831 
832 /* Functions accessed by the peripheral drivers */
833 static int
834 xpt_init(void *dummy)
835 {
836 	struct cam_sim *xpt_sim;
837 	struct cam_path *path;
838 	struct cam_devq *devq;
839 	cam_status status;
840 	int error, i;
841 
842 	TAILQ_INIT(&xsoftc.xpt_busses);
843 	TAILQ_INIT(&xsoftc.ccb_scanq);
844 	STAILQ_INIT(&xsoftc.highpowerq);
845 	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
846 
847 	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
848 	mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
849 	mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF);
850 	xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
851 	    taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
852 
853 #ifdef CAM_BOOT_DELAY
854 	/*
855 	 * Override this value at compile time to assist our users
856 	 * who don't use loader to boot a kernel.
857 	 */
858 	xsoftc.boot_delay = CAM_BOOT_DELAY;
859 #endif
860 	/*
861 	 * The xpt layer is, itself, the equivelent of a SIM.
862 	 * Allow 16 ccbs in the ccb pool for it.  This should
863 	 * give decent parallelism when we probe busses and
864 	 * perform other XPT functions.
865 	 */
866 	devq = cam_simq_alloc(16);
867 	xpt_sim = cam_sim_alloc(xptaction,
868 				xptpoll,
869 				"xpt",
870 				/*softc*/NULL,
871 				/*unit*/0,
872 				/*mtx*/&xsoftc.xpt_lock,
873 				/*max_dev_transactions*/0,
874 				/*max_tagged_dev_transactions*/0,
875 				devq);
876 	if (xpt_sim == NULL)
877 		return (ENOMEM);
878 
879 	mtx_lock(&xsoftc.xpt_lock);
880 	if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
881 		mtx_unlock(&xsoftc.xpt_lock);
882 		printf("xpt_init: xpt_bus_register failed with status %#x,"
883 		       " failing attach\n", status);
884 		return (EINVAL);
885 	}
886 	mtx_unlock(&xsoftc.xpt_lock);
887 
888 	/*
889 	 * Looking at the XPT from the SIM layer, the XPT is
890 	 * the equivelent of a peripheral driver.  Allocate
891 	 * a peripheral driver entry for us.
892 	 */
893 	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
894 				      CAM_TARGET_WILDCARD,
895 				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
896 		mtx_unlock(&xsoftc.xpt_lock);
897 		printf("xpt_init: xpt_create_path failed with status %#x,"
898 		       " failing attach\n", status);
899 		return (EINVAL);
900 	}
901 	xpt_path_lock(path);
902 	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
903 			 path, NULL, 0, xpt_sim);
904 	xpt_path_unlock(path);
905 	xpt_free_path(path);
906 
907 	if (cam_num_doneqs < 1)
908 		cam_num_doneqs = 1 + mp_ncpus / 6;
909 	else if (cam_num_doneqs > MAXCPU)
910 		cam_num_doneqs = MAXCPU;
911 	for (i = 0; i < cam_num_doneqs; i++) {
912 		mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
913 		    MTX_DEF);
914 		STAILQ_INIT(&cam_doneqs[i].cam_doneq);
915 		error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
916 		    &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
917 		if (error != 0) {
918 			cam_num_doneqs = i;
919 			break;
920 		}
921 	}
922 	if (cam_num_doneqs < 1) {
923 		printf("xpt_init: Cannot init completion queues "
924 		       "- failing attach\n");
925 		return (ENOMEM);
926 	}
927 	/*
928 	 * Register a callback for when interrupts are enabled.
929 	 */
930 	xsoftc.xpt_config_hook =
931 	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
932 					      M_CAMXPT, M_NOWAIT | M_ZERO);
933 	if (xsoftc.xpt_config_hook == NULL) {
934 		printf("xpt_init: Cannot malloc config hook "
935 		       "- failing attach\n");
936 		return (ENOMEM);
937 	}
938 	xsoftc.xpt_config_hook->ich_func = xpt_config;
939 	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
940 		free (xsoftc.xpt_config_hook, M_CAMXPT);
941 		printf("xpt_init: config_intrhook_establish failed "
942 		       "- failing attach\n");
943 	}
944 
945 	return (0);
946 }
947 
948 static cam_status
949 xptregister(struct cam_periph *periph, void *arg)
950 {
951 	struct cam_sim *xpt_sim;
952 
953 	if (periph == NULL) {
954 		printf("xptregister: periph was NULL!!\n");
955 		return(CAM_REQ_CMP_ERR);
956 	}
957 
958 	xpt_sim = (struct cam_sim *)arg;
959 	xpt_sim->softc = periph;
960 	xpt_periph = periph;
961 	periph->softc = NULL;
962 
963 	return(CAM_REQ_CMP);
964 }
965 
966 int32_t
967 xpt_add_periph(struct cam_periph *periph)
968 {
969 	struct cam_ed *device;
970 	int32_t	 status;
971 
972 	TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
973 	device = periph->path->device;
974 	status = CAM_REQ_CMP;
975 	if (device != NULL) {
976 		mtx_lock(&device->target->bus->eb_mtx);
977 		device->generation++;
978 		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
979 		mtx_unlock(&device->target->bus->eb_mtx);
980 	}
981 
982 	return (status);
983 }
984 
985 void
986 xpt_remove_periph(struct cam_periph *periph)
987 {
988 	struct cam_ed *device;
989 
990 	device = periph->path->device;
991 	if (device != NULL) {
992 		mtx_lock(&device->target->bus->eb_mtx);
993 		device->generation++;
994 		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
995 		mtx_unlock(&device->target->bus->eb_mtx);
996 	}
997 }
998 
999 
1000 void
1001 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1002 {
1003 	struct	cam_path *path = periph->path;
1004 
1005 	cam_periph_assert(periph, MA_OWNED);
1006 	periph->flags |= CAM_PERIPH_ANNOUNCED;
1007 
1008 	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1009 	       periph->periph_name, periph->unit_number,
1010 	       path->bus->sim->sim_name,
1011 	       path->bus->sim->unit_number,
1012 	       path->bus->sim->bus_id,
1013 	       path->bus->path_id,
1014 	       path->target->target_id,
1015 	       (uintmax_t)path->device->lun_id);
1016 	printf("%s%d: ", periph->periph_name, periph->unit_number);
1017 	if (path->device->protocol == PROTO_SCSI)
1018 		scsi_print_inquiry(&path->device->inq_data);
1019 	else if (path->device->protocol == PROTO_ATA ||
1020 	    path->device->protocol == PROTO_SATAPM)
1021 		ata_print_ident(&path->device->ident_data);
1022 	else if (path->device->protocol == PROTO_SEMB)
1023 		semb_print_ident(
1024 		    (struct sep_identify_data *)&path->device->ident_data);
1025 	else
1026 		printf("Unknown protocol device\n");
1027 	if (path->device->serial_num_len > 0) {
1028 		/* Don't wrap the screen  - print only the first 60 chars */
1029 		printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1030 		       periph->unit_number, path->device->serial_num);
1031 	}
1032 	/* Announce transport details. */
1033 	(*(path->bus->xport->announce))(periph);
1034 	/* Announce command queueing. */
1035 	if (path->device->inq_flags & SID_CmdQue
1036 	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1037 		printf("%s%d: Command Queueing enabled\n",
1038 		       periph->periph_name, periph->unit_number);
1039 	}
1040 	/* Announce caller's details if they've passed in. */
1041 	if (announce_string != NULL)
1042 		printf("%s%d: %s\n", periph->periph_name,
1043 		       periph->unit_number, announce_string);
1044 }
1045 
1046 void
1047 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1048 {
1049 	if (quirks != 0) {
1050 		printf("%s%d: quirks=0x%b\n", periph->periph_name,
1051 		    periph->unit_number, quirks, bit_string);
1052 	}
1053 }
1054 
1055 void
1056 xpt_denounce_periph(struct cam_periph *periph)
1057 {
1058 	struct	cam_path *path = periph->path;
1059 
1060 	cam_periph_assert(periph, MA_OWNED);
1061 	printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1062 	       periph->periph_name, periph->unit_number,
1063 	       path->bus->sim->sim_name,
1064 	       path->bus->sim->unit_number,
1065 	       path->bus->sim->bus_id,
1066 	       path->bus->path_id,
1067 	       path->target->target_id,
1068 	       (uintmax_t)path->device->lun_id);
1069 	printf("%s%d: ", periph->periph_name, periph->unit_number);
1070 	if (path->device->protocol == PROTO_SCSI)
1071 		scsi_print_inquiry_short(&path->device->inq_data);
1072 	else if (path->device->protocol == PROTO_ATA ||
1073 	    path->device->protocol == PROTO_SATAPM)
1074 		ata_print_ident_short(&path->device->ident_data);
1075 	else if (path->device->protocol == PROTO_SEMB)
1076 		semb_print_ident_short(
1077 		    (struct sep_identify_data *)&path->device->ident_data);
1078 	else
1079 		printf("Unknown protocol device");
1080 	if (path->device->serial_num_len > 0)
1081 		printf(" s/n %.60s", path->device->serial_num);
1082 	printf(" detached\n");
1083 }
1084 
1085 
1086 int
1087 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1088 {
1089 	int ret = -1, l;
1090 	struct ccb_dev_advinfo cdai;
1091 	struct scsi_vpd_id_descriptor *idd;
1092 
1093 	xpt_path_assert(path, MA_OWNED);
1094 
1095 	memset(&cdai, 0, sizeof(cdai));
1096 	xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1097 	cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1098 	cdai.bufsiz = len;
1099 
1100 	if (!strcmp(attr, "GEOM::ident"))
1101 		cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1102 	else if (!strcmp(attr, "GEOM::physpath"))
1103 		cdai.buftype = CDAI_TYPE_PHYS_PATH;
1104 	else if (strcmp(attr, "GEOM::lunid") == 0 ||
1105 		 strcmp(attr, "GEOM::lunname") == 0) {
1106 		cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1107 		cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1108 	} else
1109 		goto out;
1110 
1111 	cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1112 	if (cdai.buf == NULL) {
1113 		ret = ENOMEM;
1114 		goto out;
1115 	}
1116 	xpt_action((union ccb *)&cdai); /* can only be synchronous */
1117 	if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1118 		cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1119 	if (cdai.provsiz == 0)
1120 		goto out;
1121 	if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1122 		if (strcmp(attr, "GEOM::lunid") == 0) {
1123 			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1124 			    cdai.provsiz, scsi_devid_is_lun_naa);
1125 			if (idd == NULL)
1126 				idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1127 				    cdai.provsiz, scsi_devid_is_lun_eui64);
1128 		} else
1129 			idd = NULL;
1130 		if (idd == NULL)
1131 			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1132 			    cdai.provsiz, scsi_devid_is_lun_t10);
1133 		if (idd == NULL)
1134 			idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1135 			    cdai.provsiz, scsi_devid_is_lun_name);
1136 		if (idd == NULL)
1137 			goto out;
1138 		ret = 0;
1139 		if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII ||
1140 		    (idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1141 			l = strnlen(idd->identifier, idd->length);
1142 			if (l < len) {
1143 				bcopy(idd->identifier, buf, l);
1144 				buf[l] = 0;
1145 			} else
1146 				ret = EFAULT;
1147 		} else {
1148 			if (idd->length * 2 < len) {
1149 				for (l = 0; l < idd->length; l++)
1150 					sprintf(buf + l * 2, "%02x",
1151 					    idd->identifier[l]);
1152 			} else
1153 				ret = EFAULT;
1154 		}
1155 	} else {
1156 		ret = 0;
1157 		if (strlcpy(buf, cdai.buf, len) >= len)
1158 			ret = EFAULT;
1159 	}
1160 
1161 out:
1162 	if (cdai.buf != NULL)
1163 		free(cdai.buf, M_CAMXPT);
1164 	return ret;
1165 }
1166 
1167 static dev_match_ret
1168 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1169 	    struct cam_eb *bus)
1170 {
1171 	dev_match_ret retval;
1172 	int i;
1173 
1174 	retval = DM_RET_NONE;
1175 
1176 	/*
1177 	 * If we aren't given something to match against, that's an error.
1178 	 */
1179 	if (bus == NULL)
1180 		return(DM_RET_ERROR);
1181 
1182 	/*
1183 	 * If there are no match entries, then this bus matches no
1184 	 * matter what.
1185 	 */
1186 	if ((patterns == NULL) || (num_patterns == 0))
1187 		return(DM_RET_DESCEND | DM_RET_COPY);
1188 
1189 	for (i = 0; i < num_patterns; i++) {
1190 		struct bus_match_pattern *cur_pattern;
1191 
1192 		/*
1193 		 * If the pattern in question isn't for a bus node, we
1194 		 * aren't interested.  However, we do indicate to the
1195 		 * calling routine that we should continue descending the
1196 		 * tree, since the user wants to match against lower-level
1197 		 * EDT elements.
1198 		 */
1199 		if (patterns[i].type != DEV_MATCH_BUS) {
1200 			if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1201 				retval |= DM_RET_DESCEND;
1202 			continue;
1203 		}
1204 
1205 		cur_pattern = &patterns[i].pattern.bus_pattern;
1206 
1207 		/*
1208 		 * If they want to match any bus node, we give them any
1209 		 * device node.
1210 		 */
1211 		if (cur_pattern->flags == BUS_MATCH_ANY) {
1212 			/* set the copy flag */
1213 			retval |= DM_RET_COPY;
1214 
1215 			/*
1216 			 * If we've already decided on an action, go ahead
1217 			 * and return.
1218 			 */
1219 			if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1220 				return(retval);
1221 		}
1222 
1223 		/*
1224 		 * Not sure why someone would do this...
1225 		 */
1226 		if (cur_pattern->flags == BUS_MATCH_NONE)
1227 			continue;
1228 
1229 		if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1230 		 && (cur_pattern->path_id != bus->path_id))
1231 			continue;
1232 
1233 		if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1234 		 && (cur_pattern->bus_id != bus->sim->bus_id))
1235 			continue;
1236 
1237 		if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1238 		 && (cur_pattern->unit_number != bus->sim->unit_number))
1239 			continue;
1240 
1241 		if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1242 		 && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1243 			     DEV_IDLEN) != 0))
1244 			continue;
1245 
1246 		/*
1247 		 * If we get to this point, the user definitely wants
1248 		 * information on this bus.  So tell the caller to copy the
1249 		 * data out.
1250 		 */
1251 		retval |= DM_RET_COPY;
1252 
1253 		/*
1254 		 * If the return action has been set to descend, then we
1255 		 * know that we've already seen a non-bus matching
1256 		 * expression, therefore we need to further descend the tree.
1257 		 * This won't change by continuing around the loop, so we
1258 		 * go ahead and return.  If we haven't seen a non-bus
1259 		 * matching expression, we keep going around the loop until
1260 		 * we exhaust the matching expressions.  We'll set the stop
1261 		 * flag once we fall out of the loop.
1262 		 */
1263 		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1264 			return(retval);
1265 	}
1266 
1267 	/*
1268 	 * If the return action hasn't been set to descend yet, that means
1269 	 * we haven't seen anything other than bus matching patterns.  So
1270 	 * tell the caller to stop descending the tree -- the user doesn't
1271 	 * want to match against lower level tree elements.
1272 	 */
1273 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1274 		retval |= DM_RET_STOP;
1275 
1276 	return(retval);
1277 }
1278 
1279 static dev_match_ret
1280 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1281 	       struct cam_ed *device)
1282 {
1283 	dev_match_ret retval;
1284 	int i;
1285 
1286 	retval = DM_RET_NONE;
1287 
1288 	/*
1289 	 * If we aren't given something to match against, that's an error.
1290 	 */
1291 	if (device == NULL)
1292 		return(DM_RET_ERROR);
1293 
1294 	/*
1295 	 * If there are no match entries, then this device matches no
1296 	 * matter what.
1297 	 */
1298 	if ((patterns == NULL) || (num_patterns == 0))
1299 		return(DM_RET_DESCEND | DM_RET_COPY);
1300 
1301 	for (i = 0; i < num_patterns; i++) {
1302 		struct device_match_pattern *cur_pattern;
1303 		struct scsi_vpd_device_id *device_id_page;
1304 
1305 		/*
1306 		 * If the pattern in question isn't for a device node, we
1307 		 * aren't interested.
1308 		 */
1309 		if (patterns[i].type != DEV_MATCH_DEVICE) {
1310 			if ((patterns[i].type == DEV_MATCH_PERIPH)
1311 			 && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1312 				retval |= DM_RET_DESCEND;
1313 			continue;
1314 		}
1315 
1316 		cur_pattern = &patterns[i].pattern.device_pattern;
1317 
1318 		/* Error out if mutually exclusive options are specified. */
1319 		if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1320 		 == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1321 			return(DM_RET_ERROR);
1322 
1323 		/*
1324 		 * If they want to match any device node, we give them any
1325 		 * device node.
1326 		 */
1327 		if (cur_pattern->flags == DEV_MATCH_ANY)
1328 			goto copy_dev_node;
1329 
1330 		/*
1331 		 * Not sure why someone would do this...
1332 		 */
1333 		if (cur_pattern->flags == DEV_MATCH_NONE)
1334 			continue;
1335 
1336 		if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1337 		 && (cur_pattern->path_id != device->target->bus->path_id))
1338 			continue;
1339 
1340 		if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1341 		 && (cur_pattern->target_id != device->target->target_id))
1342 			continue;
1343 
1344 		if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1345 		 && (cur_pattern->target_lun != device->lun_id))
1346 			continue;
1347 
1348 		if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1349 		 && (cam_quirkmatch((caddr_t)&device->inq_data,
1350 				    (caddr_t)&cur_pattern->data.inq_pat,
1351 				    1, sizeof(cur_pattern->data.inq_pat),
1352 				    scsi_static_inquiry_match) == NULL))
1353 			continue;
1354 
1355 		device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1356 		if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1357 		 && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1358 		  || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1359 				      device->device_id_len
1360 				    - SVPD_DEVICE_ID_HDR_LEN,
1361 				      cur_pattern->data.devid_pat.id,
1362 				      cur_pattern->data.devid_pat.id_len) != 0))
1363 			continue;
1364 
1365 copy_dev_node:
1366 		/*
1367 		 * If we get to this point, the user definitely wants
1368 		 * information on this device.  So tell the caller to copy
1369 		 * the data out.
1370 		 */
1371 		retval |= DM_RET_COPY;
1372 
1373 		/*
1374 		 * If the return action has been set to descend, then we
1375 		 * know that we've already seen a peripheral matching
1376 		 * expression, therefore we need to further descend the tree.
1377 		 * This won't change by continuing around the loop, so we
1378 		 * go ahead and return.  If we haven't seen a peripheral
1379 		 * matching expression, we keep going around the loop until
1380 		 * we exhaust the matching expressions.  We'll set the stop
1381 		 * flag once we fall out of the loop.
1382 		 */
1383 		if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1384 			return(retval);
1385 	}
1386 
1387 	/*
1388 	 * If the return action hasn't been set to descend yet, that means
1389 	 * we haven't seen any peripheral matching patterns.  So tell the
1390 	 * caller to stop descending the tree -- the user doesn't want to
1391 	 * match against lower level tree elements.
1392 	 */
1393 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1394 		retval |= DM_RET_STOP;
1395 
1396 	return(retval);
1397 }
1398 
1399 /*
1400  * Match a single peripheral against any number of match patterns.
1401  */
1402 static dev_match_ret
1403 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1404 	       struct cam_periph *periph)
1405 {
1406 	dev_match_ret retval;
1407 	int i;
1408 
1409 	/*
1410 	 * If we aren't given something to match against, that's an error.
1411 	 */
1412 	if (periph == NULL)
1413 		return(DM_RET_ERROR);
1414 
1415 	/*
1416 	 * If there are no match entries, then this peripheral matches no
1417 	 * matter what.
1418 	 */
1419 	if ((patterns == NULL) || (num_patterns == 0))
1420 		return(DM_RET_STOP | DM_RET_COPY);
1421 
1422 	/*
1423 	 * There aren't any nodes below a peripheral node, so there's no
1424 	 * reason to descend the tree any further.
1425 	 */
1426 	retval = DM_RET_STOP;
1427 
1428 	for (i = 0; i < num_patterns; i++) {
1429 		struct periph_match_pattern *cur_pattern;
1430 
1431 		/*
1432 		 * If the pattern in question isn't for a peripheral, we
1433 		 * aren't interested.
1434 		 */
1435 		if (patterns[i].type != DEV_MATCH_PERIPH)
1436 			continue;
1437 
1438 		cur_pattern = &patterns[i].pattern.periph_pattern;
1439 
1440 		/*
1441 		 * If they want to match on anything, then we will do so.
1442 		 */
1443 		if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1444 			/* set the copy flag */
1445 			retval |= DM_RET_COPY;
1446 
1447 			/*
1448 			 * We've already set the return action to stop,
1449 			 * since there are no nodes below peripherals in
1450 			 * the tree.
1451 			 */
1452 			return(retval);
1453 		}
1454 
1455 		/*
1456 		 * Not sure why someone would do this...
1457 		 */
1458 		if (cur_pattern->flags == PERIPH_MATCH_NONE)
1459 			continue;
1460 
1461 		if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1462 		 && (cur_pattern->path_id != periph->path->bus->path_id))
1463 			continue;
1464 
1465 		/*
1466 		 * For the target and lun id's, we have to make sure the
1467 		 * target and lun pointers aren't NULL.  The xpt peripheral
1468 		 * has a wildcard target and device.
1469 		 */
1470 		if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1471 		 && ((periph->path->target == NULL)
1472 		 ||(cur_pattern->target_id != periph->path->target->target_id)))
1473 			continue;
1474 
1475 		if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1476 		 && ((periph->path->device == NULL)
1477 		 || (cur_pattern->target_lun != periph->path->device->lun_id)))
1478 			continue;
1479 
1480 		if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1481 		 && (cur_pattern->unit_number != periph->unit_number))
1482 			continue;
1483 
1484 		if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1485 		 && (strncmp(cur_pattern->periph_name, periph->periph_name,
1486 			     DEV_IDLEN) != 0))
1487 			continue;
1488 
1489 		/*
1490 		 * If we get to this point, the user definitely wants
1491 		 * information on this peripheral.  So tell the caller to
1492 		 * copy the data out.
1493 		 */
1494 		retval |= DM_RET_COPY;
1495 
1496 		/*
1497 		 * The return action has already been set to stop, since
1498 		 * peripherals don't have any nodes below them in the EDT.
1499 		 */
1500 		return(retval);
1501 	}
1502 
1503 	/*
1504 	 * If we get to this point, the peripheral that was passed in
1505 	 * doesn't match any of the patterns.
1506 	 */
1507 	return(retval);
1508 }
1509 
1510 static int
1511 xptedtbusfunc(struct cam_eb *bus, void *arg)
1512 {
1513 	struct ccb_dev_match *cdm;
1514 	struct cam_et *target;
1515 	dev_match_ret retval;
1516 
1517 	cdm = (struct ccb_dev_match *)arg;
1518 
1519 	/*
1520 	 * If our position is for something deeper in the tree, that means
1521 	 * that we've already seen this node.  So, we keep going down.
1522 	 */
1523 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1524 	 && (cdm->pos.cookie.bus == bus)
1525 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1526 	 && (cdm->pos.cookie.target != NULL))
1527 		retval = DM_RET_DESCEND;
1528 	else
1529 		retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1530 
1531 	/*
1532 	 * If we got an error, bail out of the search.
1533 	 */
1534 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1535 		cdm->status = CAM_DEV_MATCH_ERROR;
1536 		return(0);
1537 	}
1538 
1539 	/*
1540 	 * If the copy flag is set, copy this bus out.
1541 	 */
1542 	if (retval & DM_RET_COPY) {
1543 		int spaceleft, j;
1544 
1545 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1546 			sizeof(struct dev_match_result));
1547 
1548 		/*
1549 		 * If we don't have enough space to put in another
1550 		 * match result, save our position and tell the
1551 		 * user there are more devices to check.
1552 		 */
1553 		if (spaceleft < sizeof(struct dev_match_result)) {
1554 			bzero(&cdm->pos, sizeof(cdm->pos));
1555 			cdm->pos.position_type =
1556 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1557 
1558 			cdm->pos.cookie.bus = bus;
1559 			cdm->pos.generations[CAM_BUS_GENERATION]=
1560 				xsoftc.bus_generation;
1561 			cdm->status = CAM_DEV_MATCH_MORE;
1562 			return(0);
1563 		}
1564 		j = cdm->num_matches;
1565 		cdm->num_matches++;
1566 		cdm->matches[j].type = DEV_MATCH_BUS;
1567 		cdm->matches[j].result.bus_result.path_id = bus->path_id;
1568 		cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1569 		cdm->matches[j].result.bus_result.unit_number =
1570 			bus->sim->unit_number;
1571 		strncpy(cdm->matches[j].result.bus_result.dev_name,
1572 			bus->sim->sim_name, DEV_IDLEN);
1573 	}
1574 
1575 	/*
1576 	 * If the user is only interested in busses, there's no
1577 	 * reason to descend to the next level in the tree.
1578 	 */
1579 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1580 		return(1);
1581 
1582 	/*
1583 	 * If there is a target generation recorded, check it to
1584 	 * make sure the target list hasn't changed.
1585 	 */
1586 	mtx_lock(&bus->eb_mtx);
1587 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1588 	 && (cdm->pos.cookie.bus == bus)
1589 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1590 	 && (cdm->pos.cookie.target != NULL)) {
1591 		if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1592 		    bus->generation)) {
1593 			mtx_unlock(&bus->eb_mtx);
1594 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1595 			return (0);
1596 		}
1597 		target = (struct cam_et *)cdm->pos.cookie.target;
1598 		target->refcount++;
1599 	} else
1600 		target = NULL;
1601 	mtx_unlock(&bus->eb_mtx);
1602 
1603 	return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1604 }
1605 
1606 static int
1607 xptedttargetfunc(struct cam_et *target, void *arg)
1608 {
1609 	struct ccb_dev_match *cdm;
1610 	struct cam_eb *bus;
1611 	struct cam_ed *device;
1612 
1613 	cdm = (struct ccb_dev_match *)arg;
1614 	bus = target->bus;
1615 
1616 	/*
1617 	 * If there is a device list generation recorded, check it to
1618 	 * make sure the device list hasn't changed.
1619 	 */
1620 	mtx_lock(&bus->eb_mtx);
1621 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1622 	 && (cdm->pos.cookie.bus == bus)
1623 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1624 	 && (cdm->pos.cookie.target == target)
1625 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1626 	 && (cdm->pos.cookie.device != NULL)) {
1627 		if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1628 		    target->generation) {
1629 			mtx_unlock(&bus->eb_mtx);
1630 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1631 			return(0);
1632 		}
1633 		device = (struct cam_ed *)cdm->pos.cookie.device;
1634 		device->refcount++;
1635 	} else
1636 		device = NULL;
1637 	mtx_unlock(&bus->eb_mtx);
1638 
1639 	return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1640 }
1641 
1642 static int
1643 xptedtdevicefunc(struct cam_ed *device, void *arg)
1644 {
1645 	struct cam_eb *bus;
1646 	struct cam_periph *periph;
1647 	struct ccb_dev_match *cdm;
1648 	dev_match_ret retval;
1649 
1650 	cdm = (struct ccb_dev_match *)arg;
1651 	bus = device->target->bus;
1652 
1653 	/*
1654 	 * If our position is for something deeper in the tree, that means
1655 	 * that we've already seen this node.  So, we keep going down.
1656 	 */
1657 	if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1658 	 && (cdm->pos.cookie.device == device)
1659 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1660 	 && (cdm->pos.cookie.periph != NULL))
1661 		retval = DM_RET_DESCEND;
1662 	else
1663 		retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1664 					device);
1665 
1666 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1667 		cdm->status = CAM_DEV_MATCH_ERROR;
1668 		return(0);
1669 	}
1670 
1671 	/*
1672 	 * If the copy flag is set, copy this device out.
1673 	 */
1674 	if (retval & DM_RET_COPY) {
1675 		int spaceleft, j;
1676 
1677 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1678 			sizeof(struct dev_match_result));
1679 
1680 		/*
1681 		 * If we don't have enough space to put in another
1682 		 * match result, save our position and tell the
1683 		 * user there are more devices to check.
1684 		 */
1685 		if (spaceleft < sizeof(struct dev_match_result)) {
1686 			bzero(&cdm->pos, sizeof(cdm->pos));
1687 			cdm->pos.position_type =
1688 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1689 				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1690 
1691 			cdm->pos.cookie.bus = device->target->bus;
1692 			cdm->pos.generations[CAM_BUS_GENERATION]=
1693 				xsoftc.bus_generation;
1694 			cdm->pos.cookie.target = device->target;
1695 			cdm->pos.generations[CAM_TARGET_GENERATION] =
1696 				device->target->bus->generation;
1697 			cdm->pos.cookie.device = device;
1698 			cdm->pos.generations[CAM_DEV_GENERATION] =
1699 				device->target->generation;
1700 			cdm->status = CAM_DEV_MATCH_MORE;
1701 			return(0);
1702 		}
1703 		j = cdm->num_matches;
1704 		cdm->num_matches++;
1705 		cdm->matches[j].type = DEV_MATCH_DEVICE;
1706 		cdm->matches[j].result.device_result.path_id =
1707 			device->target->bus->path_id;
1708 		cdm->matches[j].result.device_result.target_id =
1709 			device->target->target_id;
1710 		cdm->matches[j].result.device_result.target_lun =
1711 			device->lun_id;
1712 		cdm->matches[j].result.device_result.protocol =
1713 			device->protocol;
1714 		bcopy(&device->inq_data,
1715 		      &cdm->matches[j].result.device_result.inq_data,
1716 		      sizeof(struct scsi_inquiry_data));
1717 		bcopy(&device->ident_data,
1718 		      &cdm->matches[j].result.device_result.ident_data,
1719 		      sizeof(struct ata_params));
1720 
1721 		/* Let the user know whether this device is unconfigured */
1722 		if (device->flags & CAM_DEV_UNCONFIGURED)
1723 			cdm->matches[j].result.device_result.flags =
1724 				DEV_RESULT_UNCONFIGURED;
1725 		else
1726 			cdm->matches[j].result.device_result.flags =
1727 				DEV_RESULT_NOFLAG;
1728 	}
1729 
1730 	/*
1731 	 * If the user isn't interested in peripherals, don't descend
1732 	 * the tree any further.
1733 	 */
1734 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1735 		return(1);
1736 
1737 	/*
1738 	 * If there is a peripheral list generation recorded, make sure
1739 	 * it hasn't changed.
1740 	 */
1741 	xpt_lock_buses();
1742 	mtx_lock(&bus->eb_mtx);
1743 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1744 	 && (cdm->pos.cookie.bus == bus)
1745 	 && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1746 	 && (cdm->pos.cookie.target == device->target)
1747 	 && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1748 	 && (cdm->pos.cookie.device == device)
1749 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1750 	 && (cdm->pos.cookie.periph != NULL)) {
1751 		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1752 		    device->generation) {
1753 			mtx_unlock(&bus->eb_mtx);
1754 			xpt_unlock_buses();
1755 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1756 			return(0);
1757 		}
1758 		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1759 		periph->refcount++;
1760 	} else
1761 		periph = NULL;
1762 	mtx_unlock(&bus->eb_mtx);
1763 	xpt_unlock_buses();
1764 
1765 	return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1766 }
1767 
1768 static int
1769 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1770 {
1771 	struct ccb_dev_match *cdm;
1772 	dev_match_ret retval;
1773 
1774 	cdm = (struct ccb_dev_match *)arg;
1775 
1776 	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1777 
1778 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1779 		cdm->status = CAM_DEV_MATCH_ERROR;
1780 		return(0);
1781 	}
1782 
1783 	/*
1784 	 * If the copy flag is set, copy this peripheral out.
1785 	 */
1786 	if (retval & DM_RET_COPY) {
1787 		int spaceleft, j;
1788 
1789 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1790 			sizeof(struct dev_match_result));
1791 
1792 		/*
1793 		 * If we don't have enough space to put in another
1794 		 * match result, save our position and tell the
1795 		 * user there are more devices to check.
1796 		 */
1797 		if (spaceleft < sizeof(struct dev_match_result)) {
1798 			bzero(&cdm->pos, sizeof(cdm->pos));
1799 			cdm->pos.position_type =
1800 				CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1801 				CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1802 				CAM_DEV_POS_PERIPH;
1803 
1804 			cdm->pos.cookie.bus = periph->path->bus;
1805 			cdm->pos.generations[CAM_BUS_GENERATION]=
1806 				xsoftc.bus_generation;
1807 			cdm->pos.cookie.target = periph->path->target;
1808 			cdm->pos.generations[CAM_TARGET_GENERATION] =
1809 				periph->path->bus->generation;
1810 			cdm->pos.cookie.device = periph->path->device;
1811 			cdm->pos.generations[CAM_DEV_GENERATION] =
1812 				periph->path->target->generation;
1813 			cdm->pos.cookie.periph = periph;
1814 			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1815 				periph->path->device->generation;
1816 			cdm->status = CAM_DEV_MATCH_MORE;
1817 			return(0);
1818 		}
1819 
1820 		j = cdm->num_matches;
1821 		cdm->num_matches++;
1822 		cdm->matches[j].type = DEV_MATCH_PERIPH;
1823 		cdm->matches[j].result.periph_result.path_id =
1824 			periph->path->bus->path_id;
1825 		cdm->matches[j].result.periph_result.target_id =
1826 			periph->path->target->target_id;
1827 		cdm->matches[j].result.periph_result.target_lun =
1828 			periph->path->device->lun_id;
1829 		cdm->matches[j].result.periph_result.unit_number =
1830 			periph->unit_number;
1831 		strncpy(cdm->matches[j].result.periph_result.periph_name,
1832 			periph->periph_name, DEV_IDLEN);
1833 	}
1834 
1835 	return(1);
1836 }
1837 
1838 static int
1839 xptedtmatch(struct ccb_dev_match *cdm)
1840 {
1841 	struct cam_eb *bus;
1842 	int ret;
1843 
1844 	cdm->num_matches = 0;
1845 
1846 	/*
1847 	 * Check the bus list generation.  If it has changed, the user
1848 	 * needs to reset everything and start over.
1849 	 */
1850 	xpt_lock_buses();
1851 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1852 	 && (cdm->pos.cookie.bus != NULL)) {
1853 		if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1854 		    xsoftc.bus_generation) {
1855 			xpt_unlock_buses();
1856 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1857 			return(0);
1858 		}
1859 		bus = (struct cam_eb *)cdm->pos.cookie.bus;
1860 		bus->refcount++;
1861 	} else
1862 		bus = NULL;
1863 	xpt_unlock_buses();
1864 
1865 	ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1866 
1867 	/*
1868 	 * If we get back 0, that means that we had to stop before fully
1869 	 * traversing the EDT.  It also means that one of the subroutines
1870 	 * has set the status field to the proper value.  If we get back 1,
1871 	 * we've fully traversed the EDT and copied out any matching entries.
1872 	 */
1873 	if (ret == 1)
1874 		cdm->status = CAM_DEV_MATCH_LAST;
1875 
1876 	return(ret);
1877 }
1878 
1879 static int
1880 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1881 {
1882 	struct cam_periph *periph;
1883 	struct ccb_dev_match *cdm;
1884 
1885 	cdm = (struct ccb_dev_match *)arg;
1886 
1887 	xpt_lock_buses();
1888 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1889 	 && (cdm->pos.cookie.pdrv == pdrv)
1890 	 && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1891 	 && (cdm->pos.cookie.periph != NULL)) {
1892 		if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1893 		    (*pdrv)->generation) {
1894 			xpt_unlock_buses();
1895 			cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1896 			return(0);
1897 		}
1898 		periph = (struct cam_periph *)cdm->pos.cookie.periph;
1899 		periph->refcount++;
1900 	} else
1901 		periph = NULL;
1902 	xpt_unlock_buses();
1903 
1904 	return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1905 }
1906 
1907 static int
1908 xptplistperiphfunc(struct cam_periph *periph, void *arg)
1909 {
1910 	struct ccb_dev_match *cdm;
1911 	dev_match_ret retval;
1912 
1913 	cdm = (struct ccb_dev_match *)arg;
1914 
1915 	retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1916 
1917 	if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1918 		cdm->status = CAM_DEV_MATCH_ERROR;
1919 		return(0);
1920 	}
1921 
1922 	/*
1923 	 * If the copy flag is set, copy this peripheral out.
1924 	 */
1925 	if (retval & DM_RET_COPY) {
1926 		int spaceleft, j;
1927 
1928 		spaceleft = cdm->match_buf_len - (cdm->num_matches *
1929 			sizeof(struct dev_match_result));
1930 
1931 		/*
1932 		 * If we don't have enough space to put in another
1933 		 * match result, save our position and tell the
1934 		 * user there are more devices to check.
1935 		 */
1936 		if (spaceleft < sizeof(struct dev_match_result)) {
1937 			struct periph_driver **pdrv;
1938 
1939 			pdrv = NULL;
1940 			bzero(&cdm->pos, sizeof(cdm->pos));
1941 			cdm->pos.position_type =
1942 				CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1943 				CAM_DEV_POS_PERIPH;
1944 
1945 			/*
1946 			 * This may look a bit non-sensical, but it is
1947 			 * actually quite logical.  There are very few
1948 			 * peripheral drivers, and bloating every peripheral
1949 			 * structure with a pointer back to its parent
1950 			 * peripheral driver linker set entry would cost
1951 			 * more in the long run than doing this quick lookup.
1952 			 */
1953 			for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1954 				if (strcmp((*pdrv)->driver_name,
1955 				    periph->periph_name) == 0)
1956 					break;
1957 			}
1958 
1959 			if (*pdrv == NULL) {
1960 				cdm->status = CAM_DEV_MATCH_ERROR;
1961 				return(0);
1962 			}
1963 
1964 			cdm->pos.cookie.pdrv = pdrv;
1965 			/*
1966 			 * The periph generation slot does double duty, as
1967 			 * does the periph pointer slot.  They are used for
1968 			 * both edt and pdrv lookups and positioning.
1969 			 */
1970 			cdm->pos.cookie.periph = periph;
1971 			cdm->pos.generations[CAM_PERIPH_GENERATION] =
1972 				(*pdrv)->generation;
1973 			cdm->status = CAM_DEV_MATCH_MORE;
1974 			return(0);
1975 		}
1976 
1977 		j = cdm->num_matches;
1978 		cdm->num_matches++;
1979 		cdm->matches[j].type = DEV_MATCH_PERIPH;
1980 		cdm->matches[j].result.periph_result.path_id =
1981 			periph->path->bus->path_id;
1982 
1983 		/*
1984 		 * The transport layer peripheral doesn't have a target or
1985 		 * lun.
1986 		 */
1987 		if (periph->path->target)
1988 			cdm->matches[j].result.periph_result.target_id =
1989 				periph->path->target->target_id;
1990 		else
1991 			cdm->matches[j].result.periph_result.target_id =
1992 				CAM_TARGET_WILDCARD;
1993 
1994 		if (periph->path->device)
1995 			cdm->matches[j].result.periph_result.target_lun =
1996 				periph->path->device->lun_id;
1997 		else
1998 			cdm->matches[j].result.periph_result.target_lun =
1999 				CAM_LUN_WILDCARD;
2000 
2001 		cdm->matches[j].result.periph_result.unit_number =
2002 			periph->unit_number;
2003 		strncpy(cdm->matches[j].result.periph_result.periph_name,
2004 			periph->periph_name, DEV_IDLEN);
2005 	}
2006 
2007 	return(1);
2008 }
2009 
2010 static int
2011 xptperiphlistmatch(struct ccb_dev_match *cdm)
2012 {
2013 	int ret;
2014 
2015 	cdm->num_matches = 0;
2016 
2017 	/*
2018 	 * At this point in the edt traversal function, we check the bus
2019 	 * list generation to make sure that no busses have been added or
2020 	 * removed since the user last sent a XPT_DEV_MATCH ccb through.
2021 	 * For the peripheral driver list traversal function, however, we
2022 	 * don't have to worry about new peripheral driver types coming or
2023 	 * going; they're in a linker set, and therefore can't change
2024 	 * without a recompile.
2025 	 */
2026 
2027 	if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2028 	 && (cdm->pos.cookie.pdrv != NULL))
2029 		ret = xptpdrvtraverse(
2030 				(struct periph_driver **)cdm->pos.cookie.pdrv,
2031 				xptplistpdrvfunc, cdm);
2032 	else
2033 		ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2034 
2035 	/*
2036 	 * If we get back 0, that means that we had to stop before fully
2037 	 * traversing the peripheral driver tree.  It also means that one of
2038 	 * the subroutines has set the status field to the proper value.  If
2039 	 * we get back 1, we've fully traversed the EDT and copied out any
2040 	 * matching entries.
2041 	 */
2042 	if (ret == 1)
2043 		cdm->status = CAM_DEV_MATCH_LAST;
2044 
2045 	return(ret);
2046 }
2047 
2048 static int
2049 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2050 {
2051 	struct cam_eb *bus, *next_bus;
2052 	int retval;
2053 
2054 	retval = 1;
2055 	if (start_bus)
2056 		bus = start_bus;
2057 	else {
2058 		xpt_lock_buses();
2059 		bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2060 		if (bus == NULL) {
2061 			xpt_unlock_buses();
2062 			return (retval);
2063 		}
2064 		bus->refcount++;
2065 		xpt_unlock_buses();
2066 	}
2067 	for (; bus != NULL; bus = next_bus) {
2068 		retval = tr_func(bus, arg);
2069 		if (retval == 0) {
2070 			xpt_release_bus(bus);
2071 			break;
2072 		}
2073 		xpt_lock_buses();
2074 		next_bus = TAILQ_NEXT(bus, links);
2075 		if (next_bus)
2076 			next_bus->refcount++;
2077 		xpt_unlock_buses();
2078 		xpt_release_bus(bus);
2079 	}
2080 	return(retval);
2081 }
2082 
2083 static int
2084 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2085 		  xpt_targetfunc_t *tr_func, void *arg)
2086 {
2087 	struct cam_et *target, *next_target;
2088 	int retval;
2089 
2090 	retval = 1;
2091 	if (start_target)
2092 		target = start_target;
2093 	else {
2094 		mtx_lock(&bus->eb_mtx);
2095 		target = TAILQ_FIRST(&bus->et_entries);
2096 		if (target == NULL) {
2097 			mtx_unlock(&bus->eb_mtx);
2098 			return (retval);
2099 		}
2100 		target->refcount++;
2101 		mtx_unlock(&bus->eb_mtx);
2102 	}
2103 	for (; target != NULL; target = next_target) {
2104 		retval = tr_func(target, arg);
2105 		if (retval == 0) {
2106 			xpt_release_target(target);
2107 			break;
2108 		}
2109 		mtx_lock(&bus->eb_mtx);
2110 		next_target = TAILQ_NEXT(target, links);
2111 		if (next_target)
2112 			next_target->refcount++;
2113 		mtx_unlock(&bus->eb_mtx);
2114 		xpt_release_target(target);
2115 	}
2116 	return(retval);
2117 }
2118 
2119 static int
2120 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2121 		  xpt_devicefunc_t *tr_func, void *arg)
2122 {
2123 	struct cam_eb *bus;
2124 	struct cam_ed *device, *next_device;
2125 	int retval;
2126 
2127 	retval = 1;
2128 	bus = target->bus;
2129 	if (start_device)
2130 		device = start_device;
2131 	else {
2132 		mtx_lock(&bus->eb_mtx);
2133 		device = TAILQ_FIRST(&target->ed_entries);
2134 		if (device == NULL) {
2135 			mtx_unlock(&bus->eb_mtx);
2136 			return (retval);
2137 		}
2138 		device->refcount++;
2139 		mtx_unlock(&bus->eb_mtx);
2140 	}
2141 	for (; device != NULL; device = next_device) {
2142 		mtx_lock(&device->device_mtx);
2143 		retval = tr_func(device, arg);
2144 		mtx_unlock(&device->device_mtx);
2145 		if (retval == 0) {
2146 			xpt_release_device(device);
2147 			break;
2148 		}
2149 		mtx_lock(&bus->eb_mtx);
2150 		next_device = TAILQ_NEXT(device, links);
2151 		if (next_device)
2152 			next_device->refcount++;
2153 		mtx_unlock(&bus->eb_mtx);
2154 		xpt_release_device(device);
2155 	}
2156 	return(retval);
2157 }
2158 
2159 static int
2160 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2161 		  xpt_periphfunc_t *tr_func, void *arg)
2162 {
2163 	struct cam_eb *bus;
2164 	struct cam_periph *periph, *next_periph;
2165 	int retval;
2166 
2167 	retval = 1;
2168 
2169 	bus = device->target->bus;
2170 	if (start_periph)
2171 		periph = start_periph;
2172 	else {
2173 		xpt_lock_buses();
2174 		mtx_lock(&bus->eb_mtx);
2175 		periph = SLIST_FIRST(&device->periphs);
2176 		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2177 			periph = SLIST_NEXT(periph, periph_links);
2178 		if (periph == NULL) {
2179 			mtx_unlock(&bus->eb_mtx);
2180 			xpt_unlock_buses();
2181 			return (retval);
2182 		}
2183 		periph->refcount++;
2184 		mtx_unlock(&bus->eb_mtx);
2185 		xpt_unlock_buses();
2186 	}
2187 	for (; periph != NULL; periph = next_periph) {
2188 		retval = tr_func(periph, arg);
2189 		if (retval == 0) {
2190 			cam_periph_release_locked(periph);
2191 			break;
2192 		}
2193 		xpt_lock_buses();
2194 		mtx_lock(&bus->eb_mtx);
2195 		next_periph = SLIST_NEXT(periph, periph_links);
2196 		while (next_periph != NULL &&
2197 		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2198 			next_periph = SLIST_NEXT(periph, periph_links);
2199 		if (next_periph)
2200 			next_periph->refcount++;
2201 		mtx_unlock(&bus->eb_mtx);
2202 		xpt_unlock_buses();
2203 		cam_periph_release_locked(periph);
2204 	}
2205 	return(retval);
2206 }
2207 
2208 static int
2209 xptpdrvtraverse(struct periph_driver **start_pdrv,
2210 		xpt_pdrvfunc_t *tr_func, void *arg)
2211 {
2212 	struct periph_driver **pdrv;
2213 	int retval;
2214 
2215 	retval = 1;
2216 
2217 	/*
2218 	 * We don't traverse the peripheral driver list like we do the
2219 	 * other lists, because it is a linker set, and therefore cannot be
2220 	 * changed during runtime.  If the peripheral driver list is ever
2221 	 * re-done to be something other than a linker set (i.e. it can
2222 	 * change while the system is running), the list traversal should
2223 	 * be modified to work like the other traversal functions.
2224 	 */
2225 	for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2226 	     *pdrv != NULL; pdrv++) {
2227 		retval = tr_func(pdrv, arg);
2228 
2229 		if (retval == 0)
2230 			return(retval);
2231 	}
2232 
2233 	return(retval);
2234 }
2235 
2236 static int
2237 xptpdperiphtraverse(struct periph_driver **pdrv,
2238 		    struct cam_periph *start_periph,
2239 		    xpt_periphfunc_t *tr_func, void *arg)
2240 {
2241 	struct cam_periph *periph, *next_periph;
2242 	int retval;
2243 
2244 	retval = 1;
2245 
2246 	if (start_periph)
2247 		periph = start_periph;
2248 	else {
2249 		xpt_lock_buses();
2250 		periph = TAILQ_FIRST(&(*pdrv)->units);
2251 		while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2252 			periph = TAILQ_NEXT(periph, unit_links);
2253 		if (periph == NULL) {
2254 			xpt_unlock_buses();
2255 			return (retval);
2256 		}
2257 		periph->refcount++;
2258 		xpt_unlock_buses();
2259 	}
2260 	for (; periph != NULL; periph = next_periph) {
2261 		cam_periph_lock(periph);
2262 		retval = tr_func(periph, arg);
2263 		cam_periph_unlock(periph);
2264 		if (retval == 0) {
2265 			cam_periph_release(periph);
2266 			break;
2267 		}
2268 		xpt_lock_buses();
2269 		next_periph = TAILQ_NEXT(periph, unit_links);
2270 		while (next_periph != NULL &&
2271 		    (next_periph->flags & CAM_PERIPH_FREE) != 0)
2272 			next_periph = TAILQ_NEXT(periph, unit_links);
2273 		if (next_periph)
2274 			next_periph->refcount++;
2275 		xpt_unlock_buses();
2276 		cam_periph_release(periph);
2277 	}
2278 	return(retval);
2279 }
2280 
2281 static int
2282 xptdefbusfunc(struct cam_eb *bus, void *arg)
2283 {
2284 	struct xpt_traverse_config *tr_config;
2285 
2286 	tr_config = (struct xpt_traverse_config *)arg;
2287 
2288 	if (tr_config->depth == XPT_DEPTH_BUS) {
2289 		xpt_busfunc_t *tr_func;
2290 
2291 		tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2292 
2293 		return(tr_func(bus, tr_config->tr_arg));
2294 	} else
2295 		return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2296 }
2297 
2298 static int
2299 xptdeftargetfunc(struct cam_et *target, void *arg)
2300 {
2301 	struct xpt_traverse_config *tr_config;
2302 
2303 	tr_config = (struct xpt_traverse_config *)arg;
2304 
2305 	if (tr_config->depth == XPT_DEPTH_TARGET) {
2306 		xpt_targetfunc_t *tr_func;
2307 
2308 		tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2309 
2310 		return(tr_func(target, tr_config->tr_arg));
2311 	} else
2312 		return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2313 }
2314 
2315 static int
2316 xptdefdevicefunc(struct cam_ed *device, void *arg)
2317 {
2318 	struct xpt_traverse_config *tr_config;
2319 
2320 	tr_config = (struct xpt_traverse_config *)arg;
2321 
2322 	if (tr_config->depth == XPT_DEPTH_DEVICE) {
2323 		xpt_devicefunc_t *tr_func;
2324 
2325 		tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2326 
2327 		return(tr_func(device, tr_config->tr_arg));
2328 	} else
2329 		return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2330 }
2331 
2332 static int
2333 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2334 {
2335 	struct xpt_traverse_config *tr_config;
2336 	xpt_periphfunc_t *tr_func;
2337 
2338 	tr_config = (struct xpt_traverse_config *)arg;
2339 
2340 	tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2341 
2342 	/*
2343 	 * Unlike the other default functions, we don't check for depth
2344 	 * here.  The peripheral driver level is the last level in the EDT,
2345 	 * so if we're here, we should execute the function in question.
2346 	 */
2347 	return(tr_func(periph, tr_config->tr_arg));
2348 }
2349 
2350 /*
2351  * Execute the given function for every bus in the EDT.
2352  */
2353 static int
2354 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2355 {
2356 	struct xpt_traverse_config tr_config;
2357 
2358 	tr_config.depth = XPT_DEPTH_BUS;
2359 	tr_config.tr_func = tr_func;
2360 	tr_config.tr_arg = arg;
2361 
2362 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2363 }
2364 
2365 /*
2366  * Execute the given function for every device in the EDT.
2367  */
2368 static int
2369 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2370 {
2371 	struct xpt_traverse_config tr_config;
2372 
2373 	tr_config.depth = XPT_DEPTH_DEVICE;
2374 	tr_config.tr_func = tr_func;
2375 	tr_config.tr_arg = arg;
2376 
2377 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2378 }
2379 
2380 static int
2381 xptsetasyncfunc(struct cam_ed *device, void *arg)
2382 {
2383 	struct cam_path path;
2384 	struct ccb_getdev cgd;
2385 	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2386 
2387 	/*
2388 	 * Don't report unconfigured devices (Wildcard devs,
2389 	 * devices only for target mode, device instances
2390 	 * that have been invalidated but are waiting for
2391 	 * their last reference count to be released).
2392 	 */
2393 	if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2394 		return (1);
2395 
2396 	xpt_compile_path(&path,
2397 			 NULL,
2398 			 device->target->bus->path_id,
2399 			 device->target->target_id,
2400 			 device->lun_id);
2401 	xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2402 	cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2403 	xpt_action((union ccb *)&cgd);
2404 	csa->callback(csa->callback_arg,
2405 			    AC_FOUND_DEVICE,
2406 			    &path, &cgd);
2407 	xpt_release_path(&path);
2408 
2409 	return(1);
2410 }
2411 
2412 static int
2413 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2414 {
2415 	struct cam_path path;
2416 	struct ccb_pathinq cpi;
2417 	struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2418 
2419 	xpt_compile_path(&path, /*periph*/NULL,
2420 			 bus->path_id,
2421 			 CAM_TARGET_WILDCARD,
2422 			 CAM_LUN_WILDCARD);
2423 	xpt_path_lock(&path);
2424 	xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2425 	cpi.ccb_h.func_code = XPT_PATH_INQ;
2426 	xpt_action((union ccb *)&cpi);
2427 	csa->callback(csa->callback_arg,
2428 			    AC_PATH_REGISTERED,
2429 			    &path, &cpi);
2430 	xpt_path_unlock(&path);
2431 	xpt_release_path(&path);
2432 
2433 	return(1);
2434 }
2435 
2436 void
2437 xpt_action(union ccb *start_ccb)
2438 {
2439 
2440 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2441 
2442 	start_ccb->ccb_h.status = CAM_REQ_INPROG;
2443 	(*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2444 }
2445 
2446 void
2447 xpt_action_default(union ccb *start_ccb)
2448 {
2449 	struct cam_path *path;
2450 	struct cam_sim *sim;
2451 	int lock;
2452 
2453 	path = start_ccb->ccb_h.path;
2454 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2455 
2456 	switch (start_ccb->ccb_h.func_code) {
2457 	case XPT_SCSI_IO:
2458 	{
2459 		struct cam_ed *device;
2460 
2461 		/*
2462 		 * For the sake of compatibility with SCSI-1
2463 		 * devices that may not understand the identify
2464 		 * message, we include lun information in the
2465 		 * second byte of all commands.  SCSI-1 specifies
2466 		 * that luns are a 3 bit value and reserves only 3
2467 		 * bits for lun information in the CDB.  Later
2468 		 * revisions of the SCSI spec allow for more than 8
2469 		 * luns, but have deprecated lun information in the
2470 		 * CDB.  So, if the lun won't fit, we must omit.
2471 		 *
2472 		 * Also be aware that during initial probing for devices,
2473 		 * the inquiry information is unknown but initialized to 0.
2474 		 * This means that this code will be exercised while probing
2475 		 * devices with an ANSI revision greater than 2.
2476 		 */
2477 		device = path->device;
2478 		if (device->protocol_version <= SCSI_REV_2
2479 		 && start_ccb->ccb_h.target_lun < 8
2480 		 && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2481 
2482 			start_ccb->csio.cdb_io.cdb_bytes[1] |=
2483 			    start_ccb->ccb_h.target_lun << 5;
2484 		}
2485 		start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2486 	}
2487 	/* FALLTHROUGH */
2488 	case XPT_TARGET_IO:
2489 	case XPT_CONT_TARGET_IO:
2490 		start_ccb->csio.sense_resid = 0;
2491 		start_ccb->csio.resid = 0;
2492 		/* FALLTHROUGH */
2493 	case XPT_ATA_IO:
2494 		if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2495 			start_ccb->ataio.resid = 0;
2496 		/* FALLTHROUGH */
2497 	case XPT_RESET_DEV:
2498 	case XPT_ENG_EXEC:
2499 	case XPT_SMP_IO:
2500 	{
2501 		struct cam_devq *devq;
2502 
2503 		devq = path->bus->sim->devq;
2504 		mtx_lock(&devq->send_mtx);
2505 		cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2506 		if (xpt_schedule_devq(devq, path->device) != 0)
2507 			xpt_run_devq(devq);
2508 		mtx_unlock(&devq->send_mtx);
2509 		break;
2510 	}
2511 	case XPT_CALC_GEOMETRY:
2512 		/* Filter out garbage */
2513 		if (start_ccb->ccg.block_size == 0
2514 		 || start_ccb->ccg.volume_size == 0) {
2515 			start_ccb->ccg.cylinders = 0;
2516 			start_ccb->ccg.heads = 0;
2517 			start_ccb->ccg.secs_per_track = 0;
2518 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2519 			break;
2520 		}
2521 #if defined(PC98) || defined(__sparc64__)
2522 		/*
2523 		 * In a PC-98 system, geometry translation depens on
2524 		 * the "real" device geometry obtained from mode page 4.
2525 		 * SCSI geometry translation is performed in the
2526 		 * initialization routine of the SCSI BIOS and the result
2527 		 * stored in host memory.  If the translation is available
2528 		 * in host memory, use it.  If not, rely on the default
2529 		 * translation the device driver performs.
2530 		 * For sparc64, we may need adjust the geometry of large
2531 		 * disks in order to fit the limitations of the 16-bit
2532 		 * fields of the VTOC8 disk label.
2533 		 */
2534 		if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2535 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2536 			break;
2537 		}
2538 #endif
2539 		goto call_sim;
2540 	case XPT_ABORT:
2541 	{
2542 		union ccb* abort_ccb;
2543 
2544 		abort_ccb = start_ccb->cab.abort_ccb;
2545 		if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2546 
2547 			if (abort_ccb->ccb_h.pinfo.index >= 0) {
2548 				struct cam_ccbq *ccbq;
2549 				struct cam_ed *device;
2550 
2551 				device = abort_ccb->ccb_h.path->device;
2552 				ccbq = &device->ccbq;
2553 				cam_ccbq_remove_ccb(ccbq, abort_ccb);
2554 				abort_ccb->ccb_h.status =
2555 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2556 				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2557 				xpt_done(abort_ccb);
2558 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2559 				break;
2560 			}
2561 			if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2562 			 && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2563 				/*
2564 				 * We've caught this ccb en route to
2565 				 * the SIM.  Flag it for abort and the
2566 				 * SIM will do so just before starting
2567 				 * real work on the CCB.
2568 				 */
2569 				abort_ccb->ccb_h.status =
2570 				    CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2571 				xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2572 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2573 				break;
2574 			}
2575 		}
2576 		if (XPT_FC_IS_QUEUED(abort_ccb)
2577 		 && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2578 			/*
2579 			 * It's already completed but waiting
2580 			 * for our SWI to get to it.
2581 			 */
2582 			start_ccb->ccb_h.status = CAM_UA_ABORT;
2583 			break;
2584 		}
2585 		/*
2586 		 * If we weren't able to take care of the abort request
2587 		 * in the XPT, pass the request down to the SIM for processing.
2588 		 */
2589 	}
2590 	/* FALLTHROUGH */
2591 	case XPT_ACCEPT_TARGET_IO:
2592 	case XPT_EN_LUN:
2593 	case XPT_IMMED_NOTIFY:
2594 	case XPT_NOTIFY_ACK:
2595 	case XPT_RESET_BUS:
2596 	case XPT_IMMEDIATE_NOTIFY:
2597 	case XPT_NOTIFY_ACKNOWLEDGE:
2598 	case XPT_GET_SIM_KNOB:
2599 	case XPT_SET_SIM_KNOB:
2600 	case XPT_GET_TRAN_SETTINGS:
2601 	case XPT_SET_TRAN_SETTINGS:
2602 	case XPT_PATH_INQ:
2603 call_sim:
2604 		sim = path->bus->sim;
2605 		lock = (mtx_owned(sim->mtx) == 0);
2606 		if (lock)
2607 			CAM_SIM_LOCK(sim);
2608 		(*(sim->sim_action))(sim, start_ccb);
2609 		if (lock)
2610 			CAM_SIM_UNLOCK(sim);
2611 		break;
2612 	case XPT_PATH_STATS:
2613 		start_ccb->cpis.last_reset = path->bus->last_reset;
2614 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2615 		break;
2616 	case XPT_GDEV_TYPE:
2617 	{
2618 		struct cam_ed *dev;
2619 
2620 		dev = path->device;
2621 		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2622 			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2623 		} else {
2624 			struct ccb_getdev *cgd;
2625 
2626 			cgd = &start_ccb->cgd;
2627 			cgd->protocol = dev->protocol;
2628 			cgd->inq_data = dev->inq_data;
2629 			cgd->ident_data = dev->ident_data;
2630 			cgd->inq_flags = dev->inq_flags;
2631 			cgd->ccb_h.status = CAM_REQ_CMP;
2632 			cgd->serial_num_len = dev->serial_num_len;
2633 			if ((dev->serial_num_len > 0)
2634 			 && (dev->serial_num != NULL))
2635 				bcopy(dev->serial_num, cgd->serial_num,
2636 				      dev->serial_num_len);
2637 		}
2638 		break;
2639 	}
2640 	case XPT_GDEV_STATS:
2641 	{
2642 		struct cam_ed *dev;
2643 
2644 		dev = path->device;
2645 		if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2646 			start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2647 		} else {
2648 			struct ccb_getdevstats *cgds;
2649 			struct cam_eb *bus;
2650 			struct cam_et *tar;
2651 
2652 			cgds = &start_ccb->cgds;
2653 			bus = path->bus;
2654 			tar = path->target;
2655 			cgds->dev_openings = dev->ccbq.dev_openings;
2656 			cgds->dev_active = dev->ccbq.dev_active;
2657 			cgds->devq_openings = dev->ccbq.devq_openings;
2658 			cgds->devq_queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2659 			cgds->held = dev->ccbq.held;
2660 			cgds->last_reset = tar->last_reset;
2661 			cgds->maxtags = dev->maxtags;
2662 			cgds->mintags = dev->mintags;
2663 			if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2664 				cgds->last_reset = bus->last_reset;
2665 			cgds->ccb_h.status = CAM_REQ_CMP;
2666 		}
2667 		break;
2668 	}
2669 	case XPT_GDEVLIST:
2670 	{
2671 		struct cam_periph	*nperiph;
2672 		struct periph_list	*periph_head;
2673 		struct ccb_getdevlist	*cgdl;
2674 		u_int			i;
2675 		struct cam_ed		*device;
2676 		int			found;
2677 
2678 
2679 		found = 0;
2680 
2681 		/*
2682 		 * Don't want anyone mucking with our data.
2683 		 */
2684 		device = path->device;
2685 		periph_head = &device->periphs;
2686 		cgdl = &start_ccb->cgdl;
2687 
2688 		/*
2689 		 * Check and see if the list has changed since the user
2690 		 * last requested a list member.  If so, tell them that the
2691 		 * list has changed, and therefore they need to start over
2692 		 * from the beginning.
2693 		 */
2694 		if ((cgdl->index != 0) &&
2695 		    (cgdl->generation != device->generation)) {
2696 			cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2697 			break;
2698 		}
2699 
2700 		/*
2701 		 * Traverse the list of peripherals and attempt to find
2702 		 * the requested peripheral.
2703 		 */
2704 		for (nperiph = SLIST_FIRST(periph_head), i = 0;
2705 		     (nperiph != NULL) && (i <= cgdl->index);
2706 		     nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2707 			if (i == cgdl->index) {
2708 				strncpy(cgdl->periph_name,
2709 					nperiph->periph_name,
2710 					DEV_IDLEN);
2711 				cgdl->unit_number = nperiph->unit_number;
2712 				found = 1;
2713 			}
2714 		}
2715 		if (found == 0) {
2716 			cgdl->status = CAM_GDEVLIST_ERROR;
2717 			break;
2718 		}
2719 
2720 		if (nperiph == NULL)
2721 			cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2722 		else
2723 			cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2724 
2725 		cgdl->index++;
2726 		cgdl->generation = device->generation;
2727 
2728 		cgdl->ccb_h.status = CAM_REQ_CMP;
2729 		break;
2730 	}
2731 	case XPT_DEV_MATCH:
2732 	{
2733 		dev_pos_type position_type;
2734 		struct ccb_dev_match *cdm;
2735 
2736 		cdm = &start_ccb->cdm;
2737 
2738 		/*
2739 		 * There are two ways of getting at information in the EDT.
2740 		 * The first way is via the primary EDT tree.  It starts
2741 		 * with a list of busses, then a list of targets on a bus,
2742 		 * then devices/luns on a target, and then peripherals on a
2743 		 * device/lun.  The "other" way is by the peripheral driver
2744 		 * lists.  The peripheral driver lists are organized by
2745 		 * peripheral driver.  (obviously)  So it makes sense to
2746 		 * use the peripheral driver list if the user is looking
2747 		 * for something like "da1", or all "da" devices.  If the
2748 		 * user is looking for something on a particular bus/target
2749 		 * or lun, it's generally better to go through the EDT tree.
2750 		 */
2751 
2752 		if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2753 			position_type = cdm->pos.position_type;
2754 		else {
2755 			u_int i;
2756 
2757 			position_type = CAM_DEV_POS_NONE;
2758 
2759 			for (i = 0; i < cdm->num_patterns; i++) {
2760 				if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2761 				 ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2762 					position_type = CAM_DEV_POS_EDT;
2763 					break;
2764 				}
2765 			}
2766 
2767 			if (cdm->num_patterns == 0)
2768 				position_type = CAM_DEV_POS_EDT;
2769 			else if (position_type == CAM_DEV_POS_NONE)
2770 				position_type = CAM_DEV_POS_PDRV;
2771 		}
2772 
2773 		switch(position_type & CAM_DEV_POS_TYPEMASK) {
2774 		case CAM_DEV_POS_EDT:
2775 			xptedtmatch(cdm);
2776 			break;
2777 		case CAM_DEV_POS_PDRV:
2778 			xptperiphlistmatch(cdm);
2779 			break;
2780 		default:
2781 			cdm->status = CAM_DEV_MATCH_ERROR;
2782 			break;
2783 		}
2784 
2785 		if (cdm->status == CAM_DEV_MATCH_ERROR)
2786 			start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2787 		else
2788 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2789 
2790 		break;
2791 	}
2792 	case XPT_SASYNC_CB:
2793 	{
2794 		struct ccb_setasync *csa;
2795 		struct async_node *cur_entry;
2796 		struct async_list *async_head;
2797 		u_int32_t added;
2798 
2799 		csa = &start_ccb->csa;
2800 		added = csa->event_enable;
2801 		async_head = &path->device->asyncs;
2802 
2803 		/*
2804 		 * If there is already an entry for us, simply
2805 		 * update it.
2806 		 */
2807 		cur_entry = SLIST_FIRST(async_head);
2808 		while (cur_entry != NULL) {
2809 			if ((cur_entry->callback_arg == csa->callback_arg)
2810 			 && (cur_entry->callback == csa->callback))
2811 				break;
2812 			cur_entry = SLIST_NEXT(cur_entry, links);
2813 		}
2814 
2815 		if (cur_entry != NULL) {
2816 		 	/*
2817 			 * If the request has no flags set,
2818 			 * remove the entry.
2819 			 */
2820 			added &= ~cur_entry->event_enable;
2821 			if (csa->event_enable == 0) {
2822 				SLIST_REMOVE(async_head, cur_entry,
2823 					     async_node, links);
2824 				xpt_release_device(path->device);
2825 				free(cur_entry, M_CAMXPT);
2826 			} else {
2827 				cur_entry->event_enable = csa->event_enable;
2828 			}
2829 			csa->event_enable = added;
2830 		} else {
2831 			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2832 					   M_NOWAIT);
2833 			if (cur_entry == NULL) {
2834 				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2835 				break;
2836 			}
2837 			cur_entry->event_enable = csa->event_enable;
2838 			cur_entry->event_lock =
2839 			    mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2840 			cur_entry->callback_arg = csa->callback_arg;
2841 			cur_entry->callback = csa->callback;
2842 			SLIST_INSERT_HEAD(async_head, cur_entry, links);
2843 			xpt_acquire_device(path->device);
2844 		}
2845 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2846 		break;
2847 	}
2848 	case XPT_REL_SIMQ:
2849 	{
2850 		struct ccb_relsim *crs;
2851 		struct cam_ed *dev;
2852 
2853 		crs = &start_ccb->crs;
2854 		dev = path->device;
2855 		if (dev == NULL) {
2856 
2857 			crs->ccb_h.status = CAM_DEV_NOT_THERE;
2858 			break;
2859 		}
2860 
2861 		if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2862 
2863 			/* Don't ever go below one opening */
2864 			if (crs->openings > 0) {
2865 				xpt_dev_ccbq_resize(path, crs->openings);
2866 				if (bootverbose) {
2867 					xpt_print(path,
2868 					    "number of openings is now %d\n",
2869 					    crs->openings);
2870 				}
2871 			}
2872 		}
2873 
2874 		mtx_lock(&dev->sim->devq->send_mtx);
2875 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2876 
2877 			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2878 
2879 				/*
2880 				 * Just extend the old timeout and decrement
2881 				 * the freeze count so that a single timeout
2882 				 * is sufficient for releasing the queue.
2883 				 */
2884 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2885 				callout_stop(&dev->callout);
2886 			} else {
2887 
2888 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2889 			}
2890 
2891 			callout_reset(&dev->callout,
2892 			    (crs->release_timeout * hz) / 1000,
2893 			    xpt_release_devq_timeout, dev);
2894 
2895 			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2896 
2897 		}
2898 
2899 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2900 
2901 			if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2902 				/*
2903 				 * Decrement the freeze count so that a single
2904 				 * completion is still sufficient to unfreeze
2905 				 * the queue.
2906 				 */
2907 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2908 			} else {
2909 
2910 				dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2911 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2912 			}
2913 		}
2914 
2915 		if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2916 
2917 			if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2918 			 || (dev->ccbq.dev_active == 0)) {
2919 
2920 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2921 			} else {
2922 
2923 				dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2924 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2925 			}
2926 		}
2927 		mtx_unlock(&dev->sim->devq->send_mtx);
2928 
2929 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2930 			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2931 		start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2932 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2933 		break;
2934 	}
2935 	case XPT_DEBUG: {
2936 		struct cam_path *oldpath;
2937 
2938 		/* Check that all request bits are supported. */
2939 		if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2940 			start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2941 			break;
2942 		}
2943 
2944 		cam_dflags = CAM_DEBUG_NONE;
2945 		if (cam_dpath != NULL) {
2946 			oldpath = cam_dpath;
2947 			cam_dpath = NULL;
2948 			xpt_free_path(oldpath);
2949 		}
2950 		if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2951 			if (xpt_create_path(&cam_dpath, NULL,
2952 					    start_ccb->ccb_h.path_id,
2953 					    start_ccb->ccb_h.target_id,
2954 					    start_ccb->ccb_h.target_lun) !=
2955 					    CAM_REQ_CMP) {
2956 				start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2957 			} else {
2958 				cam_dflags = start_ccb->cdbg.flags;
2959 				start_ccb->ccb_h.status = CAM_REQ_CMP;
2960 				xpt_print(cam_dpath, "debugging flags now %x\n",
2961 				    cam_dflags);
2962 			}
2963 		} else
2964 			start_ccb->ccb_h.status = CAM_REQ_CMP;
2965 		break;
2966 	}
2967 	case XPT_NOOP:
2968 		if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2969 			xpt_freeze_devq(path, 1);
2970 		start_ccb->ccb_h.status = CAM_REQ_CMP;
2971 		break;
2972 	default:
2973 	case XPT_SDEV_TYPE:
2974 	case XPT_TERM_IO:
2975 	case XPT_ENG_INQ:
2976 		/* XXX Implement */
2977 		printf("%s: CCB type %#x not supported\n", __func__,
2978 		       start_ccb->ccb_h.func_code);
2979 		start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
2980 		if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
2981 			xpt_done(start_ccb);
2982 		}
2983 		break;
2984 	}
2985 }
2986 
2987 void
2988 xpt_polled_action(union ccb *start_ccb)
2989 {
2990 	u_int32_t timeout;
2991 	struct	  cam_sim *sim;
2992 	struct	  cam_devq *devq;
2993 	struct	  cam_ed *dev;
2994 
2995 	timeout = start_ccb->ccb_h.timeout * 10;
2996 	sim = start_ccb->ccb_h.path->bus->sim;
2997 	devq = sim->devq;
2998 	dev = start_ccb->ccb_h.path->device;
2999 
3000 	mtx_unlock(&dev->device_mtx);
3001 
3002 	/*
3003 	 * Steal an opening so that no other queued requests
3004 	 * can get it before us while we simulate interrupts.
3005 	 */
3006 	mtx_lock(&devq->send_mtx);
3007 	dev->ccbq.devq_openings--;
3008 	dev->ccbq.dev_openings--;
3009 	while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3010 	    (--timeout > 0)) {
3011 		mtx_unlock(&devq->send_mtx);
3012 		DELAY(100);
3013 		CAM_SIM_LOCK(sim);
3014 		(*(sim->sim_poll))(sim);
3015 		CAM_SIM_UNLOCK(sim);
3016 		camisr_runqueue();
3017 		mtx_lock(&devq->send_mtx);
3018 	}
3019 	dev->ccbq.devq_openings++;
3020 	dev->ccbq.dev_openings++;
3021 	mtx_unlock(&devq->send_mtx);
3022 
3023 	if (timeout != 0) {
3024 		xpt_action(start_ccb);
3025 		while(--timeout > 0) {
3026 			CAM_SIM_LOCK(sim);
3027 			(*(sim->sim_poll))(sim);
3028 			CAM_SIM_UNLOCK(sim);
3029 			camisr_runqueue();
3030 			if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3031 			    != CAM_REQ_INPROG)
3032 				break;
3033 			DELAY(100);
3034 		}
3035 		if (timeout == 0) {
3036 			/*
3037 			 * XXX Is it worth adding a sim_timeout entry
3038 			 * point so we can attempt recovery?  If
3039 			 * this is only used for dumps, I don't think
3040 			 * it is.
3041 			 */
3042 			start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3043 		}
3044 	} else {
3045 		start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3046 	}
3047 
3048 	mtx_lock(&dev->device_mtx);
3049 }
3050 
3051 /*
3052  * Schedule a peripheral driver to receive a ccb when it's
3053  * target device has space for more transactions.
3054  */
3055 void
3056 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3057 {
3058 
3059 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3060 	cam_periph_assert(periph, MA_OWNED);
3061 	if (new_priority < periph->scheduled_priority) {
3062 		periph->scheduled_priority = new_priority;
3063 		xpt_run_allocq(periph, 0);
3064 	}
3065 }
3066 
3067 
3068 /*
3069  * Schedule a device to run on a given queue.
3070  * If the device was inserted as a new entry on the queue,
3071  * return 1 meaning the device queue should be run. If we
3072  * were already queued, implying someone else has already
3073  * started the queue, return 0 so the caller doesn't attempt
3074  * to run the queue.
3075  */
3076 static int
3077 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3078 		 u_int32_t new_priority)
3079 {
3080 	int retval;
3081 	u_int32_t old_priority;
3082 
3083 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3084 
3085 	old_priority = pinfo->priority;
3086 
3087 	/*
3088 	 * Are we already queued?
3089 	 */
3090 	if (pinfo->index != CAM_UNQUEUED_INDEX) {
3091 		/* Simply reorder based on new priority */
3092 		if (new_priority < old_priority) {
3093 			camq_change_priority(queue, pinfo->index,
3094 					     new_priority);
3095 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3096 					("changed priority to %d\n",
3097 					 new_priority));
3098 			retval = 1;
3099 		} else
3100 			retval = 0;
3101 	} else {
3102 		/* New entry on the queue */
3103 		if (new_priority < old_priority)
3104 			pinfo->priority = new_priority;
3105 
3106 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3107 				("Inserting onto queue\n"));
3108 		pinfo->generation = ++queue->generation;
3109 		camq_insert(queue, pinfo);
3110 		retval = 1;
3111 	}
3112 	return (retval);
3113 }
3114 
3115 static void
3116 xpt_run_allocq_task(void *context, int pending)
3117 {
3118 	struct cam_periph *periph = context;
3119 
3120 	cam_periph_lock(periph);
3121 	periph->flags &= ~CAM_PERIPH_RUN_TASK;
3122 	xpt_run_allocq(periph, 1);
3123 	cam_periph_unlock(periph);
3124 	cam_periph_release(periph);
3125 }
3126 
3127 static void
3128 xpt_run_allocq(struct cam_periph *periph, int sleep)
3129 {
3130 	struct cam_ed	*device;
3131 	union ccb	*ccb;
3132 	uint32_t	 prio;
3133 
3134 	cam_periph_assert(periph, MA_OWNED);
3135 	if (periph->periph_allocating)
3136 		return;
3137 	periph->periph_allocating = 1;
3138 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3139 	device = periph->path->device;
3140 	ccb = NULL;
3141 restart:
3142 	while ((prio = min(periph->scheduled_priority,
3143 	    periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3144 	    (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3145 	     device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3146 
3147 		if (ccb == NULL &&
3148 		    (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3149 			if (sleep) {
3150 				ccb = xpt_get_ccb(periph);
3151 				goto restart;
3152 			}
3153 			if (periph->flags & CAM_PERIPH_RUN_TASK)
3154 				break;
3155 			cam_periph_doacquire(periph);
3156 			periph->flags |= CAM_PERIPH_RUN_TASK;
3157 			taskqueue_enqueue(xsoftc.xpt_taskq,
3158 			    &periph->periph_run_task);
3159 			break;
3160 		}
3161 		xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3162 		if (prio == periph->immediate_priority) {
3163 			periph->immediate_priority = CAM_PRIORITY_NONE;
3164 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3165 					("waking cam_periph_getccb()\n"));
3166 			SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3167 					  periph_links.sle);
3168 			wakeup(&periph->ccb_list);
3169 		} else {
3170 			periph->scheduled_priority = CAM_PRIORITY_NONE;
3171 			CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3172 					("calling periph_start()\n"));
3173 			periph->periph_start(periph, ccb);
3174 		}
3175 		ccb = NULL;
3176 	}
3177 	if (ccb != NULL)
3178 		xpt_release_ccb(ccb);
3179 	periph->periph_allocating = 0;
3180 }
3181 
3182 static void
3183 xpt_run_devq(struct cam_devq *devq)
3184 {
3185 	char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3186 	int lock;
3187 
3188 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3189 
3190 	devq->send_queue.qfrozen_cnt++;
3191 	while ((devq->send_queue.entries > 0)
3192 	    && (devq->send_openings > 0)
3193 	    && (devq->send_queue.qfrozen_cnt <= 1)) {
3194 		struct	cam_ed *device;
3195 		union ccb *work_ccb;
3196 		struct	cam_sim *sim;
3197 
3198 		device = (struct cam_ed *)camq_remove(&devq->send_queue,
3199 							   CAMQ_HEAD);
3200 		CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3201 				("running device %p\n", device));
3202 
3203 		work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3204 		if (work_ccb == NULL) {
3205 			printf("device on run queue with no ccbs???\n");
3206 			continue;
3207 		}
3208 
3209 		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3210 
3211 			mtx_lock(&xsoftc.xpt_highpower_lock);
3212 		 	if (xsoftc.num_highpower <= 0) {
3213 				/*
3214 				 * We got a high power command, but we
3215 				 * don't have any available slots.  Freeze
3216 				 * the device queue until we have a slot
3217 				 * available.
3218 				 */
3219 				xpt_freeze_devq_device(device, 1);
3220 				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3221 						   highpowerq_entry);
3222 
3223 				mtx_unlock(&xsoftc.xpt_highpower_lock);
3224 				continue;
3225 			} else {
3226 				/*
3227 				 * Consume a high power slot while
3228 				 * this ccb runs.
3229 				 */
3230 				xsoftc.num_highpower--;
3231 			}
3232 			mtx_unlock(&xsoftc.xpt_highpower_lock);
3233 		}
3234 		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3235 		cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3236 		devq->send_openings--;
3237 		devq->send_active++;
3238 		xpt_schedule_devq(devq, device);
3239 		mtx_unlock(&devq->send_mtx);
3240 
3241 		if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3242 			/*
3243 			 * The client wants to freeze the queue
3244 			 * after this CCB is sent.
3245 			 */
3246 			xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3247 		}
3248 
3249 		/* In Target mode, the peripheral driver knows best... */
3250 		if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3251 			if ((device->inq_flags & SID_CmdQue) != 0
3252 			 && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3253 				work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3254 			else
3255 				/*
3256 				 * Clear this in case of a retried CCB that
3257 				 * failed due to a rejected tag.
3258 				 */
3259 				work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3260 		}
3261 
3262 		switch (work_ccb->ccb_h.func_code) {
3263 		case XPT_SCSI_IO:
3264 			CAM_DEBUG(work_ccb->ccb_h.path,
3265 			    CAM_DEBUG_CDB,("%s. CDB: %s\n",
3266 			     scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3267 					  &device->inq_data),
3268 			     scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3269 					     cdb_str, sizeof(cdb_str))));
3270 			break;
3271 		case XPT_ATA_IO:
3272 			CAM_DEBUG(work_ccb->ccb_h.path,
3273 			    CAM_DEBUG_CDB,("%s. ACB: %s\n",
3274 			     ata_op_string(&work_ccb->ataio.cmd),
3275 			     ata_cmd_string(&work_ccb->ataio.cmd,
3276 					    cdb_str, sizeof(cdb_str))));
3277 			break;
3278 		default:
3279 			break;
3280 		}
3281 
3282 		/*
3283 		 * Device queues can be shared among multiple SIM instances
3284 		 * that reside on different busses.  Use the SIM from the
3285 		 * queued device, rather than the one from the calling bus.
3286 		 */
3287 		sim = device->sim;
3288 		lock = (mtx_owned(sim->mtx) == 0);
3289 		if (lock)
3290 			CAM_SIM_LOCK(sim);
3291 		(*(sim->sim_action))(sim, work_ccb);
3292 		if (lock)
3293 			CAM_SIM_UNLOCK(sim);
3294 		mtx_lock(&devq->send_mtx);
3295 	}
3296 	devq->send_queue.qfrozen_cnt--;
3297 }
3298 
3299 /*
3300  * This function merges stuff from the slave ccb into the master ccb, while
3301  * keeping important fields in the master ccb constant.
3302  */
3303 void
3304 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3305 {
3306 
3307 	/*
3308 	 * Pull fields that are valid for peripheral drivers to set
3309 	 * into the master CCB along with the CCB "payload".
3310 	 */
3311 	master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3312 	master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3313 	master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3314 	master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3315 	bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3316 	      sizeof(union ccb) - sizeof(struct ccb_hdr));
3317 }
3318 
3319 void
3320 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3321 {
3322 
3323 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3324 	ccb_h->pinfo.priority = priority;
3325 	ccb_h->path = path;
3326 	ccb_h->path_id = path->bus->path_id;
3327 	if (path->target)
3328 		ccb_h->target_id = path->target->target_id;
3329 	else
3330 		ccb_h->target_id = CAM_TARGET_WILDCARD;
3331 	if (path->device) {
3332 		ccb_h->target_lun = path->device->lun_id;
3333 		ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3334 	} else {
3335 		ccb_h->target_lun = CAM_TARGET_WILDCARD;
3336 	}
3337 	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3338 	ccb_h->flags = 0;
3339 	ccb_h->xflags = 0;
3340 }
3341 
3342 /* Path manipulation functions */
3343 cam_status
3344 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3345 		path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3346 {
3347 	struct	   cam_path *path;
3348 	cam_status status;
3349 
3350 	path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3351 
3352 	if (path == NULL) {
3353 		status = CAM_RESRC_UNAVAIL;
3354 		return(status);
3355 	}
3356 	status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3357 	if (status != CAM_REQ_CMP) {
3358 		free(path, M_CAMPATH);
3359 		path = NULL;
3360 	}
3361 	*new_path_ptr = path;
3362 	return (status);
3363 }
3364 
3365 cam_status
3366 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3367 			 struct cam_periph *periph, path_id_t path_id,
3368 			 target_id_t target_id, lun_id_t lun_id)
3369 {
3370 
3371 	return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3372 	    lun_id));
3373 }
3374 
3375 cam_status
3376 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3377 		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3378 {
3379 	struct	     cam_eb *bus;
3380 	struct	     cam_et *target;
3381 	struct	     cam_ed *device;
3382 	cam_status   status;
3383 
3384 	status = CAM_REQ_CMP;	/* Completed without error */
3385 	target = NULL;		/* Wildcarded */
3386 	device = NULL;		/* Wildcarded */
3387 
3388 	/*
3389 	 * We will potentially modify the EDT, so block interrupts
3390 	 * that may attempt to create cam paths.
3391 	 */
3392 	bus = xpt_find_bus(path_id);
3393 	if (bus == NULL) {
3394 		status = CAM_PATH_INVALID;
3395 	} else {
3396 		xpt_lock_buses();
3397 		mtx_lock(&bus->eb_mtx);
3398 		target = xpt_find_target(bus, target_id);
3399 		if (target == NULL) {
3400 			/* Create one */
3401 			struct cam_et *new_target;
3402 
3403 			new_target = xpt_alloc_target(bus, target_id);
3404 			if (new_target == NULL) {
3405 				status = CAM_RESRC_UNAVAIL;
3406 			} else {
3407 				target = new_target;
3408 			}
3409 		}
3410 		xpt_unlock_buses();
3411 		if (target != NULL) {
3412 			device = xpt_find_device(target, lun_id);
3413 			if (device == NULL) {
3414 				/* Create one */
3415 				struct cam_ed *new_device;
3416 
3417 				new_device =
3418 				    (*(bus->xport->alloc_device))(bus,
3419 								      target,
3420 								      lun_id);
3421 				if (new_device == NULL) {
3422 					status = CAM_RESRC_UNAVAIL;
3423 				} else {
3424 					device = new_device;
3425 				}
3426 			}
3427 		}
3428 		mtx_unlock(&bus->eb_mtx);
3429 	}
3430 
3431 	/*
3432 	 * Only touch the user's data if we are successful.
3433 	 */
3434 	if (status == CAM_REQ_CMP) {
3435 		new_path->periph = perph;
3436 		new_path->bus = bus;
3437 		new_path->target = target;
3438 		new_path->device = device;
3439 		CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3440 	} else {
3441 		if (device != NULL)
3442 			xpt_release_device(device);
3443 		if (target != NULL)
3444 			xpt_release_target(target);
3445 		if (bus != NULL)
3446 			xpt_release_bus(bus);
3447 	}
3448 	return (status);
3449 }
3450 
3451 cam_status
3452 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3453 {
3454 	struct	   cam_path *new_path;
3455 
3456 	new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3457 	if (new_path == NULL)
3458 		return(CAM_RESRC_UNAVAIL);
3459 	xpt_copy_path(new_path, path);
3460 	*new_path_ptr = new_path;
3461 	return (CAM_REQ_CMP);
3462 }
3463 
3464 void
3465 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3466 {
3467 
3468 	*new_path = *path;
3469 	if (path->bus != NULL)
3470 		xpt_acquire_bus(path->bus);
3471 	if (path->target != NULL)
3472 		xpt_acquire_target(path->target);
3473 	if (path->device != NULL)
3474 		xpt_acquire_device(path->device);
3475 }
3476 
3477 void
3478 xpt_release_path(struct cam_path *path)
3479 {
3480 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3481 	if (path->device != NULL) {
3482 		xpt_release_device(path->device);
3483 		path->device = NULL;
3484 	}
3485 	if (path->target != NULL) {
3486 		xpt_release_target(path->target);
3487 		path->target = NULL;
3488 	}
3489 	if (path->bus != NULL) {
3490 		xpt_release_bus(path->bus);
3491 		path->bus = NULL;
3492 	}
3493 }
3494 
3495 void
3496 xpt_free_path(struct cam_path *path)
3497 {
3498 
3499 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3500 	xpt_release_path(path);
3501 	free(path, M_CAMPATH);
3502 }
3503 
3504 void
3505 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3506     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3507 {
3508 
3509 	xpt_lock_buses();
3510 	if (bus_ref) {
3511 		if (path->bus)
3512 			*bus_ref = path->bus->refcount;
3513 		else
3514 			*bus_ref = 0;
3515 	}
3516 	if (periph_ref) {
3517 		if (path->periph)
3518 			*periph_ref = path->periph->refcount;
3519 		else
3520 			*periph_ref = 0;
3521 	}
3522 	xpt_unlock_buses();
3523 	if (target_ref) {
3524 		if (path->target)
3525 			*target_ref = path->target->refcount;
3526 		else
3527 			*target_ref = 0;
3528 	}
3529 	if (device_ref) {
3530 		if (path->device)
3531 			*device_ref = path->device->refcount;
3532 		else
3533 			*device_ref = 0;
3534 	}
3535 }
3536 
3537 /*
3538  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3539  * in path1, 2 for match with wildcards in path2.
3540  */
3541 int
3542 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3543 {
3544 	int retval = 0;
3545 
3546 	if (path1->bus != path2->bus) {
3547 		if (path1->bus->path_id == CAM_BUS_WILDCARD)
3548 			retval = 1;
3549 		else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3550 			retval = 2;
3551 		else
3552 			return (-1);
3553 	}
3554 	if (path1->target != path2->target) {
3555 		if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3556 			if (retval == 0)
3557 				retval = 1;
3558 		} else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3559 			retval = 2;
3560 		else
3561 			return (-1);
3562 	}
3563 	if (path1->device != path2->device) {
3564 		if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3565 			if (retval == 0)
3566 				retval = 1;
3567 		} else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3568 			retval = 2;
3569 		else
3570 			return (-1);
3571 	}
3572 	return (retval);
3573 }
3574 
3575 int
3576 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3577 {
3578 	int retval = 0;
3579 
3580 	if (path->bus != dev->target->bus) {
3581 		if (path->bus->path_id == CAM_BUS_WILDCARD)
3582 			retval = 1;
3583 		else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3584 			retval = 2;
3585 		else
3586 			return (-1);
3587 	}
3588 	if (path->target != dev->target) {
3589 		if (path->target->target_id == CAM_TARGET_WILDCARD) {
3590 			if (retval == 0)
3591 				retval = 1;
3592 		} else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3593 			retval = 2;
3594 		else
3595 			return (-1);
3596 	}
3597 	if (path->device != dev) {
3598 		if (path->device->lun_id == CAM_LUN_WILDCARD) {
3599 			if (retval == 0)
3600 				retval = 1;
3601 		} else if (dev->lun_id == CAM_LUN_WILDCARD)
3602 			retval = 2;
3603 		else
3604 			return (-1);
3605 	}
3606 	return (retval);
3607 }
3608 
3609 void
3610 xpt_print_path(struct cam_path *path)
3611 {
3612 
3613 	if (path == NULL)
3614 		printf("(nopath): ");
3615 	else {
3616 		if (path->periph != NULL)
3617 			printf("(%s%d:", path->periph->periph_name,
3618 			       path->periph->unit_number);
3619 		else
3620 			printf("(noperiph:");
3621 
3622 		if (path->bus != NULL)
3623 			printf("%s%d:%d:", path->bus->sim->sim_name,
3624 			       path->bus->sim->unit_number,
3625 			       path->bus->sim->bus_id);
3626 		else
3627 			printf("nobus:");
3628 
3629 		if (path->target != NULL)
3630 			printf("%d:", path->target->target_id);
3631 		else
3632 			printf("X:");
3633 
3634 		if (path->device != NULL)
3635 			printf("%jx): ", (uintmax_t)path->device->lun_id);
3636 		else
3637 			printf("X): ");
3638 	}
3639 }
3640 
3641 void
3642 xpt_print_device(struct cam_ed *device)
3643 {
3644 
3645 	if (device == NULL)
3646 		printf("(nopath): ");
3647 	else {
3648 		printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3649 		       device->sim->unit_number,
3650 		       device->sim->bus_id,
3651 		       device->target->target_id,
3652 		       (uintmax_t)device->lun_id);
3653 	}
3654 }
3655 
3656 void
3657 xpt_print(struct cam_path *path, const char *fmt, ...)
3658 {
3659 	va_list ap;
3660 	xpt_print_path(path);
3661 	va_start(ap, fmt);
3662 	vprintf(fmt, ap);
3663 	va_end(ap);
3664 }
3665 
3666 int
3667 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3668 {
3669 	struct sbuf sb;
3670 
3671 	sbuf_new(&sb, str, str_len, 0);
3672 
3673 	if (path == NULL)
3674 		sbuf_printf(&sb, "(nopath): ");
3675 	else {
3676 		if (path->periph != NULL)
3677 			sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3678 				    path->periph->unit_number);
3679 		else
3680 			sbuf_printf(&sb, "(noperiph:");
3681 
3682 		if (path->bus != NULL)
3683 			sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3684 				    path->bus->sim->unit_number,
3685 				    path->bus->sim->bus_id);
3686 		else
3687 			sbuf_printf(&sb, "nobus:");
3688 
3689 		if (path->target != NULL)
3690 			sbuf_printf(&sb, "%d:", path->target->target_id);
3691 		else
3692 			sbuf_printf(&sb, "X:");
3693 
3694 		if (path->device != NULL)
3695 			sbuf_printf(&sb, "%jx): ",
3696 			    (uintmax_t)path->device->lun_id);
3697 		else
3698 			sbuf_printf(&sb, "X): ");
3699 	}
3700 	sbuf_finish(&sb);
3701 
3702 	return(sbuf_len(&sb));
3703 }
3704 
3705 path_id_t
3706 xpt_path_path_id(struct cam_path *path)
3707 {
3708 	return(path->bus->path_id);
3709 }
3710 
3711 target_id_t
3712 xpt_path_target_id(struct cam_path *path)
3713 {
3714 	if (path->target != NULL)
3715 		return (path->target->target_id);
3716 	else
3717 		return (CAM_TARGET_WILDCARD);
3718 }
3719 
3720 lun_id_t
3721 xpt_path_lun_id(struct cam_path *path)
3722 {
3723 	if (path->device != NULL)
3724 		return (path->device->lun_id);
3725 	else
3726 		return (CAM_LUN_WILDCARD);
3727 }
3728 
3729 struct cam_sim *
3730 xpt_path_sim(struct cam_path *path)
3731 {
3732 
3733 	return (path->bus->sim);
3734 }
3735 
3736 struct cam_periph*
3737 xpt_path_periph(struct cam_path *path)
3738 {
3739 
3740 	return (path->periph);
3741 }
3742 
3743 int
3744 xpt_path_legacy_ata_id(struct cam_path *path)
3745 {
3746 	struct cam_eb *bus;
3747 	int bus_id;
3748 
3749 	if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3750 	    strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3751 	    strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3752 	    strcmp(path->bus->sim->sim_name, "siisch") != 0)
3753 		return (-1);
3754 
3755 	if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3756 	    path->bus->sim->unit_number < 2) {
3757 		bus_id = path->bus->sim->unit_number;
3758 	} else {
3759 		bus_id = 2;
3760 		xpt_lock_buses();
3761 		TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3762 			if (bus == path->bus)
3763 				break;
3764 			if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3765 			     bus->sim->unit_number >= 2) ||
3766 			    strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3767 			    strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3768 			    strcmp(bus->sim->sim_name, "siisch") == 0)
3769 				bus_id++;
3770 		}
3771 		xpt_unlock_buses();
3772 	}
3773 	if (path->target != NULL) {
3774 		if (path->target->target_id < 2)
3775 			return (bus_id * 2 + path->target->target_id);
3776 		else
3777 			return (-1);
3778 	} else
3779 		return (bus_id * 2);
3780 }
3781 
3782 /*
3783  * Release a CAM control block for the caller.  Remit the cost of the structure
3784  * to the device referenced by the path.  If the this device had no 'credits'
3785  * and peripheral drivers have registered async callbacks for this notification
3786  * call them now.
3787  */
3788 void
3789 xpt_release_ccb(union ccb *free_ccb)
3790 {
3791 	struct	 cam_ed *device;
3792 	struct	 cam_periph *periph;
3793 
3794 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3795 	xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3796 	device = free_ccb->ccb_h.path->device;
3797 	periph = free_ccb->ccb_h.path->periph;
3798 
3799 	xpt_free_ccb(free_ccb);
3800 	periph->periph_allocated--;
3801 	cam_ccbq_release_opening(&device->ccbq);
3802 	xpt_run_allocq(periph, 0);
3803 }
3804 
3805 /* Functions accessed by SIM drivers */
3806 
3807 static struct xpt_xport xport_default = {
3808 	.alloc_device = xpt_alloc_device_default,
3809 	.action = xpt_action_default,
3810 	.async = xpt_dev_async_default,
3811 };
3812 
3813 /*
3814  * A sim structure, listing the SIM entry points and instance
3815  * identification info is passed to xpt_bus_register to hook the SIM
3816  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3817  * for this new bus and places it in the array of busses and assigns
3818  * it a path_id.  The path_id may be influenced by "hard wiring"
3819  * information specified by the user.  Once interrupt services are
3820  * available, the bus will be probed.
3821  */
3822 int32_t
3823 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3824 {
3825 	struct cam_eb *new_bus;
3826 	struct cam_eb *old_bus;
3827 	struct ccb_pathinq cpi;
3828 	struct cam_path *path;
3829 	cam_status status;
3830 
3831 	mtx_assert(sim->mtx, MA_OWNED);
3832 
3833 	sim->bus_id = bus;
3834 	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3835 					  M_CAMXPT, M_NOWAIT|M_ZERO);
3836 	if (new_bus == NULL) {
3837 		/* Couldn't satisfy request */
3838 		return (CAM_RESRC_UNAVAIL);
3839 	}
3840 
3841 	mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3842 	TAILQ_INIT(&new_bus->et_entries);
3843 	cam_sim_hold(sim);
3844 	new_bus->sim = sim;
3845 	timevalclear(&new_bus->last_reset);
3846 	new_bus->flags = 0;
3847 	new_bus->refcount = 1;	/* Held until a bus_deregister event */
3848 	new_bus->generation = 0;
3849 
3850 	xpt_lock_buses();
3851 	sim->path_id = new_bus->path_id =
3852 	    xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3853 	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3854 	while (old_bus != NULL
3855 	    && old_bus->path_id < new_bus->path_id)
3856 		old_bus = TAILQ_NEXT(old_bus, links);
3857 	if (old_bus != NULL)
3858 		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3859 	else
3860 		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3861 	xsoftc.bus_generation++;
3862 	xpt_unlock_buses();
3863 
3864 	/*
3865 	 * Set a default transport so that a PATH_INQ can be issued to
3866 	 * the SIM.  This will then allow for probing and attaching of
3867 	 * a more appropriate transport.
3868 	 */
3869 	new_bus->xport = &xport_default;
3870 
3871 	status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3872 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3873 	if (status != CAM_REQ_CMP) {
3874 		xpt_release_bus(new_bus);
3875 		free(path, M_CAMXPT);
3876 		return (CAM_RESRC_UNAVAIL);
3877 	}
3878 
3879 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3880 	cpi.ccb_h.func_code = XPT_PATH_INQ;
3881 	xpt_action((union ccb *)&cpi);
3882 
3883 	if (cpi.ccb_h.status == CAM_REQ_CMP) {
3884 		switch (cpi.transport) {
3885 		case XPORT_SPI:
3886 		case XPORT_SAS:
3887 		case XPORT_FC:
3888 		case XPORT_USB:
3889 		case XPORT_ISCSI:
3890 		case XPORT_SRP:
3891 		case XPORT_PPB:
3892 			new_bus->xport = scsi_get_xport();
3893 			break;
3894 		case XPORT_ATA:
3895 		case XPORT_SATA:
3896 			new_bus->xport = ata_get_xport();
3897 			break;
3898 		default:
3899 			new_bus->xport = &xport_default;
3900 			break;
3901 		}
3902 	}
3903 
3904 	/* Notify interested parties */
3905 	if (sim->path_id != CAM_XPT_PATH_ID) {
3906 
3907 		xpt_async(AC_PATH_REGISTERED, path, &cpi);
3908 		if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3909 			union	ccb *scan_ccb;
3910 
3911 			/* Initiate bus rescan. */
3912 			scan_ccb = xpt_alloc_ccb_nowait();
3913 			if (scan_ccb != NULL) {
3914 				scan_ccb->ccb_h.path = path;
3915 				scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3916 				scan_ccb->crcn.flags = 0;
3917 				xpt_rescan(scan_ccb);
3918 			} else {
3919 				xpt_print(path,
3920 					  "Can't allocate CCB to scan bus\n");
3921 				xpt_free_path(path);
3922 			}
3923 		} else
3924 			xpt_free_path(path);
3925 	} else
3926 		xpt_free_path(path);
3927 	return (CAM_SUCCESS);
3928 }
3929 
3930 int32_t
3931 xpt_bus_deregister(path_id_t pathid)
3932 {
3933 	struct cam_path bus_path;
3934 	cam_status status;
3935 
3936 	status = xpt_compile_path(&bus_path, NULL, pathid,
3937 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3938 	if (status != CAM_REQ_CMP)
3939 		return (status);
3940 
3941 	xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3942 	xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3943 
3944 	/* Release the reference count held while registered. */
3945 	xpt_release_bus(bus_path.bus);
3946 	xpt_release_path(&bus_path);
3947 
3948 	return (CAM_REQ_CMP);
3949 }
3950 
3951 static path_id_t
3952 xptnextfreepathid(void)
3953 {
3954 	struct cam_eb *bus;
3955 	path_id_t pathid;
3956 	const char *strval;
3957 
3958 	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3959 	pathid = 0;
3960 	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3961 retry:
3962 	/* Find an unoccupied pathid */
3963 	while (bus != NULL && bus->path_id <= pathid) {
3964 		if (bus->path_id == pathid)
3965 			pathid++;
3966 		bus = TAILQ_NEXT(bus, links);
3967 	}
3968 
3969 	/*
3970 	 * Ensure that this pathid is not reserved for
3971 	 * a bus that may be registered in the future.
3972 	 */
3973 	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
3974 		++pathid;
3975 		/* Start the search over */
3976 		goto retry;
3977 	}
3978 	return (pathid);
3979 }
3980 
3981 static path_id_t
3982 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
3983 {
3984 	path_id_t pathid;
3985 	int i, dunit, val;
3986 	char buf[32];
3987 	const char *dname;
3988 
3989 	pathid = CAM_XPT_PATH_ID;
3990 	snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
3991 	if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
3992 		return (pathid);
3993 	i = 0;
3994 	while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
3995 		if (strcmp(dname, "scbus")) {
3996 			/* Avoid a bit of foot shooting. */
3997 			continue;
3998 		}
3999 		if (dunit < 0)		/* unwired?! */
4000 			continue;
4001 		if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4002 			if (sim_bus == val) {
4003 				pathid = dunit;
4004 				break;
4005 			}
4006 		} else if (sim_bus == 0) {
4007 			/* Unspecified matches bus 0 */
4008 			pathid = dunit;
4009 			break;
4010 		} else {
4011 			printf("Ambiguous scbus configuration for %s%d "
4012 			       "bus %d, cannot wire down.  The kernel "
4013 			       "config entry for scbus%d should "
4014 			       "specify a controller bus.\n"
4015 			       "Scbus will be assigned dynamically.\n",
4016 			       sim_name, sim_unit, sim_bus, dunit);
4017 			break;
4018 		}
4019 	}
4020 
4021 	if (pathid == CAM_XPT_PATH_ID)
4022 		pathid = xptnextfreepathid();
4023 	return (pathid);
4024 }
4025 
4026 static const char *
4027 xpt_async_string(u_int32_t async_code)
4028 {
4029 
4030 	switch (async_code) {
4031 	case AC_BUS_RESET: return ("AC_BUS_RESET");
4032 	case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4033 	case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4034 	case AC_SENT_BDR: return ("AC_SENT_BDR");
4035 	case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4036 	case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4037 	case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4038 	case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4039 	case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4040 	case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4041 	case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4042 	case AC_CONTRACT: return ("AC_CONTRACT");
4043 	case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4044 	case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4045 	}
4046 	return ("AC_UNKNOWN");
4047 }
4048 
4049 static int
4050 xpt_async_size(u_int32_t async_code)
4051 {
4052 
4053 	switch (async_code) {
4054 	case AC_BUS_RESET: return (0);
4055 	case AC_UNSOL_RESEL: return (0);
4056 	case AC_SCSI_AEN: return (0);
4057 	case AC_SENT_BDR: return (0);
4058 	case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4059 	case AC_PATH_DEREGISTERED: return (0);
4060 	case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4061 	case AC_LOST_DEVICE: return (0);
4062 	case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4063 	case AC_INQ_CHANGED: return (0);
4064 	case AC_GETDEV_CHANGED: return (0);
4065 	case AC_CONTRACT: return (sizeof(struct ac_contract));
4066 	case AC_ADVINFO_CHANGED: return (-1);
4067 	case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4068 	}
4069 	return (0);
4070 }
4071 
4072 static int
4073 xpt_async_process_dev(struct cam_ed *device, void *arg)
4074 {
4075 	union ccb *ccb = arg;
4076 	struct cam_path *path = ccb->ccb_h.path;
4077 	void *async_arg = ccb->casync.async_arg_ptr;
4078 	u_int32_t async_code = ccb->casync.async_code;
4079 	int relock;
4080 
4081 	if (path->device != device
4082 	 && path->device->lun_id != CAM_LUN_WILDCARD
4083 	 && device->lun_id != CAM_LUN_WILDCARD)
4084 		return (1);
4085 
4086 	/*
4087 	 * The async callback could free the device.
4088 	 * If it is a broadcast async, it doesn't hold
4089 	 * device reference, so take our own reference.
4090 	 */
4091 	xpt_acquire_device(device);
4092 
4093 	/*
4094 	 * If async for specific device is to be delivered to
4095 	 * the wildcard client, take the specific device lock.
4096 	 * XXX: We may need a way for client to specify it.
4097 	 */
4098 	if ((device->lun_id == CAM_LUN_WILDCARD &&
4099 	     path->device->lun_id != CAM_LUN_WILDCARD) ||
4100 	    (device->target->target_id == CAM_TARGET_WILDCARD &&
4101 	     path->target->target_id != CAM_TARGET_WILDCARD) ||
4102 	    (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4103 	     path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4104 		mtx_unlock(&device->device_mtx);
4105 		xpt_path_lock(path);
4106 		relock = 1;
4107 	} else
4108 		relock = 0;
4109 
4110 	(*(device->target->bus->xport->async))(async_code,
4111 	    device->target->bus, device->target, device, async_arg);
4112 	xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4113 
4114 	if (relock) {
4115 		xpt_path_unlock(path);
4116 		mtx_lock(&device->device_mtx);
4117 	}
4118 	xpt_release_device(device);
4119 	return (1);
4120 }
4121 
4122 static int
4123 xpt_async_process_tgt(struct cam_et *target, void *arg)
4124 {
4125 	union ccb *ccb = arg;
4126 	struct cam_path *path = ccb->ccb_h.path;
4127 
4128 	if (path->target != target
4129 	 && path->target->target_id != CAM_TARGET_WILDCARD
4130 	 && target->target_id != CAM_TARGET_WILDCARD)
4131 		return (1);
4132 
4133 	if (ccb->casync.async_code == AC_SENT_BDR) {
4134 		/* Update our notion of when the last reset occurred */
4135 		microtime(&target->last_reset);
4136 	}
4137 
4138 	return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4139 }
4140 
4141 static void
4142 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4143 {
4144 	struct cam_eb *bus;
4145 	struct cam_path *path;
4146 	void *async_arg;
4147 	u_int32_t async_code;
4148 
4149 	path = ccb->ccb_h.path;
4150 	async_code = ccb->casync.async_code;
4151 	async_arg = ccb->casync.async_arg_ptr;
4152 	CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4153 	    ("xpt_async(%s)\n", xpt_async_string(async_code)));
4154 	bus = path->bus;
4155 
4156 	if (async_code == AC_BUS_RESET) {
4157 		/* Update our notion of when the last reset occurred */
4158 		microtime(&bus->last_reset);
4159 	}
4160 
4161 	xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4162 
4163 	/*
4164 	 * If this wasn't a fully wildcarded async, tell all
4165 	 * clients that want all async events.
4166 	 */
4167 	if (bus != xpt_periph->path->bus) {
4168 		xpt_path_lock(xpt_periph->path);
4169 		xpt_async_process_dev(xpt_periph->path->device, ccb);
4170 		xpt_path_unlock(xpt_periph->path);
4171 	}
4172 
4173 	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4174 		xpt_release_devq(path, 1, TRUE);
4175 	else
4176 		xpt_release_simq(path->bus->sim, TRUE);
4177 	if (ccb->casync.async_arg_size > 0)
4178 		free(async_arg, M_CAMXPT);
4179 	xpt_free_path(path);
4180 	xpt_free_ccb(ccb);
4181 }
4182 
4183 static void
4184 xpt_async_bcast(struct async_list *async_head,
4185 		u_int32_t async_code,
4186 		struct cam_path *path, void *async_arg)
4187 {
4188 	struct async_node *cur_entry;
4189 	int lock;
4190 
4191 	cur_entry = SLIST_FIRST(async_head);
4192 	while (cur_entry != NULL) {
4193 		struct async_node *next_entry;
4194 		/*
4195 		 * Grab the next list entry before we call the current
4196 		 * entry's callback.  This is because the callback function
4197 		 * can delete its async callback entry.
4198 		 */
4199 		next_entry = SLIST_NEXT(cur_entry, links);
4200 		if ((cur_entry->event_enable & async_code) != 0) {
4201 			lock = cur_entry->event_lock;
4202 			if (lock)
4203 				CAM_SIM_LOCK(path->device->sim);
4204 			cur_entry->callback(cur_entry->callback_arg,
4205 					    async_code, path,
4206 					    async_arg);
4207 			if (lock)
4208 				CAM_SIM_UNLOCK(path->device->sim);
4209 		}
4210 		cur_entry = next_entry;
4211 	}
4212 }
4213 
4214 void
4215 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4216 {
4217 	union ccb *ccb;
4218 	int size;
4219 
4220 	ccb = xpt_alloc_ccb_nowait();
4221 	if (ccb == NULL) {
4222 		xpt_print(path, "Can't allocate CCB to send %s\n",
4223 		    xpt_async_string(async_code));
4224 		return;
4225 	}
4226 
4227 	if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4228 		xpt_print(path, "Can't allocate path to send %s\n",
4229 		    xpt_async_string(async_code));
4230 		xpt_free_ccb(ccb);
4231 		return;
4232 	}
4233 	ccb->ccb_h.path->periph = NULL;
4234 	ccb->ccb_h.func_code = XPT_ASYNC;
4235 	ccb->ccb_h.cbfcnp = xpt_async_process;
4236 	ccb->ccb_h.flags |= CAM_UNLOCKED;
4237 	ccb->casync.async_code = async_code;
4238 	ccb->casync.async_arg_size = 0;
4239 	size = xpt_async_size(async_code);
4240 	if (size > 0 && async_arg != NULL) {
4241 		ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4242 		if (ccb->casync.async_arg_ptr == NULL) {
4243 			xpt_print(path, "Can't allocate argument to send %s\n",
4244 			    xpt_async_string(async_code));
4245 			xpt_free_path(ccb->ccb_h.path);
4246 			xpt_free_ccb(ccb);
4247 			return;
4248 		}
4249 		memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4250 		ccb->casync.async_arg_size = size;
4251 	} else if (size < 0)
4252 		ccb->casync.async_arg_size = size;
4253 	if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4254 		xpt_freeze_devq(path, 1);
4255 	else
4256 		xpt_freeze_simq(path->bus->sim, 1);
4257 	xpt_done(ccb);
4258 }
4259 
4260 static void
4261 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4262 		      struct cam_et *target, struct cam_ed *device,
4263 		      void *async_arg)
4264 {
4265 
4266 	/*
4267 	 * We only need to handle events for real devices.
4268 	 */
4269 	if (target->target_id == CAM_TARGET_WILDCARD
4270 	 || device->lun_id == CAM_LUN_WILDCARD)
4271 		return;
4272 
4273 	printf("%s called\n", __func__);
4274 }
4275 
4276 static uint32_t
4277 xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4278 {
4279 	struct cam_devq	*devq;
4280 	uint32_t freeze;
4281 
4282 	devq = dev->sim->devq;
4283 	mtx_assert(&devq->send_mtx, MA_OWNED);
4284 	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4285 	    ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4286 	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4287 	freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4288 	/* Remove frozen device from sendq. */
4289 	if (device_is_queued(dev))
4290 		camq_remove(&devq->send_queue, dev->devq_entry.index);
4291 	return (freeze);
4292 }
4293 
4294 u_int32_t
4295 xpt_freeze_devq(struct cam_path *path, u_int count)
4296 {
4297 	struct cam_ed	*dev = path->device;
4298 	struct cam_devq	*devq;
4299 	uint32_t	 freeze;
4300 
4301 	devq = dev->sim->devq;
4302 	mtx_lock(&devq->send_mtx);
4303 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4304 	freeze = xpt_freeze_devq_device(dev, count);
4305 	mtx_unlock(&devq->send_mtx);
4306 	return (freeze);
4307 }
4308 
4309 u_int32_t
4310 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4311 {
4312 	struct cam_devq	*devq;
4313 	uint32_t	 freeze;
4314 
4315 	devq = sim->devq;
4316 	mtx_lock(&devq->send_mtx);
4317 	freeze = (devq->send_queue.qfrozen_cnt += count);
4318 	mtx_unlock(&devq->send_mtx);
4319 	return (freeze);
4320 }
4321 
4322 static void
4323 xpt_release_devq_timeout(void *arg)
4324 {
4325 	struct cam_ed *dev;
4326 	struct cam_devq *devq;
4327 
4328 	dev = (struct cam_ed *)arg;
4329 	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4330 	devq = dev->sim->devq;
4331 	mtx_assert(&devq->send_mtx, MA_OWNED);
4332 	if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4333 		xpt_run_devq(devq);
4334 }
4335 
4336 void
4337 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4338 {
4339 	struct cam_ed *dev;
4340 	struct cam_devq *devq;
4341 
4342 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4343 	    count, run_queue));
4344 	dev = path->device;
4345 	devq = dev->sim->devq;
4346 	mtx_lock(&devq->send_mtx);
4347 	if (xpt_release_devq_device(dev, count, run_queue))
4348 		xpt_run_devq(dev->sim->devq);
4349 	mtx_unlock(&devq->send_mtx);
4350 }
4351 
4352 static int
4353 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4354 {
4355 
4356 	mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4357 	CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4358 	    ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4359 	    dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4360 	if (count > dev->ccbq.queue.qfrozen_cnt) {
4361 #ifdef INVARIANTS
4362 		printf("xpt_release_devq(): requested %u > present %u\n",
4363 		    count, dev->ccbq.queue.qfrozen_cnt);
4364 #endif
4365 		count = dev->ccbq.queue.qfrozen_cnt;
4366 	}
4367 	dev->ccbq.queue.qfrozen_cnt -= count;
4368 	if (dev->ccbq.queue.qfrozen_cnt == 0) {
4369 		/*
4370 		 * No longer need to wait for a successful
4371 		 * command completion.
4372 		 */
4373 		dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4374 		/*
4375 		 * Remove any timeouts that might be scheduled
4376 		 * to release this queue.
4377 		 */
4378 		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4379 			callout_stop(&dev->callout);
4380 			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4381 		}
4382 		/*
4383 		 * Now that we are unfrozen schedule the
4384 		 * device so any pending transactions are
4385 		 * run.
4386 		 */
4387 		xpt_schedule_devq(dev->sim->devq, dev);
4388 	} else
4389 		run_queue = 0;
4390 	return (run_queue);
4391 }
4392 
4393 void
4394 xpt_release_simq(struct cam_sim *sim, int run_queue)
4395 {
4396 	struct cam_devq	*devq;
4397 
4398 	devq = sim->devq;
4399 	mtx_lock(&devq->send_mtx);
4400 	if (devq->send_queue.qfrozen_cnt <= 0) {
4401 #ifdef INVARIANTS
4402 		printf("xpt_release_simq: requested 1 > present %u\n",
4403 		    devq->send_queue.qfrozen_cnt);
4404 #endif
4405 	} else
4406 		devq->send_queue.qfrozen_cnt--;
4407 	if (devq->send_queue.qfrozen_cnt == 0) {
4408 		/*
4409 		 * If there is a timeout scheduled to release this
4410 		 * sim queue, remove it.  The queue frozen count is
4411 		 * already at 0.
4412 		 */
4413 		if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4414 			callout_stop(&sim->callout);
4415 			sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4416 		}
4417 		if (run_queue) {
4418 			/*
4419 			 * Now that we are unfrozen run the send queue.
4420 			 */
4421 			xpt_run_devq(sim->devq);
4422 		}
4423 	}
4424 	mtx_unlock(&devq->send_mtx);
4425 }
4426 
4427 /*
4428  * XXX Appears to be unused.
4429  */
4430 static void
4431 xpt_release_simq_timeout(void *arg)
4432 {
4433 	struct cam_sim *sim;
4434 
4435 	sim = (struct cam_sim *)arg;
4436 	xpt_release_simq(sim, /* run_queue */ TRUE);
4437 }
4438 
4439 void
4440 xpt_done(union ccb *done_ccb)
4441 {
4442 	struct cam_doneq *queue;
4443 	int	run, hash;
4444 
4445 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4446 	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4447 		return;
4448 
4449 	hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4450 	    done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4451 	queue = &cam_doneqs[hash];
4452 	mtx_lock(&queue->cam_doneq_mtx);
4453 	run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4454 	STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4455 	done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4456 	mtx_unlock(&queue->cam_doneq_mtx);
4457 	if (run)
4458 		wakeup(&queue->cam_doneq);
4459 }
4460 
4461 void
4462 xpt_done_direct(union ccb *done_ccb)
4463 {
4464 
4465 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4466 	if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4467 		return;
4468 
4469 	xpt_done_process(&done_ccb->ccb_h);
4470 }
4471 
4472 union ccb *
4473 xpt_alloc_ccb()
4474 {
4475 	union ccb *new_ccb;
4476 
4477 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4478 	return (new_ccb);
4479 }
4480 
4481 union ccb *
4482 xpt_alloc_ccb_nowait()
4483 {
4484 	union ccb *new_ccb;
4485 
4486 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4487 	return (new_ccb);
4488 }
4489 
4490 void
4491 xpt_free_ccb(union ccb *free_ccb)
4492 {
4493 	free(free_ccb, M_CAMCCB);
4494 }
4495 
4496 
4497 
4498 /* Private XPT functions */
4499 
4500 /*
4501  * Get a CAM control block for the caller. Charge the structure to the device
4502  * referenced by the path.  If we don't have sufficient resources to allocate
4503  * more ccbs, we return NULL.
4504  */
4505 static union ccb *
4506 xpt_get_ccb_nowait(struct cam_periph *periph)
4507 {
4508 	union ccb *new_ccb;
4509 
4510 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_NOWAIT);
4511 	if (new_ccb == NULL)
4512 		return (NULL);
4513 	periph->periph_allocated++;
4514 	cam_ccbq_take_opening(&periph->path->device->ccbq);
4515 	return (new_ccb);
4516 }
4517 
4518 static union ccb *
4519 xpt_get_ccb(struct cam_periph *periph)
4520 {
4521 	union ccb *new_ccb;
4522 
4523 	cam_periph_unlock(periph);
4524 	new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_WAITOK);
4525 	cam_periph_lock(periph);
4526 	periph->periph_allocated++;
4527 	cam_ccbq_take_opening(&periph->path->device->ccbq);
4528 	return (new_ccb);
4529 }
4530 
4531 union ccb *
4532 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4533 {
4534 	struct ccb_hdr *ccb_h;
4535 
4536 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4537 	cam_periph_assert(periph, MA_OWNED);
4538 	while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4539 	    ccb_h->pinfo.priority != priority) {
4540 		if (priority < periph->immediate_priority) {
4541 			periph->immediate_priority = priority;
4542 			xpt_run_allocq(periph, 0);
4543 		} else
4544 			cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4545 			    "cgticb", 0);
4546 	}
4547 	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4548 	return ((union ccb *)ccb_h);
4549 }
4550 
4551 static void
4552 xpt_acquire_bus(struct cam_eb *bus)
4553 {
4554 
4555 	xpt_lock_buses();
4556 	bus->refcount++;
4557 	xpt_unlock_buses();
4558 }
4559 
4560 static void
4561 xpt_release_bus(struct cam_eb *bus)
4562 {
4563 
4564 	xpt_lock_buses();
4565 	KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4566 	if (--bus->refcount > 0) {
4567 		xpt_unlock_buses();
4568 		return;
4569 	}
4570 	TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4571 	xsoftc.bus_generation++;
4572 	xpt_unlock_buses();
4573 	KASSERT(TAILQ_EMPTY(&bus->et_entries),
4574 	    ("destroying bus, but target list is not empty"));
4575 	cam_sim_release(bus->sim);
4576 	mtx_destroy(&bus->eb_mtx);
4577 	free(bus, M_CAMXPT);
4578 }
4579 
4580 static struct cam_et *
4581 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4582 {
4583 	struct cam_et *cur_target, *target;
4584 
4585 	mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4586 	mtx_assert(&bus->eb_mtx, MA_OWNED);
4587 	target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4588 					 M_NOWAIT|M_ZERO);
4589 	if (target == NULL)
4590 		return (NULL);
4591 
4592 	TAILQ_INIT(&target->ed_entries);
4593 	target->bus = bus;
4594 	target->target_id = target_id;
4595 	target->refcount = 1;
4596 	target->generation = 0;
4597 	target->luns = NULL;
4598 	mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4599 	timevalclear(&target->last_reset);
4600 	/*
4601 	 * Hold a reference to our parent bus so it
4602 	 * will not go away before we do.
4603 	 */
4604 	bus->refcount++;
4605 
4606 	/* Insertion sort into our bus's target list */
4607 	cur_target = TAILQ_FIRST(&bus->et_entries);
4608 	while (cur_target != NULL && cur_target->target_id < target_id)
4609 		cur_target = TAILQ_NEXT(cur_target, links);
4610 	if (cur_target != NULL) {
4611 		TAILQ_INSERT_BEFORE(cur_target, target, links);
4612 	} else {
4613 		TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4614 	}
4615 	bus->generation++;
4616 	return (target);
4617 }
4618 
4619 static void
4620 xpt_acquire_target(struct cam_et *target)
4621 {
4622 	struct cam_eb *bus = target->bus;
4623 
4624 	mtx_lock(&bus->eb_mtx);
4625 	target->refcount++;
4626 	mtx_unlock(&bus->eb_mtx);
4627 }
4628 
4629 static void
4630 xpt_release_target(struct cam_et *target)
4631 {
4632 	struct cam_eb *bus = target->bus;
4633 
4634 	mtx_lock(&bus->eb_mtx);
4635 	if (--target->refcount > 0) {
4636 		mtx_unlock(&bus->eb_mtx);
4637 		return;
4638 	}
4639 	TAILQ_REMOVE(&bus->et_entries, target, links);
4640 	bus->generation++;
4641 	mtx_unlock(&bus->eb_mtx);
4642 	KASSERT(TAILQ_EMPTY(&target->ed_entries),
4643 	    ("destroying target, but device list is not empty"));
4644 	xpt_release_bus(bus);
4645 	mtx_destroy(&target->luns_mtx);
4646 	if (target->luns)
4647 		free(target->luns, M_CAMXPT);
4648 	free(target, M_CAMXPT);
4649 }
4650 
4651 static struct cam_ed *
4652 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4653 			 lun_id_t lun_id)
4654 {
4655 	struct cam_ed *device;
4656 
4657 	device = xpt_alloc_device(bus, target, lun_id);
4658 	if (device == NULL)
4659 		return (NULL);
4660 
4661 	device->mintags = 1;
4662 	device->maxtags = 1;
4663 	return (device);
4664 }
4665 
4666 static void
4667 xpt_destroy_device(void *context, int pending)
4668 {
4669 	struct cam_ed	*device = context;
4670 
4671 	mtx_lock(&device->device_mtx);
4672 	mtx_destroy(&device->device_mtx);
4673 	free(device, M_CAMDEV);
4674 }
4675 
4676 struct cam_ed *
4677 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4678 {
4679 	struct cam_ed	*cur_device, *device;
4680 	struct cam_devq	*devq;
4681 	cam_status status;
4682 
4683 	mtx_assert(&bus->eb_mtx, MA_OWNED);
4684 	/* Make space for us in the device queue on our bus */
4685 	devq = bus->sim->devq;
4686 	mtx_lock(&devq->send_mtx);
4687 	status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4688 	mtx_unlock(&devq->send_mtx);
4689 	if (status != CAM_REQ_CMP)
4690 		return (NULL);
4691 
4692 	device = (struct cam_ed *)malloc(sizeof(*device),
4693 					 M_CAMDEV, M_NOWAIT|M_ZERO);
4694 	if (device == NULL)
4695 		return (NULL);
4696 
4697 	cam_init_pinfo(&device->devq_entry);
4698 	device->target = target;
4699 	device->lun_id = lun_id;
4700 	device->sim = bus->sim;
4701 	if (cam_ccbq_init(&device->ccbq,
4702 			  bus->sim->max_dev_openings) != 0) {
4703 		free(device, M_CAMDEV);
4704 		return (NULL);
4705 	}
4706 	SLIST_INIT(&device->asyncs);
4707 	SLIST_INIT(&device->periphs);
4708 	device->generation = 0;
4709 	device->flags = CAM_DEV_UNCONFIGURED;
4710 	device->tag_delay_count = 0;
4711 	device->tag_saved_openings = 0;
4712 	device->refcount = 1;
4713 	mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4714 	callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4715 	TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4716 	/*
4717 	 * Hold a reference to our parent bus so it
4718 	 * will not go away before we do.
4719 	 */
4720 	target->refcount++;
4721 
4722 	cur_device = TAILQ_FIRST(&target->ed_entries);
4723 	while (cur_device != NULL && cur_device->lun_id < lun_id)
4724 		cur_device = TAILQ_NEXT(cur_device, links);
4725 	if (cur_device != NULL)
4726 		TAILQ_INSERT_BEFORE(cur_device, device, links);
4727 	else
4728 		TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4729 	target->generation++;
4730 	return (device);
4731 }
4732 
4733 void
4734 xpt_acquire_device(struct cam_ed *device)
4735 {
4736 	struct cam_eb *bus = device->target->bus;
4737 
4738 	mtx_lock(&bus->eb_mtx);
4739 	device->refcount++;
4740 	mtx_unlock(&bus->eb_mtx);
4741 }
4742 
4743 void
4744 xpt_release_device(struct cam_ed *device)
4745 {
4746 	struct cam_eb *bus = device->target->bus;
4747 	struct cam_devq *devq;
4748 
4749 	mtx_lock(&bus->eb_mtx);
4750 	if (--device->refcount > 0) {
4751 		mtx_unlock(&bus->eb_mtx);
4752 		return;
4753 	}
4754 
4755 	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4756 	device->target->generation++;
4757 	mtx_unlock(&bus->eb_mtx);
4758 
4759 	/* Release our slot in the devq */
4760 	devq = bus->sim->devq;
4761 	mtx_lock(&devq->send_mtx);
4762 	cam_devq_resize(devq, devq->send_queue.array_size - 1);
4763 	mtx_unlock(&devq->send_mtx);
4764 
4765 	KASSERT(SLIST_EMPTY(&device->periphs),
4766 	    ("destroying device, but periphs list is not empty"));
4767 	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4768 	    ("destroying device while still queued for ccbs"));
4769 
4770 	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4771 		callout_stop(&device->callout);
4772 
4773 	xpt_release_target(device->target);
4774 
4775 	cam_ccbq_fini(&device->ccbq);
4776 	/*
4777 	 * Free allocated memory.  free(9) does nothing if the
4778 	 * supplied pointer is NULL, so it is safe to call without
4779 	 * checking.
4780 	 */
4781 	free(device->supported_vpds, M_CAMXPT);
4782 	free(device->device_id, M_CAMXPT);
4783 	free(device->physpath, M_CAMXPT);
4784 	free(device->rcap_buf, M_CAMXPT);
4785 	free(device->serial_num, M_CAMXPT);
4786 	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4787 }
4788 
4789 u_int32_t
4790 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4791 {
4792 	int	result;
4793 	struct	cam_ed *dev;
4794 
4795 	dev = path->device;
4796 	mtx_lock(&dev->sim->devq->send_mtx);
4797 	result = cam_ccbq_resize(&dev->ccbq, newopenings);
4798 	mtx_unlock(&dev->sim->devq->send_mtx);
4799 	if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4800 	 || (dev->inq_flags & SID_CmdQue) != 0)
4801 		dev->tag_saved_openings = newopenings;
4802 	return (result);
4803 }
4804 
4805 static struct cam_eb *
4806 xpt_find_bus(path_id_t path_id)
4807 {
4808 	struct cam_eb *bus;
4809 
4810 	xpt_lock_buses();
4811 	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4812 	     bus != NULL;
4813 	     bus = TAILQ_NEXT(bus, links)) {
4814 		if (bus->path_id == path_id) {
4815 			bus->refcount++;
4816 			break;
4817 		}
4818 	}
4819 	xpt_unlock_buses();
4820 	return (bus);
4821 }
4822 
4823 static struct cam_et *
4824 xpt_find_target(struct cam_eb *bus, target_id_t	target_id)
4825 {
4826 	struct cam_et *target;
4827 
4828 	mtx_assert(&bus->eb_mtx, MA_OWNED);
4829 	for (target = TAILQ_FIRST(&bus->et_entries);
4830 	     target != NULL;
4831 	     target = TAILQ_NEXT(target, links)) {
4832 		if (target->target_id == target_id) {
4833 			target->refcount++;
4834 			break;
4835 		}
4836 	}
4837 	return (target);
4838 }
4839 
4840 static struct cam_ed *
4841 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4842 {
4843 	struct cam_ed *device;
4844 
4845 	mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4846 	for (device = TAILQ_FIRST(&target->ed_entries);
4847 	     device != NULL;
4848 	     device = TAILQ_NEXT(device, links)) {
4849 		if (device->lun_id == lun_id) {
4850 			device->refcount++;
4851 			break;
4852 		}
4853 	}
4854 	return (device);
4855 }
4856 
4857 void
4858 xpt_start_tags(struct cam_path *path)
4859 {
4860 	struct ccb_relsim crs;
4861 	struct cam_ed *device;
4862 	struct cam_sim *sim;
4863 	int    newopenings;
4864 
4865 	device = path->device;
4866 	sim = path->bus->sim;
4867 	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4868 	xpt_freeze_devq(path, /*count*/1);
4869 	device->inq_flags |= SID_CmdQue;
4870 	if (device->tag_saved_openings != 0)
4871 		newopenings = device->tag_saved_openings;
4872 	else
4873 		newopenings = min(device->maxtags,
4874 				  sim->max_tagged_dev_openings);
4875 	xpt_dev_ccbq_resize(path, newopenings);
4876 	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4877 	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4878 	crs.ccb_h.func_code = XPT_REL_SIMQ;
4879 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4880 	crs.openings
4881 	    = crs.release_timeout
4882 	    = crs.qfrozen_cnt
4883 	    = 0;
4884 	xpt_action((union ccb *)&crs);
4885 }
4886 
4887 void
4888 xpt_stop_tags(struct cam_path *path)
4889 {
4890 	struct ccb_relsim crs;
4891 	struct cam_ed *device;
4892 	struct cam_sim *sim;
4893 
4894 	device = path->device;
4895 	sim = path->bus->sim;
4896 	device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4897 	device->tag_delay_count = 0;
4898 	xpt_freeze_devq(path, /*count*/1);
4899 	device->inq_flags &= ~SID_CmdQue;
4900 	xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4901 	xpt_async(AC_GETDEV_CHANGED, path, NULL);
4902 	xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4903 	crs.ccb_h.func_code = XPT_REL_SIMQ;
4904 	crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4905 	crs.openings
4906 	    = crs.release_timeout
4907 	    = crs.qfrozen_cnt
4908 	    = 0;
4909 	xpt_action((union ccb *)&crs);
4910 }
4911 
4912 static void
4913 xpt_boot_delay(void *arg)
4914 {
4915 
4916 	xpt_release_boot();
4917 }
4918 
4919 static void
4920 xpt_config(void *arg)
4921 {
4922 	/*
4923 	 * Now that interrupts are enabled, go find our devices
4924 	 */
4925 	if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4926 		printf("xpt_config: failed to create taskqueue thread.\n");
4927 
4928 	/* Setup debugging path */
4929 	if (cam_dflags != CAM_DEBUG_NONE) {
4930 		if (xpt_create_path(&cam_dpath, NULL,
4931 				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4932 				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4933 			printf("xpt_config: xpt_create_path() failed for debug"
4934 			       " target %d:%d:%d, debugging disabled\n",
4935 			       CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4936 			cam_dflags = CAM_DEBUG_NONE;
4937 		}
4938 	} else
4939 		cam_dpath = NULL;
4940 
4941 	periphdriver_init(1);
4942 	xpt_hold_boot();
4943 	callout_init(&xsoftc.boot_callout, 1);
4944 	callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000,
4945 	    xpt_boot_delay, NULL);
4946 	/* Fire up rescan thread. */
4947 	if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4948 	    "cam", "scanner")) {
4949 		printf("xpt_config: failed to create rescan thread.\n");
4950 	}
4951 }
4952 
4953 void
4954 xpt_hold_boot(void)
4955 {
4956 	xpt_lock_buses();
4957 	xsoftc.buses_to_config++;
4958 	xpt_unlock_buses();
4959 }
4960 
4961 void
4962 xpt_release_boot(void)
4963 {
4964 	xpt_lock_buses();
4965 	xsoftc.buses_to_config--;
4966 	if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4967 		struct	xpt_task *task;
4968 
4969 		xsoftc.buses_config_done = 1;
4970 		xpt_unlock_buses();
4971 		/* Call manually because we don't have any busses */
4972 		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
4973 		if (task != NULL) {
4974 			TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
4975 			taskqueue_enqueue(taskqueue_thread, &task->task);
4976 		}
4977 	} else
4978 		xpt_unlock_buses();
4979 }
4980 
4981 /*
4982  * If the given device only has one peripheral attached to it, and if that
4983  * peripheral is the passthrough driver, announce it.  This insures that the
4984  * user sees some sort of announcement for every peripheral in their system.
4985  */
4986 static int
4987 xptpassannouncefunc(struct cam_ed *device, void *arg)
4988 {
4989 	struct cam_periph *periph;
4990 	int i;
4991 
4992 	for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
4993 	     periph = SLIST_NEXT(periph, periph_links), i++);
4994 
4995 	periph = SLIST_FIRST(&device->periphs);
4996 	if ((i == 1)
4997 	 && (strncmp(periph->periph_name, "pass", 4) == 0))
4998 		xpt_announce_periph(periph, NULL);
4999 
5000 	return(1);
5001 }
5002 
5003 static void
5004 xpt_finishconfig_task(void *context, int pending)
5005 {
5006 
5007 	periphdriver_init(2);
5008 	/*
5009 	 * Check for devices with no "standard" peripheral driver
5010 	 * attached.  For any devices like that, announce the
5011 	 * passthrough driver so the user will see something.
5012 	 */
5013 	if (!bootverbose)
5014 		xpt_for_all_devices(xptpassannouncefunc, NULL);
5015 
5016 	/* Release our hook so that the boot can continue. */
5017 	config_intrhook_disestablish(xsoftc.xpt_config_hook);
5018 	free(xsoftc.xpt_config_hook, M_CAMXPT);
5019 	xsoftc.xpt_config_hook = NULL;
5020 
5021 	free(context, M_CAMXPT);
5022 }
5023 
5024 cam_status
5025 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5026 		   struct cam_path *path)
5027 {
5028 	struct ccb_setasync csa;
5029 	cam_status status;
5030 	int xptpath = 0;
5031 
5032 	if (path == NULL) {
5033 		status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5034 					 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5035 		if (status != CAM_REQ_CMP)
5036 			return (status);
5037 		xpt_path_lock(path);
5038 		xptpath = 1;
5039 	}
5040 
5041 	xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5042 	csa.ccb_h.func_code = XPT_SASYNC_CB;
5043 	csa.event_enable = event;
5044 	csa.callback = cbfunc;
5045 	csa.callback_arg = cbarg;
5046 	xpt_action((union ccb *)&csa);
5047 	status = csa.ccb_h.status;
5048 
5049 	if (xptpath) {
5050 		xpt_path_unlock(path);
5051 		xpt_free_path(path);
5052 	}
5053 
5054 	if ((status == CAM_REQ_CMP) &&
5055 	    (csa.event_enable & AC_FOUND_DEVICE)) {
5056 		/*
5057 		 * Get this peripheral up to date with all
5058 		 * the currently existing devices.
5059 		 */
5060 		xpt_for_all_devices(xptsetasyncfunc, &csa);
5061 	}
5062 	if ((status == CAM_REQ_CMP) &&
5063 	    (csa.event_enable & AC_PATH_REGISTERED)) {
5064 		/*
5065 		 * Get this peripheral up to date with all
5066 		 * the currently existing busses.
5067 		 */
5068 		xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5069 	}
5070 
5071 	return (status);
5072 }
5073 
5074 static void
5075 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5076 {
5077 	CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5078 
5079 	switch (work_ccb->ccb_h.func_code) {
5080 	/* Common cases first */
5081 	case XPT_PATH_INQ:		/* Path routing inquiry */
5082 	{
5083 		struct ccb_pathinq *cpi;
5084 
5085 		cpi = &work_ccb->cpi;
5086 		cpi->version_num = 1; /* XXX??? */
5087 		cpi->hba_inquiry = 0;
5088 		cpi->target_sprt = 0;
5089 		cpi->hba_misc = 0;
5090 		cpi->hba_eng_cnt = 0;
5091 		cpi->max_target = 0;
5092 		cpi->max_lun = 0;
5093 		cpi->initiator_id = 0;
5094 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5095 		strncpy(cpi->hba_vid, "", HBA_IDLEN);
5096 		strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5097 		cpi->unit_number = sim->unit_number;
5098 		cpi->bus_id = sim->bus_id;
5099 		cpi->base_transfer_speed = 0;
5100 		cpi->protocol = PROTO_UNSPECIFIED;
5101 		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5102 		cpi->transport = XPORT_UNSPECIFIED;
5103 		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5104 		cpi->ccb_h.status = CAM_REQ_CMP;
5105 		xpt_done(work_ccb);
5106 		break;
5107 	}
5108 	default:
5109 		work_ccb->ccb_h.status = CAM_REQ_INVALID;
5110 		xpt_done(work_ccb);
5111 		break;
5112 	}
5113 }
5114 
5115 /*
5116  * The xpt as a "controller" has no interrupt sources, so polling
5117  * is a no-op.
5118  */
5119 static void
5120 xptpoll(struct cam_sim *sim)
5121 {
5122 }
5123 
5124 void
5125 xpt_lock_buses(void)
5126 {
5127 	mtx_lock(&xsoftc.xpt_topo_lock);
5128 }
5129 
5130 void
5131 xpt_unlock_buses(void)
5132 {
5133 	mtx_unlock(&xsoftc.xpt_topo_lock);
5134 }
5135 
5136 struct mtx *
5137 xpt_path_mtx(struct cam_path *path)
5138 {
5139 
5140 	return (&path->device->device_mtx);
5141 }
5142 
5143 static void
5144 xpt_done_process(struct ccb_hdr *ccb_h)
5145 {
5146 	struct cam_sim *sim;
5147 	struct cam_devq *devq;
5148 	struct mtx *mtx = NULL;
5149 
5150 	if (ccb_h->flags & CAM_HIGH_POWER) {
5151 		struct highpowerlist	*hphead;
5152 		struct cam_ed		*device;
5153 
5154 		mtx_lock(&xsoftc.xpt_highpower_lock);
5155 		hphead = &xsoftc.highpowerq;
5156 
5157 		device = STAILQ_FIRST(hphead);
5158 
5159 		/*
5160 		 * Increment the count since this command is done.
5161 		 */
5162 		xsoftc.num_highpower++;
5163 
5164 		/*
5165 		 * Any high powered commands queued up?
5166 		 */
5167 		if (device != NULL) {
5168 
5169 			STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5170 			mtx_unlock(&xsoftc.xpt_highpower_lock);
5171 
5172 			mtx_lock(&device->sim->devq->send_mtx);
5173 			xpt_release_devq_device(device,
5174 					 /*count*/1, /*runqueue*/TRUE);
5175 			mtx_unlock(&device->sim->devq->send_mtx);
5176 		} else
5177 			mtx_unlock(&xsoftc.xpt_highpower_lock);
5178 	}
5179 
5180 	sim = ccb_h->path->bus->sim;
5181 
5182 	if (ccb_h->status & CAM_RELEASE_SIMQ) {
5183 		xpt_release_simq(sim, /*run_queue*/FALSE);
5184 		ccb_h->status &= ~CAM_RELEASE_SIMQ;
5185 	}
5186 
5187 	if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5188 	 && (ccb_h->status & CAM_DEV_QFRZN)) {
5189 		xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5190 		ccb_h->status &= ~CAM_DEV_QFRZN;
5191 	}
5192 
5193 	devq = sim->devq;
5194 	if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5195 		struct cam_ed *dev = ccb_h->path->device;
5196 
5197 		mtx_lock(&devq->send_mtx);
5198 		devq->send_active--;
5199 		devq->send_openings++;
5200 		cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5201 
5202 		if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5203 		  && (dev->ccbq.dev_active == 0))) {
5204 			dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5205 			xpt_release_devq_device(dev, /*count*/1,
5206 					 /*run_queue*/FALSE);
5207 		}
5208 
5209 		if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5210 		  && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5211 			dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5212 			xpt_release_devq_device(dev, /*count*/1,
5213 					 /*run_queue*/FALSE);
5214 		}
5215 
5216 		if (!device_is_queued(dev))
5217 			(void)xpt_schedule_devq(devq, dev);
5218 		xpt_run_devq(devq);
5219 		mtx_unlock(&devq->send_mtx);
5220 
5221 		if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5222 			mtx = xpt_path_mtx(ccb_h->path);
5223 			mtx_lock(mtx);
5224 
5225 			if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5226 			 && (--dev->tag_delay_count == 0))
5227 				xpt_start_tags(ccb_h->path);
5228 		}
5229 	}
5230 
5231 	if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5232 		if (mtx == NULL) {
5233 			mtx = xpt_path_mtx(ccb_h->path);
5234 			mtx_lock(mtx);
5235 		}
5236 	} else {
5237 		if (mtx != NULL) {
5238 			mtx_unlock(mtx);
5239 			mtx = NULL;
5240 		}
5241 	}
5242 
5243 	/* Call the peripheral driver's callback */
5244 	ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5245 	(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5246 	if (mtx != NULL)
5247 		mtx_unlock(mtx);
5248 }
5249 
5250 void
5251 xpt_done_td(void *arg)
5252 {
5253 	struct cam_doneq *queue = arg;
5254 	struct ccb_hdr *ccb_h;
5255 	STAILQ_HEAD(, ccb_hdr)	doneq;
5256 
5257 	STAILQ_INIT(&doneq);
5258 	mtx_lock(&queue->cam_doneq_mtx);
5259 	while (1) {
5260 		while (STAILQ_EMPTY(&queue->cam_doneq)) {
5261 			queue->cam_doneq_sleep = 1;
5262 			msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5263 			    PRIBIO, "-", 0);
5264 			queue->cam_doneq_sleep = 0;
5265 		}
5266 		STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5267 		mtx_unlock(&queue->cam_doneq_mtx);
5268 
5269 		THREAD_NO_SLEEPING();
5270 		while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5271 			STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5272 			xpt_done_process(ccb_h);
5273 		}
5274 		THREAD_SLEEPING_OK();
5275 
5276 		mtx_lock(&queue->cam_doneq_mtx);
5277 	}
5278 }
5279 
5280 static void
5281 camisr_runqueue(void)
5282 {
5283 	struct	ccb_hdr *ccb_h;
5284 	struct cam_doneq *queue;
5285 	int i;
5286 
5287 	/* Process global queues. */
5288 	for (i = 0; i < cam_num_doneqs; i++) {
5289 		queue = &cam_doneqs[i];
5290 		mtx_lock(&queue->cam_doneq_mtx);
5291 		while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5292 			STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5293 			mtx_unlock(&queue->cam_doneq_mtx);
5294 			xpt_done_process(ccb_h);
5295 			mtx_lock(&queue->cam_doneq_mtx);
5296 		}
5297 		mtx_unlock(&queue->cam_doneq_mtx);
5298 	}
5299 }
5300