xref: /freebsd/sys/cam/ata/ata_xpt.c (revision a98ff317388a00b992f1bf8404dee596f9383f5e)
1 /*-
2  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/endian.h>
33 #include <sys/systm.h>
34 #include <sys/types.h>
35 #include <sys/malloc.h>
36 #include <sys/kernel.h>
37 #include <sys/time.h>
38 #include <sys/conf.h>
39 #include <sys/fcntl.h>
40 #include <sys/interrupt.h>
41 #include <sys/sbuf.h>
42 
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 
47 #include <cam/cam.h>
48 #include <cam/cam_ccb.h>
49 #include <cam/cam_queue.h>
50 #include <cam/cam_periph.h>
51 #include <cam/cam_sim.h>
52 #include <cam/cam_xpt.h>
53 #include <cam/cam_xpt_sim.h>
54 #include <cam/cam_xpt_periph.h>
55 #include <cam/cam_xpt_internal.h>
56 #include <cam/cam_debug.h>
57 
58 #include <cam/scsi/scsi_all.h>
59 #include <cam/scsi/scsi_message.h>
60 #include <cam/ata/ata_all.h>
61 #include <machine/stdarg.h>	/* for xpt_print below */
62 #include "opt_cam.h"
63 
64 struct ata_quirk_entry {
65 	struct scsi_inquiry_pattern inq_pat;
66 	u_int8_t quirks;
67 #define	CAM_QUIRK_MAXTAGS	0x01
68 	u_int mintags;
69 	u_int maxtags;
70 };
71 
72 static periph_init_t probe_periph_init;
73 
74 static struct periph_driver probe_driver =
75 {
76 	probe_periph_init, "aprobe",
77 	TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0,
78 	CAM_PERIPH_DRV_EARLY
79 };
80 
81 PERIPHDRIVER_DECLARE(aprobe, probe_driver);
82 
83 typedef enum {
84 	PROBE_RESET,
85 	PROBE_IDENTIFY,
86 	PROBE_SPINUP,
87 	PROBE_SETMODE,
88 	PROBE_SETPM,
89 	PROBE_SETAPST,
90 	PROBE_SETDMAAA,
91 	PROBE_SETAN,
92 	PROBE_SET_MULTI,
93 	PROBE_INQUIRY,
94 	PROBE_FULL_INQUIRY,
95 	PROBE_PM_PID,
96 	PROBE_PM_PRV,
97 	PROBE_IDENTIFY_SES,
98 	PROBE_IDENTIFY_SAFTE,
99 	PROBE_DONE,
100 	PROBE_INVALID
101 } probe_action;
102 
103 static char *probe_action_text[] = {
104 	"PROBE_RESET",
105 	"PROBE_IDENTIFY",
106 	"PROBE_SPINUP",
107 	"PROBE_SETMODE",
108 	"PROBE_SETPM",
109 	"PROBE_SETAPST",
110 	"PROBE_SETDMAAA",
111 	"PROBE_SETAN",
112 	"PROBE_SET_MULTI",
113 	"PROBE_INQUIRY",
114 	"PROBE_FULL_INQUIRY",
115 	"PROBE_PM_PID",
116 	"PROBE_PM_PRV",
117 	"PROBE_IDENTIFY_SES",
118 	"PROBE_IDENTIFY_SAFTE",
119 	"PROBE_DONE",
120 	"PROBE_INVALID"
121 };
122 
123 #define PROBE_SET_ACTION(softc, newaction)	\
124 do {									\
125 	char **text;							\
126 	text = probe_action_text;					\
127 	CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE,		\
128 	    ("Probe %s to %s\n", text[(softc)->action],			\
129 	    text[(newaction)]));					\
130 	(softc)->action = (newaction);					\
131 } while(0)
132 
133 typedef enum {
134 	PROBE_NO_ANNOUNCE	= 0x04
135 } probe_flags;
136 
137 typedef struct {
138 	TAILQ_HEAD(, ccb_hdr) request_ccbs;
139 	struct ata_params	ident_data;
140 	probe_action	action;
141 	probe_flags	flags;
142 	uint32_t	pm_pid;
143 	uint32_t	pm_prv;
144 	int		restart;
145 	int		spinup;
146 	int		faults;
147 	u_int		caps;
148 	struct cam_periph *periph;
149 } probe_softc;
150 
151 static struct ata_quirk_entry ata_quirk_table[] =
152 {
153 	{
154 		/* Default tagged queuing parameters for all devices */
155 		{
156 		  T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
157 		  /*vendor*/"*", /*product*/"*", /*revision*/"*"
158 		},
159 		/*quirks*/0, /*mintags*/0, /*maxtags*/0
160 	},
161 };
162 
163 static const int ata_quirk_table_size =
164 	sizeof(ata_quirk_table) / sizeof(*ata_quirk_table);
165 
166 static cam_status	proberegister(struct cam_periph *periph,
167 				      void *arg);
168 static void	 probeschedule(struct cam_periph *probe_periph);
169 static void	 probestart(struct cam_periph *periph, union ccb *start_ccb);
170 static void	 proberequestdefaultnegotiation(struct cam_periph *periph);
171 static void	 probedone(struct cam_periph *periph, union ccb *done_ccb);
172 static void	 probecleanup(struct cam_periph *periph);
173 static void	 ata_find_quirk(struct cam_ed *device);
174 static void	 ata_scan_bus(struct cam_periph *periph, union ccb *ccb);
175 static void	 ata_scan_lun(struct cam_periph *periph,
176 			       struct cam_path *path, cam_flags flags,
177 			       union ccb *ccb);
178 static void	 xptscandone(struct cam_periph *periph, union ccb *done_ccb);
179 static struct cam_ed *
180 		 ata_alloc_device(struct cam_eb *bus, struct cam_et *target,
181 				   lun_id_t lun_id);
182 static void	 ata_device_transport(struct cam_path *path);
183 static void	 ata_get_transfer_settings(struct ccb_trans_settings *cts);
184 static void	 ata_set_transfer_settings(struct ccb_trans_settings *cts,
185 					    struct cam_ed *device,
186 					    int async_update);
187 static void	 ata_dev_async(u_int32_t async_code,
188 				struct cam_eb *bus,
189 				struct cam_et *target,
190 				struct cam_ed *device,
191 				void *async_arg);
192 static void	 ata_action(union ccb *start_ccb);
193 static void	 ata_announce_periph(struct cam_periph *periph);
194 
195 static int ata_dma = 1;
196 static int atapi_dma = 1;
197 
198 TUNABLE_INT("hw.ata.ata_dma", &ata_dma);
199 TUNABLE_INT("hw.ata.atapi_dma", &atapi_dma);
200 
201 static struct xpt_xport ata_xport = {
202 	.alloc_device = ata_alloc_device,
203 	.action = ata_action,
204 	.async = ata_dev_async,
205 	.announce = ata_announce_periph,
206 };
207 
208 struct xpt_xport *
209 ata_get_xport(void)
210 {
211 	return (&ata_xport);
212 }
213 
214 static void
215 probe_periph_init()
216 {
217 }
218 
219 static cam_status
220 proberegister(struct cam_periph *periph, void *arg)
221 {
222 	union ccb *request_ccb;	/* CCB representing the probe request */
223 	cam_status status;
224 	probe_softc *softc;
225 
226 	request_ccb = (union ccb *)arg;
227 	if (request_ccb == NULL) {
228 		printf("proberegister: no probe CCB, "
229 		       "can't register device\n");
230 		return(CAM_REQ_CMP_ERR);
231 	}
232 
233 	softc = (probe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT);
234 
235 	if (softc == NULL) {
236 		printf("proberegister: Unable to probe new device. "
237 		       "Unable to allocate softc\n");
238 		return(CAM_REQ_CMP_ERR);
239 	}
240 	TAILQ_INIT(&softc->request_ccbs);
241 	TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
242 			  periph_links.tqe);
243 	softc->flags = 0;
244 	periph->softc = softc;
245 	softc->periph = periph;
246 	softc->action = PROBE_INVALID;
247 	status = cam_periph_acquire(periph);
248 	if (status != CAM_REQ_CMP) {
249 		return (status);
250 	}
251 	CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n"));
252 	probeschedule(periph);
253 	return(CAM_REQ_CMP);
254 }
255 
256 static void
257 probeschedule(struct cam_periph *periph)
258 {
259 	union ccb *ccb;
260 	probe_softc *softc;
261 
262 	softc = (probe_softc *)periph->softc;
263 	ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
264 
265 	if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) ||
266 	    periph->path->device->protocol == PROTO_SATAPM ||
267 	    periph->path->device->protocol == PROTO_SEMB)
268 		PROBE_SET_ACTION(softc, PROBE_RESET);
269 	else
270 		PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
271 
272 	if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE)
273 		softc->flags |= PROBE_NO_ANNOUNCE;
274 	else
275 		softc->flags &= ~PROBE_NO_ANNOUNCE;
276 
277 	xpt_schedule(periph, CAM_PRIORITY_XPT);
278 }
279 
280 static void
281 probestart(struct cam_periph *periph, union ccb *start_ccb)
282 {
283 	struct ccb_trans_settings cts;
284 	struct ccb_ataio *ataio;
285 	struct ccb_scsiio *csio;
286 	probe_softc *softc;
287 	struct cam_path *path;
288 	struct ata_params *ident_buf;
289 
290 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probestart\n"));
291 
292 	softc = (probe_softc *)periph->softc;
293 	path = start_ccb->ccb_h.path;
294 	ataio = &start_ccb->ataio;
295 	csio = &start_ccb->csio;
296 	ident_buf = &periph->path->device->ident_data;
297 
298 	if (softc->restart) {
299 		softc->restart = 0;
300 		if ((path->device->flags & CAM_DEV_UNCONFIGURED) ||
301 		    path->device->protocol == PROTO_SATAPM ||
302 		    path->device->protocol == PROTO_SEMB)
303 			softc->action = PROBE_RESET;
304 		else
305 			softc->action = PROBE_IDENTIFY;
306 	}
307 	switch (softc->action) {
308 	case PROBE_RESET:
309 		cam_fill_ataio(ataio,
310 		      0,
311 		      probedone,
312 		      /*flags*/CAM_DIR_NONE,
313 		      0,
314 		      /*data_ptr*/NULL,
315 		      /*dxfer_len*/0,
316 		      15 * 1000);
317 		ata_reset_cmd(ataio);
318 		break;
319 	case PROBE_IDENTIFY:
320 		cam_fill_ataio(ataio,
321 		      1,
322 		      probedone,
323 		      /*flags*/CAM_DIR_IN,
324 		      0,
325 		      /*data_ptr*/(u_int8_t *)&softc->ident_data,
326 		      /*dxfer_len*/sizeof(softc->ident_data),
327 		      30 * 1000);
328 		if (periph->path->device->protocol == PROTO_ATA)
329 			ata_28bit_cmd(ataio, ATA_ATA_IDENTIFY, 0, 0, 0);
330 		else
331 			ata_28bit_cmd(ataio, ATA_ATAPI_IDENTIFY, 0, 0, 0);
332 		break;
333 	case PROBE_SPINUP:
334 		if (bootverbose)
335 			xpt_print(path, "Spinning up device\n");
336 		cam_fill_ataio(ataio,
337 		      1,
338 		      probedone,
339 		      /*flags*/CAM_DIR_NONE | CAM_HIGH_POWER,
340 		      0,
341 		      /*data_ptr*/NULL,
342 		      /*dxfer_len*/0,
343 		      30 * 1000);
344 		ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_PUIS_SPINUP, 0, 0);
345 		break;
346 	case PROBE_SETMODE:
347 	{
348 		int mode, wantmode;
349 
350 		mode = 0;
351 		/* Fetch user modes from SIM. */
352 		bzero(&cts, sizeof(cts));
353 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
354 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
355 		cts.type = CTS_TYPE_USER_SETTINGS;
356 		xpt_action((union ccb *)&cts);
357 		if (path->device->transport == XPORT_ATA) {
358 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE)
359 				mode = cts.xport_specific.ata.mode;
360 		} else {
361 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_MODE)
362 				mode = cts.xport_specific.sata.mode;
363 		}
364 		if (periph->path->device->protocol == PROTO_ATA) {
365 			if (ata_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX))
366 				mode = ATA_PIO_MAX;
367 		} else {
368 			if (atapi_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX))
369 				mode = ATA_PIO_MAX;
370 		}
371 negotiate:
372 		/* Honor device capabilities. */
373 		wantmode = mode = ata_max_mode(ident_buf, mode);
374 		/* Report modes to SIM. */
375 		bzero(&cts, sizeof(cts));
376 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
377 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
378 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
379 		if (path->device->transport == XPORT_ATA) {
380 			cts.xport_specific.ata.mode = mode;
381 			cts.xport_specific.ata.valid = CTS_ATA_VALID_MODE;
382 		} else {
383 			cts.xport_specific.sata.mode = mode;
384 			cts.xport_specific.sata.valid = CTS_SATA_VALID_MODE;
385 		}
386 		xpt_action((union ccb *)&cts);
387 		/* Fetch current modes from SIM. */
388 		bzero(&cts, sizeof(cts));
389 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
390 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
391 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
392 		xpt_action((union ccb *)&cts);
393 		if (path->device->transport == XPORT_ATA) {
394 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE)
395 				mode = cts.xport_specific.ata.mode;
396 		} else {
397 			if (cts.xport_specific.ata.valid & CTS_SATA_VALID_MODE)
398 				mode = cts.xport_specific.sata.mode;
399 		}
400 		/* If SIM disagree - renegotiate. */
401 		if (mode != wantmode)
402 			goto negotiate;
403 		/* Remember what transport thinks about DMA. */
404 		if (mode < ATA_DMA)
405 			path->device->inq_flags &= ~SID_DMA;
406 		else
407 			path->device->inq_flags |= SID_DMA;
408 		xpt_async(AC_GETDEV_CHANGED, path, NULL);
409 		cam_fill_ataio(ataio,
410 		      1,
411 		      probedone,
412 		      /*flags*/CAM_DIR_NONE,
413 		      0,
414 		      /*data_ptr*/NULL,
415 		      /*dxfer_len*/0,
416 		      30 * 1000);
417 		ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_SETXFER, 0, mode);
418 		break;
419 	}
420 	case PROBE_SETPM:
421 		cam_fill_ataio(ataio,
422 		    1,
423 		    probedone,
424 		    CAM_DIR_NONE,
425 		    0,
426 		    NULL,
427 		    0,
428 		    30*1000);
429 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
430 		    (softc->caps & CTS_SATA_CAPS_H_PMREQ) ? 0x10 : 0x90,
431 		    0, 0x03);
432 		break;
433 	case PROBE_SETAPST:
434 		cam_fill_ataio(ataio,
435 		    1,
436 		    probedone,
437 		    CAM_DIR_NONE,
438 		    0,
439 		    NULL,
440 		    0,
441 		    30*1000);
442 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
443 		    (softc->caps & CTS_SATA_CAPS_H_APST) ? 0x10 : 0x90,
444 		    0, 0x07);
445 		break;
446 	case PROBE_SETDMAAA:
447 		cam_fill_ataio(ataio,
448 		    1,
449 		    probedone,
450 		    CAM_DIR_NONE,
451 		    0,
452 		    NULL,
453 		    0,
454 		    30*1000);
455 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
456 		    (softc->caps & CTS_SATA_CAPS_H_DMAAA) ? 0x10 : 0x90,
457 		    0, 0x02);
458 		break;
459 	case PROBE_SETAN:
460 		/* Remember what transport thinks about AEN. */
461 		if (softc->caps & CTS_SATA_CAPS_H_AN)
462 			path->device->inq_flags |= SID_AEN;
463 		else
464 			path->device->inq_flags &= ~SID_AEN;
465 		xpt_async(AC_GETDEV_CHANGED, path, NULL);
466 		cam_fill_ataio(ataio,
467 		    1,
468 		    probedone,
469 		    CAM_DIR_NONE,
470 		    0,
471 		    NULL,
472 		    0,
473 		    30*1000);
474 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
475 		    (softc->caps & CTS_SATA_CAPS_H_AN) ? 0x10 : 0x90,
476 		    0, 0x05);
477 		break;
478 	case PROBE_SET_MULTI:
479 	{
480 		u_int sectors, bytecount;
481 
482 		bytecount = 8192;	/* SATA maximum */
483 		/* Fetch user bytecount from SIM. */
484 		bzero(&cts, sizeof(cts));
485 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
486 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
487 		cts.type = CTS_TYPE_USER_SETTINGS;
488 		xpt_action((union ccb *)&cts);
489 		if (path->device->transport == XPORT_ATA) {
490 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
491 				bytecount = cts.xport_specific.ata.bytecount;
492 		} else {
493 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
494 				bytecount = cts.xport_specific.sata.bytecount;
495 		}
496 		/* Honor device capabilities. */
497 		sectors = max(1, min(ident_buf->sectors_intr & 0xff,
498 		    bytecount / ata_logical_sector_size(ident_buf)));
499 		/* Report bytecount to SIM. */
500 		bzero(&cts, sizeof(cts));
501 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
502 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
503 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
504 		if (path->device->transport == XPORT_ATA) {
505 			cts.xport_specific.ata.bytecount = sectors *
506 			    ata_logical_sector_size(ident_buf);
507 			cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
508 		} else {
509 			cts.xport_specific.sata.bytecount = sectors *
510 			    ata_logical_sector_size(ident_buf);
511 			cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
512 		}
513 		xpt_action((union ccb *)&cts);
514 		/* Fetch current bytecount from SIM. */
515 		bzero(&cts, sizeof(cts));
516 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
517 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
518 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
519 		xpt_action((union ccb *)&cts);
520 		if (path->device->transport == XPORT_ATA) {
521 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
522 				bytecount = cts.xport_specific.ata.bytecount;
523 		} else {
524 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
525 				bytecount = cts.xport_specific.sata.bytecount;
526 		}
527 		sectors = bytecount / ata_logical_sector_size(ident_buf);
528 
529 		cam_fill_ataio(ataio,
530 		    1,
531 		    probedone,
532 		    CAM_DIR_NONE,
533 		    0,
534 		    NULL,
535 		    0,
536 		    30*1000);
537 		ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, sectors);
538 		break;
539 	}
540 	case PROBE_INQUIRY:
541 	{
542 		u_int bytecount;
543 
544 		bytecount = 8192;	/* SATA maximum */
545 		/* Fetch user bytecount from SIM. */
546 		bzero(&cts, sizeof(cts));
547 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
548 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
549 		cts.type = CTS_TYPE_USER_SETTINGS;
550 		xpt_action((union ccb *)&cts);
551 		if (path->device->transport == XPORT_ATA) {
552 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
553 				bytecount = cts.xport_specific.ata.bytecount;
554 		} else {
555 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
556 				bytecount = cts.xport_specific.sata.bytecount;
557 		}
558 		/* Honor device capabilities. */
559 		bytecount &= ~1;
560 		bytecount = max(2, min(65534, bytecount));
561 		if (ident_buf->satacapabilities != 0x0000 &&
562 		    ident_buf->satacapabilities != 0xffff) {
563 			bytecount = min(8192, bytecount);
564 		}
565 		/* Report bytecount to SIM. */
566 		bzero(&cts, sizeof(cts));
567 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
568 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
569 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
570 		if (path->device->transport == XPORT_ATA) {
571 			cts.xport_specific.ata.bytecount = bytecount;
572 			cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
573 		} else {
574 			cts.xport_specific.sata.bytecount = bytecount;
575 			cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
576 		}
577 		xpt_action((union ccb *)&cts);
578 		/* FALLTHROUGH */
579 	}
580 	case PROBE_FULL_INQUIRY:
581 	{
582 		u_int inquiry_len;
583 		struct scsi_inquiry_data *inq_buf =
584 		    &periph->path->device->inq_data;
585 
586 		if (softc->action == PROBE_INQUIRY)
587 			inquiry_len = SHORT_INQUIRY_LENGTH;
588 		else
589 			inquiry_len = SID_ADDITIONAL_LENGTH(inq_buf);
590 		/*
591 		 * Some parallel SCSI devices fail to send an
592 		 * ignore wide residue message when dealing with
593 		 * odd length inquiry requests.  Round up to be
594 		 * safe.
595 		 */
596 		inquiry_len = roundup2(inquiry_len, 2);
597 		scsi_inquiry(csio,
598 			     /*retries*/1,
599 			     probedone,
600 			     MSG_SIMPLE_Q_TAG,
601 			     (u_int8_t *)inq_buf,
602 			     inquiry_len,
603 			     /*evpd*/FALSE,
604 			     /*page_code*/0,
605 			     SSD_MIN_SIZE,
606 			     /*timeout*/60 * 1000);
607 		break;
608 	}
609 	case PROBE_PM_PID:
610 		cam_fill_ataio(ataio,
611 		      1,
612 		      probedone,
613 		      /*flags*/CAM_DIR_NONE,
614 		      0,
615 		      /*data_ptr*/NULL,
616 		      /*dxfer_len*/0,
617 		      10 * 1000);
618 		ata_pm_read_cmd(ataio, 0, 15);
619 		break;
620 	case PROBE_PM_PRV:
621 		cam_fill_ataio(ataio,
622 		      1,
623 		      probedone,
624 		      /*flags*/CAM_DIR_NONE,
625 		      0,
626 		      /*data_ptr*/NULL,
627 		      /*dxfer_len*/0,
628 		      10 * 1000);
629 		ata_pm_read_cmd(ataio, 1, 15);
630 		break;
631 	case PROBE_IDENTIFY_SES:
632 		cam_fill_ataio(ataio,
633 		      1,
634 		      probedone,
635 		      /*flags*/CAM_DIR_IN,
636 		      0,
637 		      /*data_ptr*/(u_int8_t *)&softc->ident_data,
638 		      /*dxfer_len*/sizeof(softc->ident_data),
639 		      30 * 1000);
640 		ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x02,
641 		    sizeof(softc->ident_data) / 4);
642 		break;
643 	case PROBE_IDENTIFY_SAFTE:
644 		cam_fill_ataio(ataio,
645 		      1,
646 		      probedone,
647 		      /*flags*/CAM_DIR_IN,
648 		      0,
649 		      /*data_ptr*/(u_int8_t *)&softc->ident_data,
650 		      /*dxfer_len*/sizeof(softc->ident_data),
651 		      30 * 1000);
652 		ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x00,
653 		    sizeof(softc->ident_data) / 4);
654 		break;
655 	default:
656 		panic("probestart: invalid action state 0x%x\n", softc->action);
657 	}
658 	start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
659 	xpt_action(start_ccb);
660 }
661 
662 static void
663 proberequestdefaultnegotiation(struct cam_periph *periph)
664 {
665 	struct ccb_trans_settings cts;
666 
667 	xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE);
668 	cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
669 	cts.type = CTS_TYPE_USER_SETTINGS;
670 	xpt_action((union ccb *)&cts);
671 	if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
672 		return;
673 	cts.xport_specific.valid = 0;
674 	cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
675 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
676 	xpt_action((union ccb *)&cts);
677 }
678 
679 static void
680 probedone(struct cam_periph *periph, union ccb *done_ccb)
681 {
682 	struct ccb_trans_settings cts;
683 	struct ata_params *ident_buf;
684 	struct scsi_inquiry_data *inq_buf;
685 	probe_softc *softc;
686 	struct cam_path *path;
687 	cam_status status;
688 	u_int32_t  priority;
689 	u_int caps;
690 	int changed = 1, found = 1;
691 	static const uint8_t fake_device_id_hdr[8] =
692 	    {0, SVPD_DEVICE_ID, 0, 12,
693 	     SVPD_ID_CODESET_BINARY, SVPD_ID_TYPE_NAA, 0, 8};
694 
695 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probedone\n"));
696 
697 	softc = (probe_softc *)periph->softc;
698 	path = done_ccb->ccb_h.path;
699 	priority = done_ccb->ccb_h.pinfo.priority;
700 	ident_buf = &path->device->ident_data;
701 	inq_buf = &path->device->inq_data;
702 
703 	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
704 		if (cam_periph_error(done_ccb,
705 		    0, softc->restart ? (SF_NO_RECOVERY | SF_NO_RETRY) : 0,
706 		    NULL) == ERESTART) {
707 out:
708 			/* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
709 			cam_release_devq(path, 0, 0, 0, FALSE);
710 			return;
711 		}
712 		if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
713 			/* Don't wedge the queue */
714 			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
715 		}
716 		status = done_ccb->ccb_h.status & CAM_STATUS_MASK;
717 		if (softc->restart) {
718 			softc->faults++;
719 			if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) ==
720 			    CAM_CMD_TIMEOUT)
721 				softc->faults += 4;
722 			if (softc->faults < 10)
723 				goto done;
724 			else
725 				softc->restart = 0;
726 
727 		/* Old PIO2 devices may not support mode setting. */
728 		} else if (softc->action == PROBE_SETMODE &&
729 		    status == CAM_ATA_STATUS_ERROR &&
730 		    ata_max_pmode(ident_buf) <= ATA_PIO2 &&
731 		    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0) {
732 			goto noerror;
733 
734 		/*
735 		 * Some old WD SATA disks report supported and enabled
736 		 * device-initiated interface power management, but return
737 		 * ABORT on attempt to disable it.
738 		 */
739 		} else if (softc->action == PROBE_SETPM &&
740 		    status == CAM_ATA_STATUS_ERROR) {
741 			goto noerror;
742 
743 		/*
744 		 * Some HP SATA disks report supported DMA Auto-Activation,
745 		 * but return ABORT on attempt to enable it.
746 		 */
747 		} else if (softc->action == PROBE_SETDMAAA &&
748 		    status == CAM_ATA_STATUS_ERROR) {
749 			goto noerror;
750 
751 		/*
752 		 * Some Samsung SSDs report supported Asynchronous Notification,
753 		 * but return ABORT on attempt to enable it.
754 		 */
755 		} else if (softc->action == PROBE_SETAN &&
756 		    status == CAM_ATA_STATUS_ERROR) {
757 			goto noerror;
758 
759 		/*
760 		 * SES and SAF-TE SEPs have different IDENTIFY commands,
761 		 * but SATA specification doesn't tell how to identify them.
762 		 * Until better way found, just try another if first fail.
763 		 */
764 		} else if (softc->action == PROBE_IDENTIFY_SES &&
765 		    status == CAM_ATA_STATUS_ERROR) {
766 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SAFTE);
767 			xpt_release_ccb(done_ccb);
768 			xpt_schedule(periph, priority);
769 			goto out;
770 		}
771 
772 		/*
773 		 * If we get to this point, we got an error status back
774 		 * from the inquiry and the error status doesn't require
775 		 * automatically retrying the command.  Therefore, the
776 		 * inquiry failed.  If we had inquiry information before
777 		 * for this device, but this latest inquiry command failed,
778 		 * the device has probably gone away.  If this device isn't
779 		 * already marked unconfigured, notify the peripheral
780 		 * drivers that this device is no more.
781 		 */
782 device_fail:	if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
783 			xpt_async(AC_LOST_DEVICE, path, NULL);
784 		PROBE_SET_ACTION(softc, PROBE_INVALID);
785 		found = 0;
786 		goto done;
787 	}
788 noerror:
789 	if (softc->restart)
790 		goto done;
791 	switch (softc->action) {
792 	case PROBE_RESET:
793 	{
794 		int sign = (done_ccb->ataio.res.lba_high << 8) +
795 		    done_ccb->ataio.res.lba_mid;
796 		CAM_DEBUG(path, CAM_DEBUG_PROBE,
797 		    ("SIGNATURE: %04x\n", sign));
798 		if (sign == 0x0000 &&
799 		    done_ccb->ccb_h.target_id != 15) {
800 			path->device->protocol = PROTO_ATA;
801 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
802 		} else if (sign == 0x9669 &&
803 		    done_ccb->ccb_h.target_id == 15) {
804 			/* Report SIM that PM is present. */
805 			bzero(&cts, sizeof(cts));
806 			xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
807 			cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
808 			cts.type = CTS_TYPE_CURRENT_SETTINGS;
809 			cts.xport_specific.sata.pm_present = 1;
810 			cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
811 			xpt_action((union ccb *)&cts);
812 			path->device->protocol = PROTO_SATAPM;
813 			PROBE_SET_ACTION(softc, PROBE_PM_PID);
814 		} else if (sign == 0xc33c &&
815 		    done_ccb->ccb_h.target_id != 15) {
816 			path->device->protocol = PROTO_SEMB;
817 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SES);
818 		} else if (sign == 0xeb14 &&
819 		    done_ccb->ccb_h.target_id != 15) {
820 			path->device->protocol = PROTO_SCSI;
821 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
822 		} else {
823 			if (done_ccb->ccb_h.target_id != 15) {
824 				xpt_print(path,
825 				    "Unexpected signature 0x%04x\n", sign);
826 			}
827 			goto device_fail;
828 		}
829 		xpt_release_ccb(done_ccb);
830 		xpt_schedule(periph, priority);
831 		goto out;
832 	}
833 	case PROBE_IDENTIFY:
834 	{
835 		struct ccb_pathinq cpi;
836 		int16_t *ptr;
837 
838 		ident_buf = &softc->ident_data;
839 		for (ptr = (int16_t *)ident_buf;
840 		     ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) {
841 			*ptr = le16toh(*ptr);
842 		}
843 		if (strncmp(ident_buf->model, "FX", 2) &&
844 		    strncmp(ident_buf->model, "NEC", 3) &&
845 		    strncmp(ident_buf->model, "Pioneer", 7) &&
846 		    strncmp(ident_buf->model, "SHARP", 5)) {
847 			ata_bswap(ident_buf->model, sizeof(ident_buf->model));
848 			ata_bswap(ident_buf->revision, sizeof(ident_buf->revision));
849 			ata_bswap(ident_buf->serial, sizeof(ident_buf->serial));
850 		}
851 		ata_btrim(ident_buf->model, sizeof(ident_buf->model));
852 		ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model));
853 		ata_btrim(ident_buf->revision, sizeof(ident_buf->revision));
854 		ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision));
855 		ata_btrim(ident_buf->serial, sizeof(ident_buf->serial));
856 		ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial));
857 		/* Device may need spin-up before IDENTIFY become valid. */
858 		if ((ident_buf->specconf == 0x37c8 ||
859 		     ident_buf->specconf == 0x738c) &&
860 		    ((ident_buf->config & ATA_RESP_INCOMPLETE) ||
861 		     softc->spinup == 0)) {
862 			PROBE_SET_ACTION(softc, PROBE_SPINUP);
863 			xpt_release_ccb(done_ccb);
864 			xpt_schedule(periph, priority);
865 			goto out;
866 		}
867 		ident_buf = &path->device->ident_data;
868 		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
869 			/* Check that it is the same device. */
870 			if (bcmp(softc->ident_data.model, ident_buf->model,
871 			     sizeof(ident_buf->model)) ||
872 			    bcmp(softc->ident_data.revision, ident_buf->revision,
873 			     sizeof(ident_buf->revision)) ||
874 			    bcmp(softc->ident_data.serial, ident_buf->serial,
875 			     sizeof(ident_buf->serial))) {
876 				/* Device changed. */
877 				xpt_async(AC_LOST_DEVICE, path, NULL);
878 			} else {
879 				bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
880 				changed = 0;
881 			}
882 		}
883 		if (changed) {
884 			bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
885 			/* Clean up from previous instance of this device */
886 			if (path->device->serial_num != NULL) {
887 				free(path->device->serial_num, M_CAMXPT);
888 				path->device->serial_num = NULL;
889 				path->device->serial_num_len = 0;
890 			}
891 			if (path->device->device_id != NULL) {
892 				free(path->device->device_id, M_CAMXPT);
893 				path->device->device_id = NULL;
894 				path->device->device_id_len = 0;
895 			}
896 			path->device->serial_num =
897 				(u_int8_t *)malloc((sizeof(ident_buf->serial) + 1),
898 					   M_CAMXPT, M_NOWAIT);
899 			if (path->device->serial_num != NULL) {
900 				bcopy(ident_buf->serial,
901 				      path->device->serial_num,
902 				      sizeof(ident_buf->serial));
903 				path->device->serial_num[sizeof(ident_buf->serial)]
904 				    = '\0';
905 				path->device->serial_num_len =
906 				    strlen(path->device->serial_num);
907 			}
908 			if (ident_buf->enabled.extension &
909 			    ATA_SUPPORT_64BITWWN) {
910 				path->device->device_id =
911 				    malloc(16, M_CAMXPT, M_NOWAIT);
912 				if (path->device->device_id != NULL) {
913 					path->device->device_id_len = 16;
914 					bcopy(&fake_device_id_hdr,
915 					    path->device->device_id, 8);
916 					bcopy(ident_buf->wwn,
917 					    path->device->device_id + 8, 8);
918 					ata_bswap(path->device->device_id + 8, 8);
919 				}
920 			}
921 
922 			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
923 			xpt_async(AC_GETDEV_CHANGED, path, NULL);
924 		}
925 		if (ident_buf->satacapabilities & ATA_SUPPORT_NCQ) {
926 			path->device->mintags = 2;
927 			path->device->maxtags =
928 			    ATA_QUEUE_LEN(ident_buf->queue) + 1;
929 		}
930 		ata_find_quirk(path->device);
931 		if (path->device->mintags != 0 &&
932 		    path->bus->sim->max_tagged_dev_openings != 0) {
933 			/* Check if the SIM does not want queued commands. */
934 			bzero(&cpi, sizeof(cpi));
935 			xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
936 			cpi.ccb_h.func_code = XPT_PATH_INQ;
937 			xpt_action((union ccb *)&cpi);
938 			if (cpi.ccb_h.status == CAM_REQ_CMP &&
939 			    (cpi.hba_inquiry & PI_TAG_ABLE)) {
940 				/* Report SIM which tags are allowed. */
941 				bzero(&cts, sizeof(cts));
942 				xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
943 				cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
944 				cts.type = CTS_TYPE_CURRENT_SETTINGS;
945 				cts.xport_specific.sata.tags = path->device->maxtags;
946 				cts.xport_specific.sata.valid = CTS_SATA_VALID_TAGS;
947 				xpt_action((union ccb *)&cts);
948 			}
949 		}
950 		ata_device_transport(path);
951 		if (changed)
952 			proberequestdefaultnegotiation(periph);
953 		PROBE_SET_ACTION(softc, PROBE_SETMODE);
954 		xpt_release_ccb(done_ccb);
955 		xpt_schedule(periph, priority);
956 		goto out;
957 	}
958 	case PROBE_SPINUP:
959 		if (bootverbose)
960 			xpt_print(path, "Spin-up done\n");
961 		softc->spinup = 1;
962 		PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
963 		xpt_release_ccb(done_ccb);
964 		xpt_schedule(periph, priority);
965 		goto out;
966 	case PROBE_SETMODE:
967 		/* Set supported bits. */
968 		bzero(&cts, sizeof(cts));
969 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
970 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
971 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
972 		xpt_action((union ccb *)&cts);
973 		if (path->device->transport == XPORT_SATA &&
974 		    cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
975 			caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H;
976 		else if (path->device->transport == XPORT_ATA &&
977 		    cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS)
978 			caps = cts.xport_specific.ata.caps & CTS_ATA_CAPS_H;
979 		else
980 			caps = 0;
981 		if (path->device->transport == XPORT_SATA &&
982 		    ident_buf->satacapabilities != 0xffff) {
983 			if (ident_buf->satacapabilities & ATA_SUPPORT_IFPWRMNGTRCV)
984 				caps |= CTS_SATA_CAPS_D_PMREQ;
985 			if (ident_buf->satacapabilities & ATA_SUPPORT_HAPST)
986 				caps |= CTS_SATA_CAPS_D_APST;
987 		}
988 		/* Mask unwanted bits. */
989 		bzero(&cts, sizeof(cts));
990 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
991 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
992 		cts.type = CTS_TYPE_USER_SETTINGS;
993 		xpt_action((union ccb *)&cts);
994 		if (path->device->transport == XPORT_SATA &&
995 		    cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
996 			caps &= cts.xport_specific.sata.caps;
997 		else if (path->device->transport == XPORT_ATA &&
998 		    cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS)
999 			caps &= cts.xport_specific.ata.caps;
1000 		else
1001 			caps = 0;
1002 		/*
1003 		 * Remember what transport thinks about 48-bit DMA.  If
1004 		 * capability information is not provided or transport is
1005 		 * SATA, we take support for granted.
1006 		 */
1007 		if (!(path->device->inq_flags & SID_DMA) ||
1008 		    (path->device->transport == XPORT_ATA &&
1009 		    (cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS) &&
1010 		    !(caps & CTS_ATA_CAPS_H_DMA48)))
1011 			path->device->inq_flags &= ~SID_DMA48;
1012 		else
1013 			path->device->inq_flags |= SID_DMA48;
1014 		/* Store result to SIM. */
1015 		bzero(&cts, sizeof(cts));
1016 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1017 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1018 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
1019 		if (path->device->transport == XPORT_SATA) {
1020 			cts.xport_specific.sata.caps = caps;
1021 			cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS;
1022 		} else {
1023 			cts.xport_specific.ata.caps = caps;
1024 			cts.xport_specific.ata.valid = CTS_ATA_VALID_CAPS;
1025 		}
1026 		xpt_action((union ccb *)&cts);
1027 		softc->caps = caps;
1028 		if (path->device->transport != XPORT_SATA)
1029 			goto notsata;
1030 		if ((ident_buf->satasupport & ATA_SUPPORT_IFPWRMNGT) &&
1031 		    (!(softc->caps & CTS_SATA_CAPS_H_PMREQ)) !=
1032 		    (!(ident_buf->sataenabled & ATA_SUPPORT_IFPWRMNGT))) {
1033 			PROBE_SET_ACTION(softc, PROBE_SETPM);
1034 			xpt_release_ccb(done_ccb);
1035 			xpt_schedule(periph, priority);
1036 			goto out;
1037 		}
1038 		/* FALLTHROUGH */
1039 	case PROBE_SETPM:
1040 		if (ident_buf->satacapabilities != 0xffff &&
1041 		    (ident_buf->satacapabilities & ATA_SUPPORT_DAPST) &&
1042 		    (!(softc->caps & CTS_SATA_CAPS_H_APST)) !=
1043 		    (!(ident_buf->sataenabled & ATA_ENABLED_DAPST))) {
1044 			PROBE_SET_ACTION(softc, PROBE_SETAPST);
1045 			xpt_release_ccb(done_ccb);
1046 			xpt_schedule(periph, priority);
1047 			goto out;
1048 		}
1049 		/* FALLTHROUGH */
1050 	case PROBE_SETAPST:
1051 		if ((ident_buf->satasupport & ATA_SUPPORT_AUTOACTIVATE) &&
1052 		    (!(softc->caps & CTS_SATA_CAPS_H_DMAAA)) !=
1053 		    (!(ident_buf->sataenabled & ATA_SUPPORT_AUTOACTIVATE))) {
1054 			PROBE_SET_ACTION(softc, PROBE_SETDMAAA);
1055 			xpt_release_ccb(done_ccb);
1056 			xpt_schedule(periph, priority);
1057 			goto out;
1058 		}
1059 		/* FALLTHROUGH */
1060 	case PROBE_SETDMAAA:
1061 		if ((ident_buf->satasupport & ATA_SUPPORT_ASYNCNOTIF) &&
1062 		    (!(softc->caps & CTS_SATA_CAPS_H_AN)) !=
1063 		    (!(ident_buf->sataenabled & ATA_SUPPORT_ASYNCNOTIF))) {
1064 			PROBE_SET_ACTION(softc, PROBE_SETAN);
1065 			xpt_release_ccb(done_ccb);
1066 			xpt_schedule(periph, priority);
1067 			goto out;
1068 		}
1069 		/* FALLTHROUGH */
1070 	case PROBE_SETAN:
1071 notsata:
1072 		if (path->device->protocol == PROTO_ATA) {
1073 			PROBE_SET_ACTION(softc, PROBE_SET_MULTI);
1074 		} else {
1075 			PROBE_SET_ACTION(softc, PROBE_INQUIRY);
1076 		}
1077 		xpt_release_ccb(done_ccb);
1078 		xpt_schedule(periph, priority);
1079 		goto out;
1080 	case PROBE_SET_MULTI:
1081 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1082 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1083 			xpt_acquire_device(path->device);
1084 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1085 			xpt_action(done_ccb);
1086 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1087 		}
1088 		PROBE_SET_ACTION(softc, PROBE_DONE);
1089 		break;
1090 	case PROBE_INQUIRY:
1091 	case PROBE_FULL_INQUIRY:
1092 	{
1093 		u_int8_t periph_qual, len;
1094 
1095 		path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
1096 
1097 		periph_qual = SID_QUAL(inq_buf);
1098 
1099 		if (periph_qual != SID_QUAL_LU_CONNECTED)
1100 			break;
1101 
1102 		/*
1103 		 * We conservatively request only
1104 		 * SHORT_INQUIRY_LEN bytes of inquiry
1105 		 * information during our first try
1106 		 * at sending an INQUIRY. If the device
1107 		 * has more information to give,
1108 		 * perform a second request specifying
1109 		 * the amount of information the device
1110 		 * is willing to give.
1111 		 */
1112 		len = inq_buf->additional_length
1113 		    + offsetof(struct scsi_inquiry_data, additional_length) + 1;
1114 		if (softc->action == PROBE_INQUIRY
1115 		    && len > SHORT_INQUIRY_LENGTH) {
1116 			PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
1117 			xpt_release_ccb(done_ccb);
1118 			xpt_schedule(periph, priority);
1119 			goto out;
1120 		}
1121 
1122 		ata_device_transport(path);
1123 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1124 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1125 			xpt_acquire_device(path->device);
1126 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1127 			xpt_action(done_ccb);
1128 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1129 		}
1130 		PROBE_SET_ACTION(softc, PROBE_DONE);
1131 		break;
1132 	}
1133 	case PROBE_PM_PID:
1134 		if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
1135 			bzero(ident_buf, sizeof(*ident_buf));
1136 		softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
1137 		    (done_ccb->ataio.res.lba_mid << 16) +
1138 		    (done_ccb->ataio.res.lba_low << 8) +
1139 		    done_ccb->ataio.res.sector_count;
1140 		((uint32_t *)ident_buf)[0] = softc->pm_pid;
1141 		snprintf(ident_buf->model, sizeof(ident_buf->model),
1142 		    "Port Multiplier %08x", softc->pm_pid);
1143 		PROBE_SET_ACTION(softc, PROBE_PM_PRV);
1144 		xpt_release_ccb(done_ccb);
1145 		xpt_schedule(periph, priority);
1146 		goto out;
1147 	case PROBE_PM_PRV:
1148 		softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
1149 		    (done_ccb->ataio.res.lba_mid << 16) +
1150 		    (done_ccb->ataio.res.lba_low << 8) +
1151 		    done_ccb->ataio.res.sector_count;
1152 		((uint32_t *)ident_buf)[1] = softc->pm_prv;
1153 		snprintf(ident_buf->revision, sizeof(ident_buf->revision),
1154 		    "%04x", softc->pm_prv);
1155 		path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
1156 		ata_device_transport(path);
1157 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED)
1158 			proberequestdefaultnegotiation(periph);
1159 		/* Set supported bits. */
1160 		bzero(&cts, sizeof(cts));
1161 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1162 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
1163 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
1164 		xpt_action((union ccb *)&cts);
1165 		if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
1166 			caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H;
1167 		else
1168 			caps = 0;
1169 		/* All PMPs must support PM requests. */
1170 		caps |= CTS_SATA_CAPS_D_PMREQ;
1171 		/* Mask unwanted bits. */
1172 		bzero(&cts, sizeof(cts));
1173 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1174 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
1175 		cts.type = CTS_TYPE_USER_SETTINGS;
1176 		xpt_action((union ccb *)&cts);
1177 		if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
1178 			caps &= cts.xport_specific.sata.caps;
1179 		else
1180 			caps = 0;
1181 		/* Remember what transport thinks about AEN. */
1182 		if (caps & CTS_SATA_CAPS_H_AN)
1183 			path->device->inq_flags |= SID_AEN;
1184 		else
1185 			path->device->inq_flags &= ~SID_AEN;
1186 		/* Store result to SIM. */
1187 		bzero(&cts, sizeof(cts));
1188 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1189 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1190 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
1191 		cts.xport_specific.sata.caps = caps;
1192 		cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS;
1193 		xpt_action((union ccb *)&cts);
1194 		softc->caps = caps;
1195 		xpt_async(AC_GETDEV_CHANGED, path, NULL);
1196 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1197 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1198 			xpt_acquire_device(path->device);
1199 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1200 			xpt_action(done_ccb);
1201 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1202 		} else {
1203 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1204 			xpt_action(done_ccb);
1205 			xpt_async(AC_SCSI_AEN, path, done_ccb);
1206 		}
1207 		PROBE_SET_ACTION(softc, PROBE_DONE);
1208 		break;
1209 	case PROBE_IDENTIFY_SES:
1210 	case PROBE_IDENTIFY_SAFTE:
1211 		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
1212 			/* Check that it is the same device. */
1213 			if (bcmp(&softc->ident_data, ident_buf, 53)) {
1214 				/* Device changed. */
1215 				xpt_async(AC_LOST_DEVICE, path, NULL);
1216 			} else {
1217 				bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
1218 				changed = 0;
1219 			}
1220 		}
1221 		if (changed) {
1222 			bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
1223 			/* Clean up from previous instance of this device */
1224 			if (path->device->device_id != NULL) {
1225 				free(path->device->device_id, M_CAMXPT);
1226 				path->device->device_id = NULL;
1227 				path->device->device_id_len = 0;
1228 			}
1229 			path->device->device_id =
1230 			    malloc(16, M_CAMXPT, M_NOWAIT);
1231 			if (path->device->device_id != NULL) {
1232 				path->device->device_id_len = 16;
1233 				bcopy(&fake_device_id_hdr,
1234 				    path->device->device_id, 8);
1235 				bcopy(((uint8_t*)ident_buf) + 2,
1236 				    path->device->device_id + 8, 8);
1237 			}
1238 
1239 			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
1240 		}
1241 		ata_device_transport(path);
1242 		if (changed)
1243 			proberequestdefaultnegotiation(periph);
1244 
1245 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1246 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1247 			xpt_acquire_device(path->device);
1248 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1249 			xpt_action(done_ccb);
1250 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1251 		}
1252 		PROBE_SET_ACTION(softc, PROBE_DONE);
1253 		break;
1254 	default:
1255 		panic("probedone: invalid action state 0x%x\n", softc->action);
1256 	}
1257 done:
1258 	if (softc->restart) {
1259 		softc->restart = 0;
1260 		xpt_release_ccb(done_ccb);
1261 		probeschedule(periph);
1262 		goto out;
1263 	}
1264 	xpt_release_ccb(done_ccb);
1265 	CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n"));
1266 	while ((done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) {
1267 		TAILQ_REMOVE(&softc->request_ccbs,
1268 		    &done_ccb->ccb_h, periph_links.tqe);
1269 		done_ccb->ccb_h.status = found ? CAM_REQ_CMP : CAM_REQ_CMP_ERR;
1270 		xpt_done(done_ccb);
1271 	}
1272 	/* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
1273 	cam_release_devq(path, 0, 0, 0, FALSE);
1274 	cam_periph_invalidate(periph);
1275 	cam_periph_release_locked(periph);
1276 }
1277 
1278 static void
1279 probecleanup(struct cam_periph *periph)
1280 {
1281 	free(periph->softc, M_CAMXPT);
1282 }
1283 
1284 static void
1285 ata_find_quirk(struct cam_ed *device)
1286 {
1287 	struct ata_quirk_entry *quirk;
1288 	caddr_t	match;
1289 
1290 	match = cam_quirkmatch((caddr_t)&device->ident_data,
1291 			       (caddr_t)ata_quirk_table,
1292 			       ata_quirk_table_size,
1293 			       sizeof(*ata_quirk_table), ata_identify_match);
1294 
1295 	if (match == NULL)
1296 		panic("xpt_find_quirk: device didn't match wildcard entry!!");
1297 
1298 	quirk = (struct ata_quirk_entry *)match;
1299 	device->quirk = quirk;
1300 	if (quirk->quirks & CAM_QUIRK_MAXTAGS) {
1301 		device->mintags = quirk->mintags;
1302 		device->maxtags = quirk->maxtags;
1303 	}
1304 }
1305 
1306 typedef struct {
1307 	union	ccb *request_ccb;
1308 	struct 	ccb_pathinq *cpi;
1309 	int	counter;
1310 } ata_scan_bus_info;
1311 
1312 /*
1313  * To start a scan, request_ccb is an XPT_SCAN_BUS ccb.
1314  * As the scan progresses, xpt_scan_bus is used as the
1315  * callback on completion function.
1316  */
1317 static void
1318 ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
1319 {
1320 	struct	cam_path *path;
1321 	ata_scan_bus_info *scan_info;
1322 	union	ccb *work_ccb, *reset_ccb;
1323 	cam_status status;
1324 
1325 	CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE,
1326 		  ("xpt_scan_bus\n"));
1327 	switch (request_ccb->ccb_h.func_code) {
1328 	case XPT_SCAN_BUS:
1329 	case XPT_SCAN_TGT:
1330 		/* Find out the characteristics of the bus */
1331 		work_ccb = xpt_alloc_ccb_nowait();
1332 		if (work_ccb == NULL) {
1333 			request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1334 			xpt_done(request_ccb);
1335 			return;
1336 		}
1337 		xpt_setup_ccb(&work_ccb->ccb_h, request_ccb->ccb_h.path,
1338 			      request_ccb->ccb_h.pinfo.priority);
1339 		work_ccb->ccb_h.func_code = XPT_PATH_INQ;
1340 		xpt_action(work_ccb);
1341 		if (work_ccb->ccb_h.status != CAM_REQ_CMP) {
1342 			request_ccb->ccb_h.status = work_ccb->ccb_h.status;
1343 			xpt_free_ccb(work_ccb);
1344 			xpt_done(request_ccb);
1345 			return;
1346 		}
1347 
1348 		/* We may need to reset bus first, if we haven't done it yet. */
1349 		if ((work_ccb->cpi.hba_inquiry &
1350 		    (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE)) &&
1351 		    !(work_ccb->cpi.hba_misc & PIM_NOBUSRESET) &&
1352 		    !timevalisset(&request_ccb->ccb_h.path->bus->last_reset)) {
1353 			reset_ccb = xpt_alloc_ccb_nowait();
1354 			if (reset_ccb == NULL) {
1355 				request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1356 				xpt_free_ccb(work_ccb);
1357 				xpt_done(request_ccb);
1358 				return;
1359 			}
1360 			xpt_setup_ccb(&reset_ccb->ccb_h, request_ccb->ccb_h.path,
1361 			      CAM_PRIORITY_NONE);
1362 			reset_ccb->ccb_h.func_code = XPT_RESET_BUS;
1363 			xpt_action(reset_ccb);
1364 			if (reset_ccb->ccb_h.status != CAM_REQ_CMP) {
1365 				request_ccb->ccb_h.status = reset_ccb->ccb_h.status;
1366 				xpt_free_ccb(reset_ccb);
1367 				xpt_free_ccb(work_ccb);
1368 				xpt_done(request_ccb);
1369 				return;
1370 			}
1371 			xpt_free_ccb(reset_ccb);
1372 		}
1373 
1374 		/* Save some state for use while we probe for devices */
1375 		scan_info = (ata_scan_bus_info *)
1376 		    malloc(sizeof(ata_scan_bus_info), M_CAMXPT, M_NOWAIT);
1377 		if (scan_info == NULL) {
1378 			request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1379 			xpt_free_ccb(work_ccb);
1380 			xpt_done(request_ccb);
1381 			return;
1382 		}
1383 		scan_info->request_ccb = request_ccb;
1384 		scan_info->cpi = &work_ccb->cpi;
1385 		/* If PM supported, probe it first. */
1386 		if (scan_info->cpi->hba_inquiry & PI_SATAPM)
1387 			scan_info->counter = scan_info->cpi->max_target;
1388 		else
1389 			scan_info->counter = 0;
1390 
1391 		work_ccb = xpt_alloc_ccb_nowait();
1392 		if (work_ccb == NULL) {
1393 			free(scan_info, M_CAMXPT);
1394 			request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1395 			xpt_done(request_ccb);
1396 			break;
1397 		}
1398 		goto scan_next;
1399 	case XPT_SCAN_LUN:
1400 		work_ccb = request_ccb;
1401 		/* Reuse the same CCB to query if a device was really found */
1402 		scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0;
1403 		/* If there is PMP... */
1404 		if ((scan_info->cpi->hba_inquiry & PI_SATAPM) &&
1405 		    (scan_info->counter == scan_info->cpi->max_target)) {
1406 			if (work_ccb->ccb_h.status == CAM_REQ_CMP) {
1407 				/* everything else will be probed by it */
1408 				/* Free the current request path- we're done with it. */
1409 				xpt_free_path(work_ccb->ccb_h.path);
1410 				goto done;
1411 			} else {
1412 				struct ccb_trans_settings cts;
1413 
1414 				/* Report SIM that PM is absent. */
1415 				bzero(&cts, sizeof(cts));
1416 				xpt_setup_ccb(&cts.ccb_h,
1417 				    work_ccb->ccb_h.path, CAM_PRIORITY_NONE);
1418 				cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1419 				cts.type = CTS_TYPE_CURRENT_SETTINGS;
1420 				cts.xport_specific.sata.pm_present = 0;
1421 				cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
1422 				xpt_action((union ccb *)&cts);
1423 			}
1424 		}
1425 		/* Free the current request path- we're done with it. */
1426 		xpt_free_path(work_ccb->ccb_h.path);
1427 		if (scan_info->counter ==
1428 		    ((scan_info->cpi->hba_inquiry & PI_SATAPM) ?
1429 		    0 : scan_info->cpi->max_target)) {
1430 done:
1431 			xpt_free_ccb(work_ccb);
1432 			xpt_free_ccb((union ccb *)scan_info->cpi);
1433 			request_ccb = scan_info->request_ccb;
1434 			free(scan_info, M_CAMXPT);
1435 			request_ccb->ccb_h.status = CAM_REQ_CMP;
1436 			xpt_done(request_ccb);
1437 			break;
1438 		}
1439 		/* Take next device. Wrap from max (PMP) to 0. */
1440 		scan_info->counter = (scan_info->counter + 1 ) %
1441 		    (scan_info->cpi->max_target + 1);
1442 scan_next:
1443 		status = xpt_create_path(&path, NULL,
1444 		    scan_info->request_ccb->ccb_h.path_id,
1445 		    scan_info->counter, 0);
1446 		if (status != CAM_REQ_CMP) {
1447 			printf("xpt_scan_bus: xpt_create_path failed"
1448 			    " with status %#x, bus scan halted\n",
1449 			    status);
1450 			xpt_free_ccb(work_ccb);
1451 			xpt_free_ccb((union ccb *)scan_info->cpi);
1452 			request_ccb = scan_info->request_ccb;
1453 			free(scan_info, M_CAMXPT);
1454 			request_ccb->ccb_h.status = status;
1455 			xpt_done(request_ccb);
1456 			break;
1457 		}
1458 		xpt_setup_ccb(&work_ccb->ccb_h, path,
1459 		    scan_info->request_ccb->ccb_h.pinfo.priority);
1460 		work_ccb->ccb_h.func_code = XPT_SCAN_LUN;
1461 		work_ccb->ccb_h.cbfcnp = ata_scan_bus;
1462 		work_ccb->ccb_h.ppriv_ptr0 = scan_info;
1463 		work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags;
1464 		xpt_action(work_ccb);
1465 		break;
1466 	default:
1467 		break;
1468 	}
1469 }
1470 
1471 static void
1472 ata_scan_lun(struct cam_periph *periph, struct cam_path *path,
1473 	     cam_flags flags, union ccb *request_ccb)
1474 {
1475 	struct ccb_pathinq cpi;
1476 	cam_status status;
1477 	struct cam_path *new_path;
1478 	struct cam_periph *old_periph;
1479 
1480 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n"));
1481 
1482 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
1483 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1484 	xpt_action((union ccb *)&cpi);
1485 
1486 	if (cpi.ccb_h.status != CAM_REQ_CMP) {
1487 		if (request_ccb != NULL) {
1488 			request_ccb->ccb_h.status = cpi.ccb_h.status;
1489 			xpt_done(request_ccb);
1490 		}
1491 		return;
1492 	}
1493 
1494 	if (request_ccb == NULL) {
1495 		request_ccb = xpt_alloc_ccb_nowait();
1496 		if (request_ccb == NULL) {
1497 			xpt_print(path, "xpt_scan_lun: can't allocate CCB, "
1498 			    "can't continue\n");
1499 			return;
1500 		}
1501 		status = xpt_create_path(&new_path, NULL,
1502 					  path->bus->path_id,
1503 					  path->target->target_id,
1504 					  path->device->lun_id);
1505 		if (status != CAM_REQ_CMP) {
1506 			xpt_print(path, "xpt_scan_lun: can't create path, "
1507 			    "can't continue\n");
1508 			xpt_free_ccb(request_ccb);
1509 			return;
1510 		}
1511 		xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT);
1512 		request_ccb->ccb_h.cbfcnp = xptscandone;
1513 		request_ccb->ccb_h.func_code = XPT_SCAN_LUN;
1514 		request_ccb->crcn.flags = flags;
1515 	}
1516 
1517 	if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) {
1518 		if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) {
1519 			probe_softc *softc;
1520 
1521 			softc = (probe_softc *)old_periph->softc;
1522 			TAILQ_INSERT_TAIL(&softc->request_ccbs,
1523 				&request_ccb->ccb_h, periph_links.tqe);
1524 			softc->restart = 1;
1525 		} else {
1526 			request_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1527 			xpt_done(request_ccb);
1528 		}
1529 	} else {
1530 		status = cam_periph_alloc(proberegister, NULL, probecleanup,
1531 					  probestart, "aprobe",
1532 					  CAM_PERIPH_BIO,
1533 					  request_ccb->ccb_h.path, NULL, 0,
1534 					  request_ccb);
1535 
1536 		if (status != CAM_REQ_CMP) {
1537 			xpt_print(path, "xpt_scan_lun: cam_alloc_periph "
1538 			    "returned an error, can't continue probe\n");
1539 			request_ccb->ccb_h.status = status;
1540 			xpt_done(request_ccb);
1541 		}
1542 	}
1543 }
1544 
1545 static void
1546 xptscandone(struct cam_periph *periph, union ccb *done_ccb)
1547 {
1548 
1549 	xpt_free_path(done_ccb->ccb_h.path);
1550 	xpt_free_ccb(done_ccb);
1551 }
1552 
1553 static struct cam_ed *
1554 ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
1555 {
1556 	struct cam_path path;
1557 	struct ata_quirk_entry *quirk;
1558 	struct cam_ed *device;
1559 
1560 	device = xpt_alloc_device(bus, target, lun_id);
1561 	if (device == NULL)
1562 		return (NULL);
1563 
1564 	/*
1565 	 * Take the default quirk entry until we have inquiry
1566 	 * data and can determine a better quirk to use.
1567 	 */
1568 	quirk = &ata_quirk_table[ata_quirk_table_size - 1];
1569 	device->quirk = (void *)quirk;
1570 	device->mintags = 0;
1571 	device->maxtags = 0;
1572 	bzero(&device->inq_data, sizeof(device->inq_data));
1573 	device->inq_flags = 0;
1574 	device->queue_flags = 0;
1575 	device->serial_num = NULL;
1576 	device->serial_num_len = 0;
1577 
1578 	/*
1579 	 * XXX should be limited by number of CCBs this bus can
1580 	 * do.
1581 	 */
1582 	bus->sim->max_ccbs += device->ccbq.devq_openings;
1583 	if (lun_id != CAM_LUN_WILDCARD) {
1584 		xpt_compile_path(&path,
1585 				 NULL,
1586 				 bus->path_id,
1587 				 target->target_id,
1588 				 lun_id);
1589 		ata_device_transport(&path);
1590 		xpt_release_path(&path);
1591 	}
1592 
1593 	return (device);
1594 }
1595 
1596 static void
1597 ata_device_transport(struct cam_path *path)
1598 {
1599 	struct ccb_pathinq cpi;
1600 	struct ccb_trans_settings cts;
1601 	struct scsi_inquiry_data *inq_buf = NULL;
1602 	struct ata_params *ident_buf = NULL;
1603 
1604 	/* Get transport information from the SIM */
1605 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
1606 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1607 	xpt_action((union ccb *)&cpi);
1608 
1609 	path->device->transport = cpi.transport;
1610 	if ((path->device->flags & CAM_DEV_INQUIRY_DATA_VALID) != 0)
1611 		inq_buf = &path->device->inq_data;
1612 	if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) != 0)
1613 		ident_buf = &path->device->ident_data;
1614 	if (path->device->protocol == PROTO_ATA) {
1615 		path->device->protocol_version = ident_buf ?
1616 		    ata_version(ident_buf->version_major) : cpi.protocol_version;
1617 	} else if (path->device->protocol == PROTO_SCSI) {
1618 		path->device->protocol_version = inq_buf ?
1619 		    SID_ANSI_REV(inq_buf) : cpi.protocol_version;
1620 	}
1621 	path->device->transport_version = ident_buf ?
1622 	    ata_version(ident_buf->version_major) : cpi.transport_version;
1623 
1624 	/* Tell the controller what we think */
1625 	xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1626 	cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1627 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
1628 	cts.transport = path->device->transport;
1629 	cts.transport_version = path->device->transport_version;
1630 	cts.protocol = path->device->protocol;
1631 	cts.protocol_version = path->device->protocol_version;
1632 	cts.proto_specific.valid = 0;
1633 	if (ident_buf) {
1634 		if (path->device->transport == XPORT_ATA) {
1635 			cts.xport_specific.ata.atapi =
1636 			    (ident_buf->config == ATA_PROTO_CFA) ? 0 :
1637 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 :
1638 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0;
1639 			cts.xport_specific.ata.valid = CTS_ATA_VALID_ATAPI;
1640 		} else {
1641 			cts.xport_specific.sata.atapi =
1642 			    (ident_buf->config == ATA_PROTO_CFA) ? 0 :
1643 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 :
1644 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0;
1645 			cts.xport_specific.sata.valid = CTS_SATA_VALID_ATAPI;
1646 		}
1647 	} else
1648 		cts.xport_specific.valid = 0;
1649 	xpt_action((union ccb *)&cts);
1650 }
1651 
1652 static void
1653 ata_dev_advinfo(union ccb *start_ccb)
1654 {
1655 	struct cam_ed *device;
1656 	struct ccb_dev_advinfo *cdai;
1657 	off_t amt;
1658 
1659 	start_ccb->ccb_h.status = CAM_REQ_INVALID;
1660 	device = start_ccb->ccb_h.path->device;
1661 	cdai = &start_ccb->cdai;
1662 	switch(cdai->buftype) {
1663 	case CDAI_TYPE_SCSI_DEVID:
1664 		if (cdai->flags & CDAI_FLAG_STORE)
1665 			return;
1666 		cdai->provsiz = device->device_id_len;
1667 		if (device->device_id_len == 0)
1668 			break;
1669 		amt = device->device_id_len;
1670 		if (cdai->provsiz > cdai->bufsiz)
1671 			amt = cdai->bufsiz;
1672 		memcpy(cdai->buf, device->device_id, amt);
1673 		break;
1674 	case CDAI_TYPE_SERIAL_NUM:
1675 		if (cdai->flags & CDAI_FLAG_STORE)
1676 			return;
1677 		cdai->provsiz = device->serial_num_len;
1678 		if (device->serial_num_len == 0)
1679 			break;
1680 		amt = device->serial_num_len;
1681 		if (cdai->provsiz > cdai->bufsiz)
1682 			amt = cdai->bufsiz;
1683 		memcpy(cdai->buf, device->serial_num, amt);
1684 		break;
1685 	case CDAI_TYPE_PHYS_PATH:
1686 		if (cdai->flags & CDAI_FLAG_STORE) {
1687 			if (device->physpath != NULL)
1688 				free(device->physpath, M_CAMXPT);
1689 			device->physpath_len = cdai->bufsiz;
1690 			/* Clear existing buffer if zero length */
1691 			if (cdai->bufsiz == 0)
1692 				break;
1693 			device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT);
1694 			if (device->physpath == NULL) {
1695 				start_ccb->ccb_h.status = CAM_REQ_ABORTED;
1696 				return;
1697 			}
1698 			memcpy(device->physpath, cdai->buf, cdai->bufsiz);
1699 		} else {
1700 			cdai->provsiz = device->physpath_len;
1701 			if (device->physpath_len == 0)
1702 				break;
1703 			amt = device->physpath_len;
1704 			if (cdai->provsiz > cdai->bufsiz)
1705 				amt = cdai->bufsiz;
1706 			memcpy(cdai->buf, device->physpath, amt);
1707 		}
1708 		break;
1709 	default:
1710 		return;
1711 	}
1712 	start_ccb->ccb_h.status = CAM_REQ_CMP;
1713 
1714 	if (cdai->flags & CDAI_FLAG_STORE) {
1715 		int owned;
1716 
1717 		owned = mtx_owned(start_ccb->ccb_h.path->bus->sim->mtx);
1718 		if (owned == 0)
1719 			mtx_lock(start_ccb->ccb_h.path->bus->sim->mtx);
1720 		xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path,
1721 			  (void *)(uintptr_t)cdai->buftype);
1722 		if (owned == 0)
1723 			mtx_unlock(start_ccb->ccb_h.path->bus->sim->mtx);
1724 	}
1725 }
1726 
1727 static void
1728 ata_action(union ccb *start_ccb)
1729 {
1730 
1731 	switch (start_ccb->ccb_h.func_code) {
1732 	case XPT_SET_TRAN_SETTINGS:
1733 	{
1734 		ata_set_transfer_settings(&start_ccb->cts,
1735 					   start_ccb->ccb_h.path->device,
1736 					   /*async_update*/FALSE);
1737 		break;
1738 	}
1739 	case XPT_SCAN_BUS:
1740 	case XPT_SCAN_TGT:
1741 		ata_scan_bus(start_ccb->ccb_h.path->periph, start_ccb);
1742 		break;
1743 	case XPT_SCAN_LUN:
1744 		ata_scan_lun(start_ccb->ccb_h.path->periph,
1745 			      start_ccb->ccb_h.path, start_ccb->crcn.flags,
1746 			      start_ccb);
1747 		break;
1748 	case XPT_GET_TRAN_SETTINGS:
1749 	{
1750 		ata_get_transfer_settings(&start_ccb->cts);
1751 		break;
1752 	}
1753 	case XPT_SCSI_IO:
1754 	{
1755 		struct cam_ed *device;
1756 		u_int	maxlen = 0;
1757 
1758 		device = start_ccb->ccb_h.path->device;
1759 		if (device->protocol == PROTO_SCSI &&
1760 		    (device->flags & CAM_DEV_IDENTIFY_DATA_VALID)) {
1761 			uint16_t p =
1762 			    device->ident_data.config & ATA_PROTO_MASK;
1763 
1764 			maxlen =
1765 			    (device->ident_data.config == ATA_PROTO_CFA) ? 0 :
1766 			    (p == ATA_PROTO_ATAPI_16) ? 16 :
1767 			    (p == ATA_PROTO_ATAPI_12) ? 12 : 0;
1768 		}
1769 		if (start_ccb->csio.cdb_len > maxlen) {
1770 			start_ccb->ccb_h.status = CAM_REQ_INVALID;
1771 			xpt_done(start_ccb);
1772 			break;
1773 		}
1774 		xpt_action_default(start_ccb);
1775 		break;
1776 	}
1777 	case XPT_DEV_ADVINFO:
1778 	{
1779 		ata_dev_advinfo(start_ccb);
1780 		break;
1781 	}
1782 	default:
1783 		xpt_action_default(start_ccb);
1784 		break;
1785 	}
1786 }
1787 
1788 static void
1789 ata_get_transfer_settings(struct ccb_trans_settings *cts)
1790 {
1791 	struct	ccb_trans_settings_ata *ata;
1792 	struct	ccb_trans_settings_scsi *scsi;
1793 	struct	cam_ed *device;
1794 	struct	cam_sim *sim;
1795 
1796 	device = cts->ccb_h.path->device;
1797 	sim = cts->ccb_h.path->bus->sim;
1798 	(*(sim->sim_action))(sim, (union ccb *)cts);
1799 
1800 	if (cts->protocol == PROTO_UNKNOWN ||
1801 	    cts->protocol == PROTO_UNSPECIFIED) {
1802 		cts->protocol = device->protocol;
1803 		cts->protocol_version = device->protocol_version;
1804 	}
1805 
1806 	if (cts->protocol == PROTO_ATA) {
1807 		ata = &cts->proto_specific.ata;
1808 		if ((ata->valid & CTS_ATA_VALID_TQ) == 0) {
1809 			ata->valid |= CTS_ATA_VALID_TQ;
1810 			if (cts->type == CTS_TYPE_USER_SETTINGS ||
1811 			    (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
1812 			    (device->inq_flags & SID_CmdQue) != 0)
1813 				ata->flags |= CTS_ATA_FLAGS_TAG_ENB;
1814 		}
1815 	}
1816 	if (cts->protocol == PROTO_SCSI) {
1817 		scsi = &cts->proto_specific.scsi;
1818 		if ((scsi->valid & CTS_SCSI_VALID_TQ) == 0) {
1819 			scsi->valid |= CTS_SCSI_VALID_TQ;
1820 			if (cts->type == CTS_TYPE_USER_SETTINGS ||
1821 			    (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
1822 			    (device->inq_flags & SID_CmdQue) != 0)
1823 				scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
1824 		}
1825 	}
1826 
1827 	if (cts->transport == XPORT_UNKNOWN ||
1828 	    cts->transport == XPORT_UNSPECIFIED) {
1829 		cts->transport = device->transport;
1830 		cts->transport_version = device->transport_version;
1831 	}
1832 }
1833 
1834 static void
1835 ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device,
1836 			   int async_update)
1837 {
1838 	struct	ccb_pathinq cpi;
1839 	struct	ccb_trans_settings_ata *ata;
1840 	struct	ccb_trans_settings_scsi *scsi;
1841 	struct	cam_sim *sim;
1842 	struct	ata_params *ident_data;
1843 	struct	scsi_inquiry_data *inq_data;
1844 
1845 	if (device == NULL) {
1846 		cts->ccb_h.status = CAM_PATH_INVALID;
1847 		xpt_done((union ccb *)cts);
1848 		return;
1849 	}
1850 
1851 	if (cts->protocol == PROTO_UNKNOWN
1852 	 || cts->protocol == PROTO_UNSPECIFIED) {
1853 		cts->protocol = device->protocol;
1854 		cts->protocol_version = device->protocol_version;
1855 	}
1856 
1857 	if (cts->protocol_version == PROTO_VERSION_UNKNOWN
1858 	 || cts->protocol_version == PROTO_VERSION_UNSPECIFIED)
1859 		cts->protocol_version = device->protocol_version;
1860 
1861 	if (cts->protocol != device->protocol) {
1862 		xpt_print(cts->ccb_h.path, "Uninitialized Protocol %x:%x?\n",
1863 		       cts->protocol, device->protocol);
1864 		cts->protocol = device->protocol;
1865 	}
1866 
1867 	if (cts->protocol_version > device->protocol_version) {
1868 		if (bootverbose) {
1869 			xpt_print(cts->ccb_h.path, "Down reving Protocol "
1870 			    "Version from %d to %d?\n", cts->protocol_version,
1871 			    device->protocol_version);
1872 		}
1873 		cts->protocol_version = device->protocol_version;
1874 	}
1875 
1876 	if (cts->transport == XPORT_UNKNOWN
1877 	 || cts->transport == XPORT_UNSPECIFIED) {
1878 		cts->transport = device->transport;
1879 		cts->transport_version = device->transport_version;
1880 	}
1881 
1882 	if (cts->transport_version == XPORT_VERSION_UNKNOWN
1883 	 || cts->transport_version == XPORT_VERSION_UNSPECIFIED)
1884 		cts->transport_version = device->transport_version;
1885 
1886 	if (cts->transport != device->transport) {
1887 		xpt_print(cts->ccb_h.path, "Uninitialized Transport %x:%x?\n",
1888 		    cts->transport, device->transport);
1889 		cts->transport = device->transport;
1890 	}
1891 
1892 	if (cts->transport_version > device->transport_version) {
1893 		if (bootverbose) {
1894 			xpt_print(cts->ccb_h.path, "Down reving Transport "
1895 			    "Version from %d to %d?\n", cts->transport_version,
1896 			    device->transport_version);
1897 		}
1898 		cts->transport_version = device->transport_version;
1899 	}
1900 
1901 	sim = cts->ccb_h.path->bus->sim;
1902 	ident_data = &device->ident_data;
1903 	inq_data = &device->inq_data;
1904 	if (cts->protocol == PROTO_ATA)
1905 		ata = &cts->proto_specific.ata;
1906 	else
1907 		ata = NULL;
1908 	if (cts->protocol == PROTO_SCSI)
1909 		scsi = &cts->proto_specific.scsi;
1910 	else
1911 		scsi = NULL;
1912 	xpt_setup_ccb(&cpi.ccb_h, cts->ccb_h.path, CAM_PRIORITY_NONE);
1913 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1914 	xpt_action((union ccb *)&cpi);
1915 
1916 	/* Sanity checking */
1917 	if ((cpi.hba_inquiry & PI_TAG_ABLE) == 0
1918 	 || (ata && (ident_data->satacapabilities & ATA_SUPPORT_NCQ) == 0)
1919 	 || (scsi && (INQ_DATA_TQ_ENABLED(inq_data)) == 0)
1920 	 || (device->queue_flags & SCP_QUEUE_DQUE) != 0
1921 	 || (device->mintags == 0)) {
1922 		/*
1923 		 * Can't tag on hardware that doesn't support tags,
1924 		 * doesn't have it enabled, or has broken tag support.
1925 		 */
1926 		if (ata)
1927 			ata->flags &= ~CTS_ATA_FLAGS_TAG_ENB;
1928 		if (scsi)
1929 			scsi->flags &= ~CTS_SCSI_FLAGS_TAG_ENB;
1930 	}
1931 
1932 	/* Start/stop tags use. */
1933 	if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
1934 	    ((ata && (ata->valid & CTS_ATA_VALID_TQ) != 0) ||
1935 	     (scsi && (scsi->valid & CTS_SCSI_VALID_TQ) != 0))) {
1936 		int nowt, newt = 0;
1937 
1938 		nowt = ((device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
1939 			(device->inq_flags & SID_CmdQue) != 0);
1940 		if (ata)
1941 			newt = (ata->flags & CTS_ATA_FLAGS_TAG_ENB) != 0;
1942 		if (scsi)
1943 			newt = (scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0;
1944 
1945 		if (newt && !nowt) {
1946 			/*
1947 			 * Delay change to use tags until after a
1948 			 * few commands have gone to this device so
1949 			 * the controller has time to perform transfer
1950 			 * negotiations without tagged messages getting
1951 			 * in the way.
1952 			 */
1953 			device->tag_delay_count = CAM_TAG_DELAY_COUNT;
1954 			device->flags |= CAM_DEV_TAG_AFTER_COUNT;
1955 		} else if (nowt && !newt)
1956 			xpt_stop_tags(cts->ccb_h.path);
1957 	}
1958 
1959 	if (async_update == FALSE)
1960 		(*(sim->sim_action))(sim, (union ccb *)cts);
1961 }
1962 
1963 /*
1964  * Handle any per-device event notifications that require action by the XPT.
1965  */
1966 static void
1967 ata_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target,
1968 	      struct cam_ed *device, void *async_arg)
1969 {
1970 	cam_status status;
1971 	struct cam_path newpath;
1972 
1973 	/*
1974 	 * We only need to handle events for real devices.
1975 	 */
1976 	if (target->target_id == CAM_TARGET_WILDCARD
1977 	 || device->lun_id == CAM_LUN_WILDCARD)
1978 		return;
1979 
1980 	/*
1981 	 * We need our own path with wildcards expanded to
1982 	 * handle certain types of events.
1983 	 */
1984 	if ((async_code == AC_SENT_BDR)
1985 	 || (async_code == AC_BUS_RESET)
1986 	 || (async_code == AC_INQ_CHANGED))
1987 		status = xpt_compile_path(&newpath, NULL,
1988 					  bus->path_id,
1989 					  target->target_id,
1990 					  device->lun_id);
1991 	else
1992 		status = CAM_REQ_CMP_ERR;
1993 
1994 	if (status == CAM_REQ_CMP) {
1995 		if (async_code == AC_INQ_CHANGED) {
1996 			/*
1997 			 * We've sent a start unit command, or
1998 			 * something similar to a device that
1999 			 * may have caused its inquiry data to
2000 			 * change. So we re-scan the device to
2001 			 * refresh the inquiry data for it.
2002 			 */
2003 			ata_scan_lun(newpath.periph, &newpath,
2004 				     CAM_EXPECT_INQ_CHANGE, NULL);
2005 		} else {
2006 			/* We need to reinitialize device after reset. */
2007 			ata_scan_lun(newpath.periph, &newpath,
2008 				     0, NULL);
2009 		}
2010 		xpt_release_path(&newpath);
2011 	} else if (async_code == AC_LOST_DEVICE &&
2012 	    (device->flags & CAM_DEV_UNCONFIGURED) == 0) {
2013 		device->flags |= CAM_DEV_UNCONFIGURED;
2014 		xpt_release_device(device);
2015 	} else if (async_code == AC_TRANSFER_NEG) {
2016 		struct ccb_trans_settings *settings;
2017 
2018 		settings = (struct ccb_trans_settings *)async_arg;
2019 		ata_set_transfer_settings(settings, device,
2020 					  /*async_update*/TRUE);
2021 	}
2022 }
2023 
2024 static void
2025 ata_announce_periph(struct cam_periph *periph)
2026 {
2027 	struct	ccb_pathinq cpi;
2028 	struct	ccb_trans_settings cts;
2029 	struct	cam_path *path = periph->path;
2030 	u_int	speed;
2031 	u_int	mb;
2032 
2033 	mtx_assert(periph->sim->mtx, MA_OWNED);
2034 
2035 	xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
2036 	cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
2037 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
2038 	xpt_action((union ccb*)&cts);
2039 	if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
2040 		return;
2041 	/* Ask the SIM for its base transfer speed */
2042 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
2043 	cpi.ccb_h.func_code = XPT_PATH_INQ;
2044 	xpt_action((union ccb *)&cpi);
2045 	/* Report connection speed */
2046 	speed = cpi.base_transfer_speed;
2047 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) {
2048 		struct	ccb_trans_settings_pata *pata =
2049 		    &cts.xport_specific.ata;
2050 
2051 		if (pata->valid & CTS_ATA_VALID_MODE)
2052 			speed = ata_mode2speed(pata->mode);
2053 	}
2054 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
2055 		struct	ccb_trans_settings_sata *sata =
2056 		    &cts.xport_specific.sata;
2057 
2058 		if (sata->valid & CTS_SATA_VALID_REVISION)
2059 			speed = ata_revision2speed(sata->revision);
2060 	}
2061 	mb = speed / 1000;
2062 	if (mb > 0)
2063 		printf("%s%d: %d.%03dMB/s transfers",
2064 		       periph->periph_name, periph->unit_number,
2065 		       mb, speed % 1000);
2066 	else
2067 		printf("%s%d: %dKB/s transfers", periph->periph_name,
2068 		       periph->unit_number, speed);
2069 	/* Report additional information about connection */
2070 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) {
2071 		struct ccb_trans_settings_pata *pata =
2072 		    &cts.xport_specific.ata;
2073 
2074 		printf(" (");
2075 		if (pata->valid & CTS_ATA_VALID_MODE)
2076 			printf("%s, ", ata_mode2string(pata->mode));
2077 		if ((pata->valid & CTS_ATA_VALID_ATAPI) && pata->atapi != 0)
2078 			printf("ATAPI %dbytes, ", pata->atapi);
2079 		if (pata->valid & CTS_ATA_VALID_BYTECOUNT)
2080 			printf("PIO %dbytes", pata->bytecount);
2081 		printf(")");
2082 	}
2083 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
2084 		struct ccb_trans_settings_sata *sata =
2085 		    &cts.xport_specific.sata;
2086 
2087 		printf(" (");
2088 		if (sata->valid & CTS_SATA_VALID_REVISION)
2089 			printf("SATA %d.x, ", sata->revision);
2090 		else
2091 			printf("SATA, ");
2092 		if (sata->valid & CTS_SATA_VALID_MODE)
2093 			printf("%s, ", ata_mode2string(sata->mode));
2094 		if ((sata->valid & CTS_ATA_VALID_ATAPI) && sata->atapi != 0)
2095 			printf("ATAPI %dbytes, ", sata->atapi);
2096 		if (sata->valid & CTS_SATA_VALID_BYTECOUNT)
2097 			printf("PIO %dbytes", sata->bytecount);
2098 		printf(")");
2099 	}
2100 	printf("\n");
2101 }
2102