xref: /freebsd/sys/cam/ata/ata_xpt.c (revision b1f92fa22938fe29ab7e53692ffe0ed7a0ecc4d0)
1 /*-
2  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/endian.h>
33 #include <sys/systm.h>
34 #include <sys/types.h>
35 #include <sys/malloc.h>
36 #include <sys/kernel.h>
37 #include <sys/time.h>
38 #include <sys/conf.h>
39 #include <sys/fcntl.h>
40 #include <sys/interrupt.h>
41 #include <sys/sbuf.h>
42 
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 
47 #include <cam/cam.h>
48 #include <cam/cam_ccb.h>
49 #include <cam/cam_queue.h>
50 #include <cam/cam_periph.h>
51 #include <cam/cam_sim.h>
52 #include <cam/cam_xpt.h>
53 #include <cam/cam_xpt_sim.h>
54 #include <cam/cam_xpt_periph.h>
55 #include <cam/cam_xpt_internal.h>
56 #include <cam/cam_debug.h>
57 
58 #include <cam/scsi/scsi_all.h>
59 #include <cam/scsi/scsi_message.h>
60 #include <cam/ata/ata_all.h>
61 #include <machine/stdarg.h>	/* for xpt_print below */
62 #include "opt_cam.h"
63 
64 struct ata_quirk_entry {
65 	struct scsi_inquiry_pattern inq_pat;
66 	u_int8_t quirks;
67 #define	CAM_QUIRK_MAXTAGS	0x01
68 	u_int mintags;
69 	u_int maxtags;
70 };
71 
72 static periph_init_t probe_periph_init;
73 
74 static struct periph_driver probe_driver =
75 {
76 	probe_periph_init, "aprobe",
77 	TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0,
78 	CAM_PERIPH_DRV_EARLY
79 };
80 
81 PERIPHDRIVER_DECLARE(aprobe, probe_driver);
82 
83 typedef enum {
84 	PROBE_RESET,
85 	PROBE_IDENTIFY,
86 	PROBE_SPINUP,
87 	PROBE_SETMODE,
88 	PROBE_SETPM,
89 	PROBE_SETAPST,
90 	PROBE_SETDMAAA,
91 	PROBE_SETAN,
92 	PROBE_SET_MULTI,
93 	PROBE_INQUIRY,
94 	PROBE_FULL_INQUIRY,
95 	PROBE_PM_PID,
96 	PROBE_PM_PRV,
97 	PROBE_IDENTIFY_SES,
98 	PROBE_IDENTIFY_SAFTE,
99 	PROBE_DONE,
100 	PROBE_INVALID
101 } probe_action;
102 
103 static char *probe_action_text[] = {
104 	"PROBE_RESET",
105 	"PROBE_IDENTIFY",
106 	"PROBE_SPINUP",
107 	"PROBE_SETMODE",
108 	"PROBE_SETPM",
109 	"PROBE_SETAPST",
110 	"PROBE_SETDMAAA",
111 	"PROBE_SETAN",
112 	"PROBE_SET_MULTI",
113 	"PROBE_INQUIRY",
114 	"PROBE_FULL_INQUIRY",
115 	"PROBE_PM_PID",
116 	"PROBE_PM_PRV",
117 	"PROBE_IDENTIFY_SES",
118 	"PROBE_IDENTIFY_SAFTE",
119 	"PROBE_DONE",
120 	"PROBE_INVALID"
121 };
122 
123 #define PROBE_SET_ACTION(softc, newaction)	\
124 do {									\
125 	char **text;							\
126 	text = probe_action_text;					\
127 	CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE,		\
128 	    ("Probe %s to %s\n", text[(softc)->action],			\
129 	    text[(newaction)]));					\
130 	(softc)->action = (newaction);					\
131 } while(0)
132 
133 typedef enum {
134 	PROBE_NO_ANNOUNCE	= 0x04
135 } probe_flags;
136 
137 typedef struct {
138 	TAILQ_HEAD(, ccb_hdr) request_ccbs;
139 	struct ata_params	ident_data;
140 	probe_action	action;
141 	probe_flags	flags;
142 	uint32_t	pm_pid;
143 	uint32_t	pm_prv;
144 	int		restart;
145 	int		spinup;
146 	int		faults;
147 	u_int		caps;
148 	struct cam_periph *periph;
149 } probe_softc;
150 
151 static struct ata_quirk_entry ata_quirk_table[] =
152 {
153 	{
154 		/* Default tagged queuing parameters for all devices */
155 		{
156 		  T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
157 		  /*vendor*/"*", /*product*/"*", /*revision*/"*"
158 		},
159 		/*quirks*/0, /*mintags*/0, /*maxtags*/0
160 	},
161 };
162 
163 static const int ata_quirk_table_size =
164 	sizeof(ata_quirk_table) / sizeof(*ata_quirk_table);
165 
166 static cam_status	proberegister(struct cam_periph *periph,
167 				      void *arg);
168 static void	 probeschedule(struct cam_periph *probe_periph);
169 static void	 probestart(struct cam_periph *periph, union ccb *start_ccb);
170 static void	 proberequestdefaultnegotiation(struct cam_periph *periph);
171 static void	 probedone(struct cam_periph *periph, union ccb *done_ccb);
172 static void	 probecleanup(struct cam_periph *periph);
173 static void	 ata_find_quirk(struct cam_ed *device);
174 static void	 ata_scan_bus(struct cam_periph *periph, union ccb *ccb);
175 static void	 ata_scan_lun(struct cam_periph *periph,
176 			       struct cam_path *path, cam_flags flags,
177 			       union ccb *ccb);
178 static void	 xptscandone(struct cam_periph *periph, union ccb *done_ccb);
179 static struct cam_ed *
180 		 ata_alloc_device(struct cam_eb *bus, struct cam_et *target,
181 				   lun_id_t lun_id);
182 static void	 ata_device_transport(struct cam_path *path);
183 static void	 ata_get_transfer_settings(struct ccb_trans_settings *cts);
184 static void	 ata_set_transfer_settings(struct ccb_trans_settings *cts,
185 					    struct cam_path *path,
186 					    int async_update);
187 static void	 ata_dev_async(u_int32_t async_code,
188 				struct cam_eb *bus,
189 				struct cam_et *target,
190 				struct cam_ed *device,
191 				void *async_arg);
192 static void	 ata_action(union ccb *start_ccb);
193 static void	 ata_announce_periph(struct cam_periph *periph);
194 
195 static int ata_dma = 1;
196 static int atapi_dma = 1;
197 
198 TUNABLE_INT("hw.ata.ata_dma", &ata_dma);
199 TUNABLE_INT("hw.ata.atapi_dma", &atapi_dma);
200 
201 static struct xpt_xport ata_xport = {
202 	.alloc_device = ata_alloc_device,
203 	.action = ata_action,
204 	.async = ata_dev_async,
205 	.announce = ata_announce_periph,
206 };
207 
208 struct xpt_xport *
209 ata_get_xport(void)
210 {
211 	return (&ata_xport);
212 }
213 
214 static void
215 probe_periph_init()
216 {
217 }
218 
219 static cam_status
220 proberegister(struct cam_periph *periph, void *arg)
221 {
222 	union ccb *request_ccb;	/* CCB representing the probe request */
223 	cam_status status;
224 	probe_softc *softc;
225 
226 	request_ccb = (union ccb *)arg;
227 	if (request_ccb == NULL) {
228 		printf("proberegister: no probe CCB, "
229 		       "can't register device\n");
230 		return(CAM_REQ_CMP_ERR);
231 	}
232 
233 	softc = (probe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT);
234 
235 	if (softc == NULL) {
236 		printf("proberegister: Unable to probe new device. "
237 		       "Unable to allocate softc\n");
238 		return(CAM_REQ_CMP_ERR);
239 	}
240 	TAILQ_INIT(&softc->request_ccbs);
241 	TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
242 			  periph_links.tqe);
243 	softc->flags = 0;
244 	periph->softc = softc;
245 	softc->periph = periph;
246 	softc->action = PROBE_INVALID;
247 	status = cam_periph_acquire(periph);
248 	if (status != CAM_REQ_CMP) {
249 		return (status);
250 	}
251 	CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n"));
252 	ata_device_transport(periph->path);
253 	probeschedule(periph);
254 	return(CAM_REQ_CMP);
255 }
256 
257 static void
258 probeschedule(struct cam_periph *periph)
259 {
260 	union ccb *ccb;
261 	probe_softc *softc;
262 
263 	softc = (probe_softc *)periph->softc;
264 	ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
265 
266 	if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) ||
267 	    periph->path->device->protocol == PROTO_SATAPM ||
268 	    periph->path->device->protocol == PROTO_SEMB)
269 		PROBE_SET_ACTION(softc, PROBE_RESET);
270 	else
271 		PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
272 
273 	if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE)
274 		softc->flags |= PROBE_NO_ANNOUNCE;
275 	else
276 		softc->flags &= ~PROBE_NO_ANNOUNCE;
277 
278 	xpt_schedule(periph, CAM_PRIORITY_XPT);
279 }
280 
281 static void
282 probestart(struct cam_periph *periph, union ccb *start_ccb)
283 {
284 	struct ccb_trans_settings cts;
285 	struct ccb_ataio *ataio;
286 	struct ccb_scsiio *csio;
287 	probe_softc *softc;
288 	struct cam_path *path;
289 	struct ata_params *ident_buf;
290 
291 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probestart\n"));
292 
293 	softc = (probe_softc *)periph->softc;
294 	path = start_ccb->ccb_h.path;
295 	ataio = &start_ccb->ataio;
296 	csio = &start_ccb->csio;
297 	ident_buf = &periph->path->device->ident_data;
298 
299 	if (softc->restart) {
300 		softc->restart = 0;
301 		if ((path->device->flags & CAM_DEV_UNCONFIGURED) ||
302 		    path->device->protocol == PROTO_SATAPM ||
303 		    path->device->protocol == PROTO_SEMB)
304 			softc->action = PROBE_RESET;
305 		else
306 			softc->action = PROBE_IDENTIFY;
307 	}
308 	switch (softc->action) {
309 	case PROBE_RESET:
310 		cam_fill_ataio(ataio,
311 		      0,
312 		      probedone,
313 		      /*flags*/CAM_DIR_NONE,
314 		      0,
315 		      /*data_ptr*/NULL,
316 		      /*dxfer_len*/0,
317 		      15 * 1000);
318 		ata_reset_cmd(ataio);
319 		break;
320 	case PROBE_IDENTIFY:
321 		cam_fill_ataio(ataio,
322 		      1,
323 		      probedone,
324 		      /*flags*/CAM_DIR_IN,
325 		      0,
326 		      /*data_ptr*/(u_int8_t *)&softc->ident_data,
327 		      /*dxfer_len*/sizeof(softc->ident_data),
328 		      30 * 1000);
329 		if (periph->path->device->protocol == PROTO_ATA)
330 			ata_28bit_cmd(ataio, ATA_ATA_IDENTIFY, 0, 0, 0);
331 		else
332 			ata_28bit_cmd(ataio, ATA_ATAPI_IDENTIFY, 0, 0, 0);
333 		break;
334 	case PROBE_SPINUP:
335 		if (bootverbose)
336 			xpt_print(path, "Spinning up device\n");
337 		cam_fill_ataio(ataio,
338 		      1,
339 		      probedone,
340 		      /*flags*/CAM_DIR_NONE | CAM_HIGH_POWER,
341 		      0,
342 		      /*data_ptr*/NULL,
343 		      /*dxfer_len*/0,
344 		      30 * 1000);
345 		ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_PUIS_SPINUP, 0, 0);
346 		break;
347 	case PROBE_SETMODE:
348 	{
349 		int mode, wantmode;
350 
351 		mode = 0;
352 		/* Fetch user modes from SIM. */
353 		bzero(&cts, sizeof(cts));
354 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
355 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
356 		cts.type = CTS_TYPE_USER_SETTINGS;
357 		xpt_action((union ccb *)&cts);
358 		if (path->device->transport == XPORT_ATA) {
359 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE)
360 				mode = cts.xport_specific.ata.mode;
361 		} else {
362 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_MODE)
363 				mode = cts.xport_specific.sata.mode;
364 		}
365 		if (periph->path->device->protocol == PROTO_ATA) {
366 			if (ata_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX))
367 				mode = ATA_PIO_MAX;
368 		} else {
369 			if (atapi_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX))
370 				mode = ATA_PIO_MAX;
371 		}
372 negotiate:
373 		/* Honor device capabilities. */
374 		wantmode = mode = ata_max_mode(ident_buf, mode);
375 		/* Report modes to SIM. */
376 		bzero(&cts, sizeof(cts));
377 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
378 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
379 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
380 		if (path->device->transport == XPORT_ATA) {
381 			cts.xport_specific.ata.mode = mode;
382 			cts.xport_specific.ata.valid = CTS_ATA_VALID_MODE;
383 		} else {
384 			cts.xport_specific.sata.mode = mode;
385 			cts.xport_specific.sata.valid = CTS_SATA_VALID_MODE;
386 		}
387 		xpt_action((union ccb *)&cts);
388 		/* Fetch current modes from SIM. */
389 		bzero(&cts, sizeof(cts));
390 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
391 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
392 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
393 		xpt_action((union ccb *)&cts);
394 		if (path->device->transport == XPORT_ATA) {
395 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE)
396 				mode = cts.xport_specific.ata.mode;
397 		} else {
398 			if (cts.xport_specific.ata.valid & CTS_SATA_VALID_MODE)
399 				mode = cts.xport_specific.sata.mode;
400 		}
401 		/* If SIM disagree - renegotiate. */
402 		if (mode != wantmode)
403 			goto negotiate;
404 		/* Remember what transport thinks about DMA. */
405 		if (mode < ATA_DMA)
406 			path->device->inq_flags &= ~SID_DMA;
407 		else
408 			path->device->inq_flags |= SID_DMA;
409 		xpt_async(AC_GETDEV_CHANGED, path, NULL);
410 		cam_fill_ataio(ataio,
411 		      1,
412 		      probedone,
413 		      /*flags*/CAM_DIR_NONE,
414 		      0,
415 		      /*data_ptr*/NULL,
416 		      /*dxfer_len*/0,
417 		      30 * 1000);
418 		ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_SETXFER, 0, mode);
419 		break;
420 	}
421 	case PROBE_SETPM:
422 		cam_fill_ataio(ataio,
423 		    1,
424 		    probedone,
425 		    CAM_DIR_NONE,
426 		    0,
427 		    NULL,
428 		    0,
429 		    30*1000);
430 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
431 		    (softc->caps & CTS_SATA_CAPS_H_PMREQ) ? 0x10 : 0x90,
432 		    0, 0x03);
433 		break;
434 	case PROBE_SETAPST:
435 		cam_fill_ataio(ataio,
436 		    1,
437 		    probedone,
438 		    CAM_DIR_NONE,
439 		    0,
440 		    NULL,
441 		    0,
442 		    30*1000);
443 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
444 		    (softc->caps & CTS_SATA_CAPS_H_APST) ? 0x10 : 0x90,
445 		    0, 0x07);
446 		break;
447 	case PROBE_SETDMAAA:
448 		cam_fill_ataio(ataio,
449 		    1,
450 		    probedone,
451 		    CAM_DIR_NONE,
452 		    0,
453 		    NULL,
454 		    0,
455 		    30*1000);
456 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
457 		    (softc->caps & CTS_SATA_CAPS_H_DMAAA) ? 0x10 : 0x90,
458 		    0, 0x02);
459 		break;
460 	case PROBE_SETAN:
461 		/* Remember what transport thinks about AEN. */
462 		if (softc->caps & CTS_SATA_CAPS_H_AN)
463 			path->device->inq_flags |= SID_AEN;
464 		else
465 			path->device->inq_flags &= ~SID_AEN;
466 		xpt_async(AC_GETDEV_CHANGED, path, NULL);
467 		cam_fill_ataio(ataio,
468 		    1,
469 		    probedone,
470 		    CAM_DIR_NONE,
471 		    0,
472 		    NULL,
473 		    0,
474 		    30*1000);
475 		ata_28bit_cmd(ataio, ATA_SETFEATURES,
476 		    (softc->caps & CTS_SATA_CAPS_H_AN) ? 0x10 : 0x90,
477 		    0, 0x05);
478 		break;
479 	case PROBE_SET_MULTI:
480 	{
481 		u_int sectors, bytecount;
482 
483 		bytecount = 8192;	/* SATA maximum */
484 		/* Fetch user bytecount from SIM. */
485 		bzero(&cts, sizeof(cts));
486 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
487 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
488 		cts.type = CTS_TYPE_USER_SETTINGS;
489 		xpt_action((union ccb *)&cts);
490 		if (path->device->transport == XPORT_ATA) {
491 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
492 				bytecount = cts.xport_specific.ata.bytecount;
493 		} else {
494 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
495 				bytecount = cts.xport_specific.sata.bytecount;
496 		}
497 		/* Honor device capabilities. */
498 		sectors = max(1, min(ident_buf->sectors_intr & 0xff,
499 		    bytecount / ata_logical_sector_size(ident_buf)));
500 		/* Report bytecount to SIM. */
501 		bzero(&cts, sizeof(cts));
502 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
503 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
504 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
505 		if (path->device->transport == XPORT_ATA) {
506 			cts.xport_specific.ata.bytecount = sectors *
507 			    ata_logical_sector_size(ident_buf);
508 			cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
509 		} else {
510 			cts.xport_specific.sata.bytecount = sectors *
511 			    ata_logical_sector_size(ident_buf);
512 			cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
513 		}
514 		xpt_action((union ccb *)&cts);
515 		/* Fetch current bytecount from SIM. */
516 		bzero(&cts, sizeof(cts));
517 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
518 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
519 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
520 		xpt_action((union ccb *)&cts);
521 		if (path->device->transport == XPORT_ATA) {
522 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
523 				bytecount = cts.xport_specific.ata.bytecount;
524 		} else {
525 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
526 				bytecount = cts.xport_specific.sata.bytecount;
527 		}
528 		sectors = bytecount / ata_logical_sector_size(ident_buf);
529 
530 		cam_fill_ataio(ataio,
531 		    1,
532 		    probedone,
533 		    CAM_DIR_NONE,
534 		    0,
535 		    NULL,
536 		    0,
537 		    30*1000);
538 		ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, sectors);
539 		break;
540 	}
541 	case PROBE_INQUIRY:
542 	{
543 		u_int bytecount;
544 
545 		bytecount = 8192;	/* SATA maximum */
546 		/* Fetch user bytecount from SIM. */
547 		bzero(&cts, sizeof(cts));
548 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
549 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
550 		cts.type = CTS_TYPE_USER_SETTINGS;
551 		xpt_action((union ccb *)&cts);
552 		if (path->device->transport == XPORT_ATA) {
553 			if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
554 				bytecount = cts.xport_specific.ata.bytecount;
555 		} else {
556 			if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
557 				bytecount = cts.xport_specific.sata.bytecount;
558 		}
559 		/* Honor device capabilities. */
560 		bytecount &= ~1;
561 		bytecount = max(2, min(65534, bytecount));
562 		if (ident_buf->satacapabilities != 0x0000 &&
563 		    ident_buf->satacapabilities != 0xffff) {
564 			bytecount = min(8192, bytecount);
565 		}
566 		/* Report bytecount to SIM. */
567 		bzero(&cts, sizeof(cts));
568 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
569 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
570 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
571 		if (path->device->transport == XPORT_ATA) {
572 			cts.xport_specific.ata.bytecount = bytecount;
573 			cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
574 		} else {
575 			cts.xport_specific.sata.bytecount = bytecount;
576 			cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
577 		}
578 		xpt_action((union ccb *)&cts);
579 		/* FALLTHROUGH */
580 	}
581 	case PROBE_FULL_INQUIRY:
582 	{
583 		u_int inquiry_len;
584 		struct scsi_inquiry_data *inq_buf =
585 		    &periph->path->device->inq_data;
586 
587 		if (softc->action == PROBE_INQUIRY)
588 			inquiry_len = SHORT_INQUIRY_LENGTH;
589 		else
590 			inquiry_len = SID_ADDITIONAL_LENGTH(inq_buf);
591 		/*
592 		 * Some parallel SCSI devices fail to send an
593 		 * ignore wide residue message when dealing with
594 		 * odd length inquiry requests.  Round up to be
595 		 * safe.
596 		 */
597 		inquiry_len = roundup2(inquiry_len, 2);
598 		scsi_inquiry(csio,
599 			     /*retries*/1,
600 			     probedone,
601 			     MSG_SIMPLE_Q_TAG,
602 			     (u_int8_t *)inq_buf,
603 			     inquiry_len,
604 			     /*evpd*/FALSE,
605 			     /*page_code*/0,
606 			     SSD_MIN_SIZE,
607 			     /*timeout*/60 * 1000);
608 		break;
609 	}
610 	case PROBE_PM_PID:
611 		cam_fill_ataio(ataio,
612 		      1,
613 		      probedone,
614 		      /*flags*/CAM_DIR_NONE,
615 		      0,
616 		      /*data_ptr*/NULL,
617 		      /*dxfer_len*/0,
618 		      10 * 1000);
619 		ata_pm_read_cmd(ataio, 0, 15);
620 		break;
621 	case PROBE_PM_PRV:
622 		cam_fill_ataio(ataio,
623 		      1,
624 		      probedone,
625 		      /*flags*/CAM_DIR_NONE,
626 		      0,
627 		      /*data_ptr*/NULL,
628 		      /*dxfer_len*/0,
629 		      10 * 1000);
630 		ata_pm_read_cmd(ataio, 1, 15);
631 		break;
632 	case PROBE_IDENTIFY_SES:
633 		cam_fill_ataio(ataio,
634 		      1,
635 		      probedone,
636 		      /*flags*/CAM_DIR_IN,
637 		      0,
638 		      /*data_ptr*/(u_int8_t *)&softc->ident_data,
639 		      /*dxfer_len*/sizeof(softc->ident_data),
640 		      30 * 1000);
641 		ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x02,
642 		    sizeof(softc->ident_data) / 4);
643 		break;
644 	case PROBE_IDENTIFY_SAFTE:
645 		cam_fill_ataio(ataio,
646 		      1,
647 		      probedone,
648 		      /*flags*/CAM_DIR_IN,
649 		      0,
650 		      /*data_ptr*/(u_int8_t *)&softc->ident_data,
651 		      /*dxfer_len*/sizeof(softc->ident_data),
652 		      30 * 1000);
653 		ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x00,
654 		    sizeof(softc->ident_data) / 4);
655 		break;
656 	default:
657 		panic("probestart: invalid action state 0x%x\n", softc->action);
658 	}
659 	start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
660 	xpt_action(start_ccb);
661 }
662 
663 static void
664 proberequestdefaultnegotiation(struct cam_periph *periph)
665 {
666 	struct ccb_trans_settings cts;
667 
668 	xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE);
669 	cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
670 	cts.type = CTS_TYPE_USER_SETTINGS;
671 	xpt_action((union ccb *)&cts);
672 	if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
673 		return;
674 	cts.xport_specific.valid = 0;
675 	cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
676 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
677 	xpt_action((union ccb *)&cts);
678 }
679 
680 static void
681 probedone(struct cam_periph *periph, union ccb *done_ccb)
682 {
683 	struct ccb_trans_settings cts;
684 	struct ata_params *ident_buf;
685 	struct scsi_inquiry_data *inq_buf;
686 	probe_softc *softc;
687 	struct cam_path *path;
688 	cam_status status;
689 	u_int32_t  priority;
690 	u_int caps;
691 	int changed = 1, found = 1;
692 	static const uint8_t fake_device_id_hdr[8] =
693 	    {0, SVPD_DEVICE_ID, 0, 12,
694 	     SVPD_ID_CODESET_BINARY, SVPD_ID_TYPE_NAA, 0, 8};
695 
696 	CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probedone\n"));
697 
698 	softc = (probe_softc *)periph->softc;
699 	path = done_ccb->ccb_h.path;
700 	priority = done_ccb->ccb_h.pinfo.priority;
701 	ident_buf = &path->device->ident_data;
702 	inq_buf = &path->device->inq_data;
703 
704 	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
705 		if (cam_periph_error(done_ccb,
706 		    0, softc->restart ? (SF_NO_RECOVERY | SF_NO_RETRY) : 0,
707 		    NULL) == ERESTART) {
708 out:
709 			/* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
710 			cam_release_devq(path, 0, 0, 0, FALSE);
711 			return;
712 		}
713 		if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
714 			/* Don't wedge the queue */
715 			xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
716 		}
717 		status = done_ccb->ccb_h.status & CAM_STATUS_MASK;
718 		if (softc->restart) {
719 			softc->faults++;
720 			if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) ==
721 			    CAM_CMD_TIMEOUT)
722 				softc->faults += 4;
723 			if (softc->faults < 10)
724 				goto done;
725 			else
726 				softc->restart = 0;
727 
728 		/* Old PIO2 devices may not support mode setting. */
729 		} else if (softc->action == PROBE_SETMODE &&
730 		    status == CAM_ATA_STATUS_ERROR &&
731 		    ata_max_pmode(ident_buf) <= ATA_PIO2 &&
732 		    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0) {
733 			goto noerror;
734 
735 		/*
736 		 * Some old WD SATA disks report supported and enabled
737 		 * device-initiated interface power management, but return
738 		 * ABORT on attempt to disable it.
739 		 */
740 		} else if (softc->action == PROBE_SETPM &&
741 		    status == CAM_ATA_STATUS_ERROR) {
742 			goto noerror;
743 
744 		/*
745 		 * Some HP SATA disks report supported DMA Auto-Activation,
746 		 * but return ABORT on attempt to enable it.
747 		 */
748 		} else if (softc->action == PROBE_SETDMAAA &&
749 		    status == CAM_ATA_STATUS_ERROR) {
750 			goto noerror;
751 
752 		/*
753 		 * SES and SAF-TE SEPs have different IDENTIFY commands,
754 		 * but SATA specification doesn't tell how to identify them.
755 		 * Until better way found, just try another if first fail.
756 		 */
757 		} else if (softc->action == PROBE_IDENTIFY_SES &&
758 		    status == CAM_ATA_STATUS_ERROR) {
759 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SAFTE);
760 			xpt_release_ccb(done_ccb);
761 			xpt_schedule(periph, priority);
762 			goto out;
763 		}
764 
765 		/*
766 		 * If we get to this point, we got an error status back
767 		 * from the inquiry and the error status doesn't require
768 		 * automatically retrying the command.  Therefore, the
769 		 * inquiry failed.  If we had inquiry information before
770 		 * for this device, but this latest inquiry command failed,
771 		 * the device has probably gone away.  If this device isn't
772 		 * already marked unconfigured, notify the peripheral
773 		 * drivers that this device is no more.
774 		 */
775 device_fail:	if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
776 			xpt_async(AC_LOST_DEVICE, path, NULL);
777 		PROBE_SET_ACTION(softc, PROBE_INVALID);
778 		found = 0;
779 		goto done;
780 	}
781 noerror:
782 	if (softc->restart)
783 		goto done;
784 	switch (softc->action) {
785 	case PROBE_RESET:
786 	{
787 		int sign = (done_ccb->ataio.res.lba_high << 8) +
788 		    done_ccb->ataio.res.lba_mid;
789 		CAM_DEBUG(path, CAM_DEBUG_PROBE,
790 		    ("SIGNATURE: %04x\n", sign));
791 		if (sign == 0x0000 &&
792 		    done_ccb->ccb_h.target_id != 15) {
793 			path->device->protocol = PROTO_ATA;
794 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
795 		} else if (sign == 0x9669 &&
796 		    done_ccb->ccb_h.target_id == 15) {
797 			/* Report SIM that PM is present. */
798 			bzero(&cts, sizeof(cts));
799 			xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
800 			cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
801 			cts.type = CTS_TYPE_CURRENT_SETTINGS;
802 			cts.xport_specific.sata.pm_present = 1;
803 			cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
804 			xpt_action((union ccb *)&cts);
805 			path->device->protocol = PROTO_SATAPM;
806 			PROBE_SET_ACTION(softc, PROBE_PM_PID);
807 		} else if (sign == 0xc33c &&
808 		    done_ccb->ccb_h.target_id != 15) {
809 			path->device->protocol = PROTO_SEMB;
810 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SES);
811 		} else if (sign == 0xeb14 &&
812 		    done_ccb->ccb_h.target_id != 15) {
813 			path->device->protocol = PROTO_SCSI;
814 			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
815 		} else {
816 			if (done_ccb->ccb_h.target_id != 15) {
817 				xpt_print(path,
818 				    "Unexpected signature 0x%04x\n", sign);
819 			}
820 			goto device_fail;
821 		}
822 		xpt_release_ccb(done_ccb);
823 		xpt_schedule(periph, priority);
824 		goto out;
825 	}
826 	case PROBE_IDENTIFY:
827 	{
828 		struct ccb_pathinq cpi;
829 		int16_t *ptr;
830 
831 		ident_buf = &softc->ident_data;
832 		for (ptr = (int16_t *)ident_buf;
833 		     ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) {
834 			*ptr = le16toh(*ptr);
835 		}
836 		if (strncmp(ident_buf->model, "FX", 2) &&
837 		    strncmp(ident_buf->model, "NEC", 3) &&
838 		    strncmp(ident_buf->model, "Pioneer", 7) &&
839 		    strncmp(ident_buf->model, "SHARP", 5)) {
840 			ata_bswap(ident_buf->model, sizeof(ident_buf->model));
841 			ata_bswap(ident_buf->revision, sizeof(ident_buf->revision));
842 			ata_bswap(ident_buf->serial, sizeof(ident_buf->serial));
843 		}
844 		ata_btrim(ident_buf->model, sizeof(ident_buf->model));
845 		ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model));
846 		ata_btrim(ident_buf->revision, sizeof(ident_buf->revision));
847 		ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision));
848 		ata_btrim(ident_buf->serial, sizeof(ident_buf->serial));
849 		ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial));
850 		/* Device may need spin-up before IDENTIFY become valid. */
851 		if ((ident_buf->specconf == 0x37c8 ||
852 		     ident_buf->specconf == 0x738c) &&
853 		    ((ident_buf->config & ATA_RESP_INCOMPLETE) ||
854 		     softc->spinup == 0)) {
855 			PROBE_SET_ACTION(softc, PROBE_SPINUP);
856 			xpt_release_ccb(done_ccb);
857 			xpt_schedule(periph, priority);
858 			goto out;
859 		}
860 		ident_buf = &path->device->ident_data;
861 		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
862 			/* Check that it is the same device. */
863 			if (bcmp(softc->ident_data.model, ident_buf->model,
864 			     sizeof(ident_buf->model)) ||
865 			    bcmp(softc->ident_data.revision, ident_buf->revision,
866 			     sizeof(ident_buf->revision)) ||
867 			    bcmp(softc->ident_data.serial, ident_buf->serial,
868 			     sizeof(ident_buf->serial))) {
869 				/* Device changed. */
870 				xpt_async(AC_LOST_DEVICE, path, NULL);
871 			} else {
872 				bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
873 				changed = 0;
874 			}
875 		}
876 		if (changed) {
877 			bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
878 			/* Clean up from previous instance of this device */
879 			if (path->device->serial_num != NULL) {
880 				free(path->device->serial_num, M_CAMXPT);
881 				path->device->serial_num = NULL;
882 				path->device->serial_num_len = 0;
883 			}
884 			if (path->device->device_id != NULL) {
885 				free(path->device->device_id, M_CAMXPT);
886 				path->device->device_id = NULL;
887 				path->device->device_id_len = 0;
888 			}
889 			path->device->serial_num =
890 				(u_int8_t *)malloc((sizeof(ident_buf->serial) + 1),
891 					   M_CAMXPT, M_NOWAIT);
892 			if (path->device->serial_num != NULL) {
893 				bcopy(ident_buf->serial,
894 				      path->device->serial_num,
895 				      sizeof(ident_buf->serial));
896 				path->device->serial_num[sizeof(ident_buf->serial)]
897 				    = '\0';
898 				path->device->serial_num_len =
899 				    strlen(path->device->serial_num);
900 			}
901 			if (ident_buf->enabled.extension &
902 			    ATA_SUPPORT_64BITWWN) {
903 				path->device->device_id =
904 				    malloc(16, M_CAMXPT, M_NOWAIT);
905 				if (path->device->device_id != NULL) {
906 					path->device->device_id_len = 16;
907 					bcopy(&fake_device_id_hdr,
908 					    path->device->device_id, 8);
909 					bcopy(ident_buf->wwn,
910 					    path->device->device_id + 8, 8);
911 					ata_bswap(path->device->device_id + 8, 8);
912 				}
913 			}
914 
915 			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
916 			xpt_async(AC_GETDEV_CHANGED, path, NULL);
917 		}
918 		if (ident_buf->satacapabilities & ATA_SUPPORT_NCQ) {
919 			path->device->mintags = 2;
920 			path->device->maxtags =
921 			    ATA_QUEUE_LEN(ident_buf->queue) + 1;
922 		}
923 		ata_find_quirk(path->device);
924 		if (path->device->mintags != 0 &&
925 		    path->bus->sim->max_tagged_dev_openings != 0) {
926 			/* Check if the SIM does not want queued commands. */
927 			bzero(&cpi, sizeof(cpi));
928 			xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
929 			cpi.ccb_h.func_code = XPT_PATH_INQ;
930 			xpt_action((union ccb *)&cpi);
931 			if (cpi.ccb_h.status == CAM_REQ_CMP &&
932 			    (cpi.hba_inquiry & PI_TAG_ABLE)) {
933 				/* Report SIM which tags are allowed. */
934 				bzero(&cts, sizeof(cts));
935 				xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
936 				cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
937 				cts.type = CTS_TYPE_CURRENT_SETTINGS;
938 				cts.xport_specific.sata.tags = path->device->maxtags;
939 				cts.xport_specific.sata.valid = CTS_SATA_VALID_TAGS;
940 				xpt_action((union ccb *)&cts);
941 			}
942 		}
943 		ata_device_transport(path);
944 		if (changed)
945 			proberequestdefaultnegotiation(periph);
946 		PROBE_SET_ACTION(softc, PROBE_SETMODE);
947 		xpt_release_ccb(done_ccb);
948 		xpt_schedule(periph, priority);
949 		goto out;
950 	}
951 	case PROBE_SPINUP:
952 		if (bootverbose)
953 			xpt_print(path, "Spin-up done\n");
954 		softc->spinup = 1;
955 		PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
956 		xpt_release_ccb(done_ccb);
957 		xpt_schedule(periph, priority);
958 		goto out;
959 	case PROBE_SETMODE:
960 		/* Set supported bits. */
961 		bzero(&cts, sizeof(cts));
962 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
963 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
964 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
965 		xpt_action((union ccb *)&cts);
966 		if (path->device->transport == XPORT_SATA &&
967 		    cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
968 			caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H;
969 		else if (path->device->transport == XPORT_ATA &&
970 		    cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS)
971 			caps = cts.xport_specific.ata.caps & CTS_ATA_CAPS_H;
972 		else
973 			caps = 0;
974 		if (path->device->transport == XPORT_SATA &&
975 		    ident_buf->satacapabilities != 0xffff) {
976 			if (ident_buf->satacapabilities & ATA_SUPPORT_IFPWRMNGTRCV)
977 				caps |= CTS_SATA_CAPS_D_PMREQ;
978 			if (ident_buf->satacapabilities & ATA_SUPPORT_HAPST)
979 				caps |= CTS_SATA_CAPS_D_APST;
980 		}
981 		/* Mask unwanted bits. */
982 		bzero(&cts, sizeof(cts));
983 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
984 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
985 		cts.type = CTS_TYPE_USER_SETTINGS;
986 		xpt_action((union ccb *)&cts);
987 		if (path->device->transport == XPORT_SATA &&
988 		    cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
989 			caps &= cts.xport_specific.sata.caps;
990 		else if (path->device->transport == XPORT_ATA &&
991 		    cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS)
992 			caps &= cts.xport_specific.ata.caps;
993 		else
994 			caps = 0;
995 		/*
996 		 * Remember what transport thinks about 48-bit DMA.  If
997 		 * capability information is not provided or transport is
998 		 * SATA, we take support for granted.
999 		 */
1000 		if (!(path->device->inq_flags & SID_DMA) ||
1001 		    (path->device->transport == XPORT_ATA &&
1002 		    (cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS) &&
1003 		    !(caps & CTS_ATA_CAPS_H_DMA48)))
1004 			path->device->inq_flags &= ~SID_DMA48;
1005 		else
1006 			path->device->inq_flags |= SID_DMA48;
1007 		/* Store result to SIM. */
1008 		bzero(&cts, sizeof(cts));
1009 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1010 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1011 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
1012 		if (path->device->transport == XPORT_SATA) {
1013 			cts.xport_specific.sata.caps = caps;
1014 			cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS;
1015 		} else {
1016 			cts.xport_specific.ata.caps = caps;
1017 			cts.xport_specific.ata.valid = CTS_ATA_VALID_CAPS;
1018 		}
1019 		xpt_action((union ccb *)&cts);
1020 		softc->caps = caps;
1021 		if (path->device->transport != XPORT_SATA)
1022 			goto notsata;
1023 		if ((ident_buf->satasupport & ATA_SUPPORT_IFPWRMNGT) &&
1024 		    (!(softc->caps & CTS_SATA_CAPS_H_PMREQ)) !=
1025 		    (!(ident_buf->sataenabled & ATA_SUPPORT_IFPWRMNGT))) {
1026 			PROBE_SET_ACTION(softc, PROBE_SETPM);
1027 			xpt_release_ccb(done_ccb);
1028 			xpt_schedule(periph, priority);
1029 			goto out;
1030 		}
1031 		/* FALLTHROUGH */
1032 	case PROBE_SETPM:
1033 		if (ident_buf->satacapabilities != 0xffff &&
1034 		    (ident_buf->satacapabilities & ATA_SUPPORT_DAPST) &&
1035 		    (!(softc->caps & CTS_SATA_CAPS_H_APST)) !=
1036 		    (!(ident_buf->sataenabled & ATA_ENABLED_DAPST))) {
1037 			PROBE_SET_ACTION(softc, PROBE_SETAPST);
1038 			xpt_release_ccb(done_ccb);
1039 			xpt_schedule(periph, priority);
1040 			goto out;
1041 		}
1042 		/* FALLTHROUGH */
1043 	case PROBE_SETAPST:
1044 		if ((ident_buf->satasupport & ATA_SUPPORT_AUTOACTIVATE) &&
1045 		    (!(softc->caps & CTS_SATA_CAPS_H_DMAAA)) !=
1046 		    (!(ident_buf->sataenabled & ATA_SUPPORT_AUTOACTIVATE))) {
1047 			PROBE_SET_ACTION(softc, PROBE_SETDMAAA);
1048 			xpt_release_ccb(done_ccb);
1049 			xpt_schedule(periph, priority);
1050 			goto out;
1051 		}
1052 		/* FALLTHROUGH */
1053 	case PROBE_SETDMAAA:
1054 		if (path->device->protocol != PROTO_ATA &&
1055 		    (ident_buf->satasupport & ATA_SUPPORT_ASYNCNOTIF) &&
1056 		    (!(softc->caps & CTS_SATA_CAPS_H_AN)) !=
1057 		    (!(ident_buf->sataenabled & ATA_SUPPORT_ASYNCNOTIF))) {
1058 			PROBE_SET_ACTION(softc, PROBE_SETAN);
1059 			xpt_release_ccb(done_ccb);
1060 			xpt_schedule(periph, priority);
1061 			goto out;
1062 		}
1063 		/* FALLTHROUGH */
1064 	case PROBE_SETAN:
1065 notsata:
1066 		if (path->device->protocol == PROTO_ATA) {
1067 			PROBE_SET_ACTION(softc, PROBE_SET_MULTI);
1068 		} else {
1069 			PROBE_SET_ACTION(softc, PROBE_INQUIRY);
1070 		}
1071 		xpt_release_ccb(done_ccb);
1072 		xpt_schedule(periph, priority);
1073 		goto out;
1074 	case PROBE_SET_MULTI:
1075 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1076 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1077 			xpt_acquire_device(path->device);
1078 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1079 			xpt_action(done_ccb);
1080 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1081 		}
1082 		PROBE_SET_ACTION(softc, PROBE_DONE);
1083 		break;
1084 	case PROBE_INQUIRY:
1085 	case PROBE_FULL_INQUIRY:
1086 	{
1087 		u_int8_t periph_qual, len;
1088 
1089 		path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
1090 
1091 		periph_qual = SID_QUAL(inq_buf);
1092 
1093 		if (periph_qual != SID_QUAL_LU_CONNECTED &&
1094 		    periph_qual != SID_QUAL_LU_OFFLINE)
1095 			break;
1096 
1097 		/*
1098 		 * We conservatively request only
1099 		 * SHORT_INQUIRY_LEN bytes of inquiry
1100 		 * information during our first try
1101 		 * at sending an INQUIRY. If the device
1102 		 * has more information to give,
1103 		 * perform a second request specifying
1104 		 * the amount of information the device
1105 		 * is willing to give.
1106 		 */
1107 		len = inq_buf->additional_length
1108 		    + offsetof(struct scsi_inquiry_data, additional_length) + 1;
1109 		if (softc->action == PROBE_INQUIRY
1110 		    && len > SHORT_INQUIRY_LENGTH) {
1111 			PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
1112 			xpt_release_ccb(done_ccb);
1113 			xpt_schedule(periph, priority);
1114 			goto out;
1115 		}
1116 
1117 		ata_device_transport(path);
1118 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1119 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1120 			xpt_acquire_device(path->device);
1121 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1122 			xpt_action(done_ccb);
1123 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1124 		}
1125 		PROBE_SET_ACTION(softc, PROBE_DONE);
1126 		break;
1127 	}
1128 	case PROBE_PM_PID:
1129 		if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
1130 			bzero(ident_buf, sizeof(*ident_buf));
1131 		softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
1132 		    (done_ccb->ataio.res.lba_mid << 16) +
1133 		    (done_ccb->ataio.res.lba_low << 8) +
1134 		    done_ccb->ataio.res.sector_count;
1135 		((uint32_t *)ident_buf)[0] = softc->pm_pid;
1136 		snprintf(ident_buf->model, sizeof(ident_buf->model),
1137 		    "Port Multiplier %08x", softc->pm_pid);
1138 		PROBE_SET_ACTION(softc, PROBE_PM_PRV);
1139 		xpt_release_ccb(done_ccb);
1140 		xpt_schedule(periph, priority);
1141 		goto out;
1142 	case PROBE_PM_PRV:
1143 		softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
1144 		    (done_ccb->ataio.res.lba_mid << 16) +
1145 		    (done_ccb->ataio.res.lba_low << 8) +
1146 		    done_ccb->ataio.res.sector_count;
1147 		((uint32_t *)ident_buf)[1] = softc->pm_prv;
1148 		snprintf(ident_buf->revision, sizeof(ident_buf->revision),
1149 		    "%04x", softc->pm_prv);
1150 		path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
1151 		ata_device_transport(path);
1152 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED)
1153 			proberequestdefaultnegotiation(periph);
1154 		/* Set supported bits. */
1155 		bzero(&cts, sizeof(cts));
1156 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1157 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
1158 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
1159 		xpt_action((union ccb *)&cts);
1160 		if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
1161 			caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H;
1162 		else
1163 			caps = 0;
1164 		/* All PMPs must support PM requests. */
1165 		caps |= CTS_SATA_CAPS_D_PMREQ;
1166 		/* Mask unwanted bits. */
1167 		bzero(&cts, sizeof(cts));
1168 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1169 		cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
1170 		cts.type = CTS_TYPE_USER_SETTINGS;
1171 		xpt_action((union ccb *)&cts);
1172 		if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
1173 			caps &= cts.xport_specific.sata.caps;
1174 		else
1175 			caps = 0;
1176 		/* Remember what transport thinks about AEN. */
1177 		if ((caps & CTS_SATA_CAPS_H_AN) && path->device->protocol != PROTO_ATA)
1178 			path->device->inq_flags |= SID_AEN;
1179 		else
1180 			path->device->inq_flags &= ~SID_AEN;
1181 		/* Store result to SIM. */
1182 		bzero(&cts, sizeof(cts));
1183 		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1184 		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1185 		cts.type = CTS_TYPE_CURRENT_SETTINGS;
1186 		cts.xport_specific.sata.caps = caps;
1187 		cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS;
1188 		xpt_action((union ccb *)&cts);
1189 		softc->caps = caps;
1190 		xpt_async(AC_GETDEV_CHANGED, path, NULL);
1191 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1192 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1193 			xpt_acquire_device(path->device);
1194 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1195 			xpt_action(done_ccb);
1196 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1197 		} else {
1198 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1199 			xpt_action(done_ccb);
1200 			xpt_async(AC_SCSI_AEN, path, done_ccb);
1201 		}
1202 		PROBE_SET_ACTION(softc, PROBE_DONE);
1203 		break;
1204 	case PROBE_IDENTIFY_SES:
1205 	case PROBE_IDENTIFY_SAFTE:
1206 		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
1207 			/* Check that it is the same device. */
1208 			if (bcmp(&softc->ident_data, ident_buf, 53)) {
1209 				/* Device changed. */
1210 				xpt_async(AC_LOST_DEVICE, path, NULL);
1211 			} else {
1212 				bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
1213 				changed = 0;
1214 			}
1215 		}
1216 		if (changed) {
1217 			bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
1218 			/* Clean up from previous instance of this device */
1219 			if (path->device->device_id != NULL) {
1220 				free(path->device->device_id, M_CAMXPT);
1221 				path->device->device_id = NULL;
1222 				path->device->device_id_len = 0;
1223 			}
1224 			path->device->device_id =
1225 			    malloc(16, M_CAMXPT, M_NOWAIT);
1226 			if (path->device->device_id != NULL) {
1227 				path->device->device_id_len = 16;
1228 				bcopy(&fake_device_id_hdr,
1229 				    path->device->device_id, 8);
1230 				bcopy(((uint8_t*)ident_buf) + 2,
1231 				    path->device->device_id + 8, 8);
1232 			}
1233 
1234 			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
1235 		}
1236 		ata_device_transport(path);
1237 		if (changed)
1238 			proberequestdefaultnegotiation(periph);
1239 
1240 		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
1241 			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
1242 			xpt_acquire_device(path->device);
1243 			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1244 			xpt_action(done_ccb);
1245 			xpt_async(AC_FOUND_DEVICE, path, done_ccb);
1246 		}
1247 		PROBE_SET_ACTION(softc, PROBE_DONE);
1248 		break;
1249 	default:
1250 		panic("probedone: invalid action state 0x%x\n", softc->action);
1251 	}
1252 done:
1253 	if (softc->restart) {
1254 		softc->restart = 0;
1255 		xpt_release_ccb(done_ccb);
1256 		probeschedule(periph);
1257 		goto out;
1258 	}
1259 	xpt_release_ccb(done_ccb);
1260 	CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n"));
1261 	while ((done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) {
1262 		TAILQ_REMOVE(&softc->request_ccbs,
1263 		    &done_ccb->ccb_h, periph_links.tqe);
1264 		done_ccb->ccb_h.status = found ? CAM_REQ_CMP : CAM_REQ_CMP_ERR;
1265 		xpt_done(done_ccb);
1266 	}
1267 	/* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
1268 	cam_release_devq(path, 0, 0, 0, FALSE);
1269 	cam_periph_invalidate(periph);
1270 	cam_periph_release_locked(periph);
1271 }
1272 
1273 static void
1274 probecleanup(struct cam_periph *periph)
1275 {
1276 	free(periph->softc, M_CAMXPT);
1277 }
1278 
1279 static void
1280 ata_find_quirk(struct cam_ed *device)
1281 {
1282 	struct ata_quirk_entry *quirk;
1283 	caddr_t	match;
1284 
1285 	match = cam_quirkmatch((caddr_t)&device->ident_data,
1286 			       (caddr_t)ata_quirk_table,
1287 			       ata_quirk_table_size,
1288 			       sizeof(*ata_quirk_table), ata_identify_match);
1289 
1290 	if (match == NULL)
1291 		panic("xpt_find_quirk: device didn't match wildcard entry!!");
1292 
1293 	quirk = (struct ata_quirk_entry *)match;
1294 	device->quirk = quirk;
1295 	if (quirk->quirks & CAM_QUIRK_MAXTAGS) {
1296 		device->mintags = quirk->mintags;
1297 		device->maxtags = quirk->maxtags;
1298 	}
1299 }
1300 
1301 typedef struct {
1302 	union	ccb *request_ccb;
1303 	struct 	ccb_pathinq *cpi;
1304 	int	counter;
1305 } ata_scan_bus_info;
1306 
1307 /*
1308  * To start a scan, request_ccb is an XPT_SCAN_BUS ccb.
1309  * As the scan progresses, xpt_scan_bus is used as the
1310  * callback on completion function.
1311  */
1312 static void
1313 ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
1314 {
1315 	struct	cam_path *path;
1316 	ata_scan_bus_info *scan_info;
1317 	union	ccb *work_ccb, *reset_ccb;
1318 	struct mtx *mtx;
1319 	cam_status status;
1320 
1321 	CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE,
1322 		  ("xpt_scan_bus\n"));
1323 	switch (request_ccb->ccb_h.func_code) {
1324 	case XPT_SCAN_BUS:
1325 	case XPT_SCAN_TGT:
1326 		/* Find out the characteristics of the bus */
1327 		work_ccb = xpt_alloc_ccb_nowait();
1328 		if (work_ccb == NULL) {
1329 			request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1330 			xpt_done(request_ccb);
1331 			return;
1332 		}
1333 		xpt_setup_ccb(&work_ccb->ccb_h, request_ccb->ccb_h.path,
1334 			      request_ccb->ccb_h.pinfo.priority);
1335 		work_ccb->ccb_h.func_code = XPT_PATH_INQ;
1336 		xpt_action(work_ccb);
1337 		if (work_ccb->ccb_h.status != CAM_REQ_CMP) {
1338 			request_ccb->ccb_h.status = work_ccb->ccb_h.status;
1339 			xpt_free_ccb(work_ccb);
1340 			xpt_done(request_ccb);
1341 			return;
1342 		}
1343 
1344 		/* We may need to reset bus first, if we haven't done it yet. */
1345 		if ((work_ccb->cpi.hba_inquiry &
1346 		    (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE)) &&
1347 		    !(work_ccb->cpi.hba_misc & PIM_NOBUSRESET) &&
1348 		    !timevalisset(&request_ccb->ccb_h.path->bus->last_reset)) {
1349 			reset_ccb = xpt_alloc_ccb_nowait();
1350 			if (reset_ccb == NULL) {
1351 				request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1352 				xpt_free_ccb(work_ccb);
1353 				xpt_done(request_ccb);
1354 				return;
1355 			}
1356 			xpt_setup_ccb(&reset_ccb->ccb_h, request_ccb->ccb_h.path,
1357 			      CAM_PRIORITY_NONE);
1358 			reset_ccb->ccb_h.func_code = XPT_RESET_BUS;
1359 			xpt_action(reset_ccb);
1360 			if (reset_ccb->ccb_h.status != CAM_REQ_CMP) {
1361 				request_ccb->ccb_h.status = reset_ccb->ccb_h.status;
1362 				xpt_free_ccb(reset_ccb);
1363 				xpt_free_ccb(work_ccb);
1364 				xpt_done(request_ccb);
1365 				return;
1366 			}
1367 			xpt_free_ccb(reset_ccb);
1368 		}
1369 
1370 		/* Save some state for use while we probe for devices */
1371 		scan_info = (ata_scan_bus_info *)
1372 		    malloc(sizeof(ata_scan_bus_info), M_CAMXPT, M_NOWAIT);
1373 		if (scan_info == NULL) {
1374 			request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1375 			xpt_free_ccb(work_ccb);
1376 			xpt_done(request_ccb);
1377 			return;
1378 		}
1379 		scan_info->request_ccb = request_ccb;
1380 		scan_info->cpi = &work_ccb->cpi;
1381 		/* If PM supported, probe it first. */
1382 		if (scan_info->cpi->hba_inquiry & PI_SATAPM)
1383 			scan_info->counter = scan_info->cpi->max_target;
1384 		else
1385 			scan_info->counter = 0;
1386 
1387 		work_ccb = xpt_alloc_ccb_nowait();
1388 		if (work_ccb == NULL) {
1389 			free(scan_info, M_CAMXPT);
1390 			request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1391 			xpt_done(request_ccb);
1392 			break;
1393 		}
1394 		mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path);
1395 		goto scan_next;
1396 	case XPT_SCAN_LUN:
1397 		work_ccb = request_ccb;
1398 		/* Reuse the same CCB to query if a device was really found */
1399 		scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0;
1400 		mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path);
1401 		mtx_lock(mtx);
1402 		/* If there is PMP... */
1403 		if ((scan_info->cpi->hba_inquiry & PI_SATAPM) &&
1404 		    (scan_info->counter == scan_info->cpi->max_target)) {
1405 			if (work_ccb->ccb_h.status == CAM_REQ_CMP) {
1406 				/* everything else will be probed by it */
1407 				/* Free the current request path- we're done with it. */
1408 				xpt_free_path(work_ccb->ccb_h.path);
1409 				goto done;
1410 			} else {
1411 				struct ccb_trans_settings cts;
1412 
1413 				/* Report SIM that PM is absent. */
1414 				bzero(&cts, sizeof(cts));
1415 				xpt_setup_ccb(&cts.ccb_h,
1416 				    work_ccb->ccb_h.path, CAM_PRIORITY_NONE);
1417 				cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1418 				cts.type = CTS_TYPE_CURRENT_SETTINGS;
1419 				cts.xport_specific.sata.pm_present = 0;
1420 				cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
1421 				xpt_action((union ccb *)&cts);
1422 			}
1423 		}
1424 		/* Free the current request path- we're done with it. */
1425 		xpt_free_path(work_ccb->ccb_h.path);
1426 		if (scan_info->counter ==
1427 		    ((scan_info->cpi->hba_inquiry & PI_SATAPM) ?
1428 		    0 : scan_info->cpi->max_target)) {
1429 done:
1430 			mtx_unlock(mtx);
1431 			xpt_free_ccb(work_ccb);
1432 			xpt_free_ccb((union ccb *)scan_info->cpi);
1433 			request_ccb = scan_info->request_ccb;
1434 			free(scan_info, M_CAMXPT);
1435 			request_ccb->ccb_h.status = CAM_REQ_CMP;
1436 			xpt_done(request_ccb);
1437 			break;
1438 		}
1439 		/* Take next device. Wrap from max (PMP) to 0. */
1440 		scan_info->counter = (scan_info->counter + 1 ) %
1441 		    (scan_info->cpi->max_target + 1);
1442 scan_next:
1443 		status = xpt_create_path(&path, NULL,
1444 		    scan_info->request_ccb->ccb_h.path_id,
1445 		    scan_info->counter, 0);
1446 		if (status != CAM_REQ_CMP) {
1447 			if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN)
1448 				mtx_unlock(mtx);
1449 			printf("xpt_scan_bus: xpt_create_path failed"
1450 			    " with status %#x, bus scan halted\n",
1451 			    status);
1452 			xpt_free_ccb(work_ccb);
1453 			xpt_free_ccb((union ccb *)scan_info->cpi);
1454 			request_ccb = scan_info->request_ccb;
1455 			free(scan_info, M_CAMXPT);
1456 			request_ccb->ccb_h.status = status;
1457 			xpt_done(request_ccb);
1458 			break;
1459 		}
1460 		xpt_setup_ccb(&work_ccb->ccb_h, path,
1461 		    scan_info->request_ccb->ccb_h.pinfo.priority);
1462 		work_ccb->ccb_h.func_code = XPT_SCAN_LUN;
1463 		work_ccb->ccb_h.cbfcnp = ata_scan_bus;
1464 		work_ccb->ccb_h.flags |= CAM_UNLOCKED;
1465 		work_ccb->ccb_h.ppriv_ptr0 = scan_info;
1466 		work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags;
1467 		mtx_unlock(mtx);
1468 		if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN)
1469 			mtx = NULL;
1470 		xpt_action(work_ccb);
1471 		if (mtx != NULL)
1472 			mtx_lock(mtx);
1473 		break;
1474 	default:
1475 		break;
1476 	}
1477 }
1478 
1479 static void
1480 ata_scan_lun(struct cam_periph *periph, struct cam_path *path,
1481 	     cam_flags flags, union ccb *request_ccb)
1482 {
1483 	struct ccb_pathinq cpi;
1484 	cam_status status;
1485 	struct cam_path *new_path;
1486 	struct cam_periph *old_periph;
1487 	int lock;
1488 
1489 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n"));
1490 
1491 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
1492 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1493 	xpt_action((union ccb *)&cpi);
1494 
1495 	if (cpi.ccb_h.status != CAM_REQ_CMP) {
1496 		if (request_ccb != NULL) {
1497 			request_ccb->ccb_h.status = cpi.ccb_h.status;
1498 			xpt_done(request_ccb);
1499 		}
1500 		return;
1501 	}
1502 
1503 	if (request_ccb == NULL) {
1504 		request_ccb = xpt_alloc_ccb_nowait();
1505 		if (request_ccb == NULL) {
1506 			xpt_print(path, "xpt_scan_lun: can't allocate CCB, "
1507 			    "can't continue\n");
1508 			return;
1509 		}
1510 		status = xpt_create_path(&new_path, NULL,
1511 					  path->bus->path_id,
1512 					  path->target->target_id,
1513 					  path->device->lun_id);
1514 		if (status != CAM_REQ_CMP) {
1515 			xpt_print(path, "xpt_scan_lun: can't create path, "
1516 			    "can't continue\n");
1517 			xpt_free_ccb(request_ccb);
1518 			return;
1519 		}
1520 		xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT);
1521 		request_ccb->ccb_h.cbfcnp = xptscandone;
1522 		request_ccb->ccb_h.flags |= CAM_UNLOCKED;
1523 		request_ccb->ccb_h.func_code = XPT_SCAN_LUN;
1524 		request_ccb->crcn.flags = flags;
1525 	}
1526 
1527 	lock = (xpt_path_owned(path) == 0);
1528 	if (lock)
1529 		xpt_path_lock(path);
1530 	if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) {
1531 		if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) {
1532 			probe_softc *softc;
1533 
1534 			softc = (probe_softc *)old_periph->softc;
1535 			TAILQ_INSERT_TAIL(&softc->request_ccbs,
1536 				&request_ccb->ccb_h, periph_links.tqe);
1537 			softc->restart = 1;
1538 		} else {
1539 			request_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1540 			xpt_done(request_ccb);
1541 		}
1542 	} else {
1543 		status = cam_periph_alloc(proberegister, NULL, probecleanup,
1544 					  probestart, "aprobe",
1545 					  CAM_PERIPH_BIO,
1546 					  request_ccb->ccb_h.path, NULL, 0,
1547 					  request_ccb);
1548 
1549 		if (status != CAM_REQ_CMP) {
1550 			xpt_print(path, "xpt_scan_lun: cam_alloc_periph "
1551 			    "returned an error, can't continue probe\n");
1552 			request_ccb->ccb_h.status = status;
1553 			xpt_done(request_ccb);
1554 		}
1555 	}
1556 	if (lock)
1557 		xpt_path_unlock(path);
1558 }
1559 
1560 static void
1561 xptscandone(struct cam_periph *periph, union ccb *done_ccb)
1562 {
1563 
1564 	xpt_free_path(done_ccb->ccb_h.path);
1565 	xpt_free_ccb(done_ccb);
1566 }
1567 
1568 static struct cam_ed *
1569 ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
1570 {
1571 	struct ata_quirk_entry *quirk;
1572 	struct cam_ed *device;
1573 
1574 	device = xpt_alloc_device(bus, target, lun_id);
1575 	if (device == NULL)
1576 		return (NULL);
1577 
1578 	/*
1579 	 * Take the default quirk entry until we have inquiry
1580 	 * data and can determine a better quirk to use.
1581 	 */
1582 	quirk = &ata_quirk_table[ata_quirk_table_size - 1];
1583 	device->quirk = (void *)quirk;
1584 	device->mintags = 0;
1585 	device->maxtags = 0;
1586 	bzero(&device->inq_data, sizeof(device->inq_data));
1587 	device->inq_flags = 0;
1588 	device->queue_flags = 0;
1589 	device->serial_num = NULL;
1590 	device->serial_num_len = 0;
1591 	return (device);
1592 }
1593 
1594 static void
1595 ata_device_transport(struct cam_path *path)
1596 {
1597 	struct ccb_pathinq cpi;
1598 	struct ccb_trans_settings cts;
1599 	struct scsi_inquiry_data *inq_buf = NULL;
1600 	struct ata_params *ident_buf = NULL;
1601 
1602 	/* Get transport information from the SIM */
1603 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
1604 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1605 	xpt_action((union ccb *)&cpi);
1606 
1607 	path->device->transport = cpi.transport;
1608 	if ((path->device->flags & CAM_DEV_INQUIRY_DATA_VALID) != 0)
1609 		inq_buf = &path->device->inq_data;
1610 	if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) != 0)
1611 		ident_buf = &path->device->ident_data;
1612 	if (path->device->protocol == PROTO_ATA) {
1613 		path->device->protocol_version = ident_buf ?
1614 		    ata_version(ident_buf->version_major) : cpi.protocol_version;
1615 	} else if (path->device->protocol == PROTO_SCSI) {
1616 		path->device->protocol_version = inq_buf ?
1617 		    SID_ANSI_REV(inq_buf) : cpi.protocol_version;
1618 	}
1619 	path->device->transport_version = ident_buf ?
1620 	    ata_version(ident_buf->version_major) : cpi.transport_version;
1621 
1622 	/* Tell the controller what we think */
1623 	xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
1624 	cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
1625 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
1626 	cts.transport = path->device->transport;
1627 	cts.transport_version = path->device->transport_version;
1628 	cts.protocol = path->device->protocol;
1629 	cts.protocol_version = path->device->protocol_version;
1630 	cts.proto_specific.valid = 0;
1631 	if (ident_buf) {
1632 		if (path->device->transport == XPORT_ATA) {
1633 			cts.xport_specific.ata.atapi =
1634 			    (ident_buf->config == ATA_PROTO_CFA) ? 0 :
1635 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 :
1636 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0;
1637 			cts.xport_specific.ata.valid = CTS_ATA_VALID_ATAPI;
1638 		} else {
1639 			cts.xport_specific.sata.atapi =
1640 			    (ident_buf->config == ATA_PROTO_CFA) ? 0 :
1641 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 :
1642 			    ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0;
1643 			cts.xport_specific.sata.valid = CTS_SATA_VALID_ATAPI;
1644 		}
1645 	} else
1646 		cts.xport_specific.valid = 0;
1647 	xpt_action((union ccb *)&cts);
1648 }
1649 
1650 static void
1651 ata_dev_advinfo(union ccb *start_ccb)
1652 {
1653 	struct cam_ed *device;
1654 	struct ccb_dev_advinfo *cdai;
1655 	off_t amt;
1656 
1657 	start_ccb->ccb_h.status = CAM_REQ_INVALID;
1658 	device = start_ccb->ccb_h.path->device;
1659 	cdai = &start_ccb->cdai;
1660 	switch(cdai->buftype) {
1661 	case CDAI_TYPE_SCSI_DEVID:
1662 		if (cdai->flags & CDAI_FLAG_STORE)
1663 			return;
1664 		cdai->provsiz = device->device_id_len;
1665 		if (device->device_id_len == 0)
1666 			break;
1667 		amt = device->device_id_len;
1668 		if (cdai->provsiz > cdai->bufsiz)
1669 			amt = cdai->bufsiz;
1670 		memcpy(cdai->buf, device->device_id, amt);
1671 		break;
1672 	case CDAI_TYPE_SERIAL_NUM:
1673 		if (cdai->flags & CDAI_FLAG_STORE)
1674 			return;
1675 		cdai->provsiz = device->serial_num_len;
1676 		if (device->serial_num_len == 0)
1677 			break;
1678 		amt = device->serial_num_len;
1679 		if (cdai->provsiz > cdai->bufsiz)
1680 			amt = cdai->bufsiz;
1681 		memcpy(cdai->buf, device->serial_num, amt);
1682 		break;
1683 	case CDAI_TYPE_PHYS_PATH:
1684 		if (cdai->flags & CDAI_FLAG_STORE) {
1685 			if (device->physpath != NULL)
1686 				free(device->physpath, M_CAMXPT);
1687 			device->physpath_len = cdai->bufsiz;
1688 			/* Clear existing buffer if zero length */
1689 			if (cdai->bufsiz == 0)
1690 				break;
1691 			device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT);
1692 			if (device->physpath == NULL) {
1693 				start_ccb->ccb_h.status = CAM_REQ_ABORTED;
1694 				return;
1695 			}
1696 			memcpy(device->physpath, cdai->buf, cdai->bufsiz);
1697 		} else {
1698 			cdai->provsiz = device->physpath_len;
1699 			if (device->physpath_len == 0)
1700 				break;
1701 			amt = device->physpath_len;
1702 			if (cdai->provsiz > cdai->bufsiz)
1703 				amt = cdai->bufsiz;
1704 			memcpy(cdai->buf, device->physpath, amt);
1705 		}
1706 		break;
1707 	default:
1708 		return;
1709 	}
1710 	start_ccb->ccb_h.status = CAM_REQ_CMP;
1711 
1712 	if (cdai->flags & CDAI_FLAG_STORE) {
1713 		xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path,
1714 			  (void *)(uintptr_t)cdai->buftype);
1715 	}
1716 }
1717 
1718 static void
1719 ata_action(union ccb *start_ccb)
1720 {
1721 
1722 	switch (start_ccb->ccb_h.func_code) {
1723 	case XPT_SET_TRAN_SETTINGS:
1724 	{
1725 		ata_set_transfer_settings(&start_ccb->cts,
1726 					   start_ccb->ccb_h.path,
1727 					   /*async_update*/FALSE);
1728 		break;
1729 	}
1730 	case XPT_SCAN_BUS:
1731 	case XPT_SCAN_TGT:
1732 		ata_scan_bus(start_ccb->ccb_h.path->periph, start_ccb);
1733 		break;
1734 	case XPT_SCAN_LUN:
1735 		ata_scan_lun(start_ccb->ccb_h.path->periph,
1736 			      start_ccb->ccb_h.path, start_ccb->crcn.flags,
1737 			      start_ccb);
1738 		break;
1739 	case XPT_GET_TRAN_SETTINGS:
1740 	{
1741 		ata_get_transfer_settings(&start_ccb->cts);
1742 		break;
1743 	}
1744 	case XPT_SCSI_IO:
1745 	{
1746 		struct cam_ed *device;
1747 		u_int	maxlen = 0;
1748 
1749 		device = start_ccb->ccb_h.path->device;
1750 		if (device->protocol == PROTO_SCSI &&
1751 		    (device->flags & CAM_DEV_IDENTIFY_DATA_VALID)) {
1752 			uint16_t p =
1753 			    device->ident_data.config & ATA_PROTO_MASK;
1754 
1755 			maxlen =
1756 			    (device->ident_data.config == ATA_PROTO_CFA) ? 0 :
1757 			    (p == ATA_PROTO_ATAPI_16) ? 16 :
1758 			    (p == ATA_PROTO_ATAPI_12) ? 12 : 0;
1759 		}
1760 		if (start_ccb->csio.cdb_len > maxlen) {
1761 			start_ccb->ccb_h.status = CAM_REQ_INVALID;
1762 			xpt_done(start_ccb);
1763 			break;
1764 		}
1765 		xpt_action_default(start_ccb);
1766 		break;
1767 	}
1768 	case XPT_DEV_ADVINFO:
1769 	{
1770 		ata_dev_advinfo(start_ccb);
1771 		break;
1772 	}
1773 	default:
1774 		xpt_action_default(start_ccb);
1775 		break;
1776 	}
1777 }
1778 
1779 static void
1780 ata_get_transfer_settings(struct ccb_trans_settings *cts)
1781 {
1782 	struct	ccb_trans_settings_ata *ata;
1783 	struct	ccb_trans_settings_scsi *scsi;
1784 	struct	cam_ed *device;
1785 
1786 	device = cts->ccb_h.path->device;
1787 	xpt_action_default((union ccb *)cts);
1788 
1789 	if (cts->protocol == PROTO_UNKNOWN ||
1790 	    cts->protocol == PROTO_UNSPECIFIED) {
1791 		cts->protocol = device->protocol;
1792 		cts->protocol_version = device->protocol_version;
1793 	}
1794 
1795 	if (cts->protocol == PROTO_ATA) {
1796 		ata = &cts->proto_specific.ata;
1797 		if ((ata->valid & CTS_ATA_VALID_TQ) == 0) {
1798 			ata->valid |= CTS_ATA_VALID_TQ;
1799 			if (cts->type == CTS_TYPE_USER_SETTINGS ||
1800 			    (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
1801 			    (device->inq_flags & SID_CmdQue) != 0)
1802 				ata->flags |= CTS_ATA_FLAGS_TAG_ENB;
1803 		}
1804 	}
1805 	if (cts->protocol == PROTO_SCSI) {
1806 		scsi = &cts->proto_specific.scsi;
1807 		if ((scsi->valid & CTS_SCSI_VALID_TQ) == 0) {
1808 			scsi->valid |= CTS_SCSI_VALID_TQ;
1809 			if (cts->type == CTS_TYPE_USER_SETTINGS ||
1810 			    (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
1811 			    (device->inq_flags & SID_CmdQue) != 0)
1812 				scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
1813 		}
1814 	}
1815 
1816 	if (cts->transport == XPORT_UNKNOWN ||
1817 	    cts->transport == XPORT_UNSPECIFIED) {
1818 		cts->transport = device->transport;
1819 		cts->transport_version = device->transport_version;
1820 	}
1821 }
1822 
1823 static void
1824 ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path,
1825 			   int async_update)
1826 {
1827 	struct	ccb_pathinq cpi;
1828 	struct	ccb_trans_settings_ata *ata;
1829 	struct	ccb_trans_settings_scsi *scsi;
1830 	struct	ata_params *ident_data;
1831 	struct	scsi_inquiry_data *inq_data;
1832 	struct	cam_ed *device;
1833 
1834 	if (path == NULL || (device = path->device) == NULL) {
1835 		cts->ccb_h.status = CAM_PATH_INVALID;
1836 		xpt_done((union ccb *)cts);
1837 		return;
1838 	}
1839 
1840 	if (cts->protocol == PROTO_UNKNOWN
1841 	 || cts->protocol == PROTO_UNSPECIFIED) {
1842 		cts->protocol = device->protocol;
1843 		cts->protocol_version = device->protocol_version;
1844 	}
1845 
1846 	if (cts->protocol_version == PROTO_VERSION_UNKNOWN
1847 	 || cts->protocol_version == PROTO_VERSION_UNSPECIFIED)
1848 		cts->protocol_version = device->protocol_version;
1849 
1850 	if (cts->protocol != device->protocol) {
1851 		xpt_print(path, "Uninitialized Protocol %x:%x?\n",
1852 		       cts->protocol, device->protocol);
1853 		cts->protocol = device->protocol;
1854 	}
1855 
1856 	if (cts->protocol_version > device->protocol_version) {
1857 		if (bootverbose) {
1858 			xpt_print(path, "Down reving Protocol "
1859 			    "Version from %d to %d?\n", cts->protocol_version,
1860 			    device->protocol_version);
1861 		}
1862 		cts->protocol_version = device->protocol_version;
1863 	}
1864 
1865 	if (cts->transport == XPORT_UNKNOWN
1866 	 || cts->transport == XPORT_UNSPECIFIED) {
1867 		cts->transport = device->transport;
1868 		cts->transport_version = device->transport_version;
1869 	}
1870 
1871 	if (cts->transport_version == XPORT_VERSION_UNKNOWN
1872 	 || cts->transport_version == XPORT_VERSION_UNSPECIFIED)
1873 		cts->transport_version = device->transport_version;
1874 
1875 	if (cts->transport != device->transport) {
1876 		xpt_print(path, "Uninitialized Transport %x:%x?\n",
1877 		    cts->transport, device->transport);
1878 		cts->transport = device->transport;
1879 	}
1880 
1881 	if (cts->transport_version > device->transport_version) {
1882 		if (bootverbose) {
1883 			xpt_print(path, "Down reving Transport "
1884 			    "Version from %d to %d?\n", cts->transport_version,
1885 			    device->transport_version);
1886 		}
1887 		cts->transport_version = device->transport_version;
1888 	}
1889 
1890 	ident_data = &device->ident_data;
1891 	inq_data = &device->inq_data;
1892 	if (cts->protocol == PROTO_ATA)
1893 		ata = &cts->proto_specific.ata;
1894 	else
1895 		ata = NULL;
1896 	if (cts->protocol == PROTO_SCSI)
1897 		scsi = &cts->proto_specific.scsi;
1898 	else
1899 		scsi = NULL;
1900 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
1901 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1902 	xpt_action((union ccb *)&cpi);
1903 
1904 	/* Sanity checking */
1905 	if ((cpi.hba_inquiry & PI_TAG_ABLE) == 0
1906 	 || (ata && (ident_data->satacapabilities & ATA_SUPPORT_NCQ) == 0)
1907 	 || (scsi && (INQ_DATA_TQ_ENABLED(inq_data)) == 0)
1908 	 || (device->queue_flags & SCP_QUEUE_DQUE) != 0
1909 	 || (device->mintags == 0)) {
1910 		/*
1911 		 * Can't tag on hardware that doesn't support tags,
1912 		 * doesn't have it enabled, or has broken tag support.
1913 		 */
1914 		if (ata)
1915 			ata->flags &= ~CTS_ATA_FLAGS_TAG_ENB;
1916 		if (scsi)
1917 			scsi->flags &= ~CTS_SCSI_FLAGS_TAG_ENB;
1918 	}
1919 
1920 	/* Start/stop tags use. */
1921 	if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
1922 	    ((ata && (ata->valid & CTS_ATA_VALID_TQ) != 0) ||
1923 	     (scsi && (scsi->valid & CTS_SCSI_VALID_TQ) != 0))) {
1924 		int nowt, newt = 0;
1925 
1926 		nowt = ((device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
1927 			(device->inq_flags & SID_CmdQue) != 0);
1928 		if (ata)
1929 			newt = (ata->flags & CTS_ATA_FLAGS_TAG_ENB) != 0;
1930 		if (scsi)
1931 			newt = (scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0;
1932 
1933 		if (newt && !nowt) {
1934 			/*
1935 			 * Delay change to use tags until after a
1936 			 * few commands have gone to this device so
1937 			 * the controller has time to perform transfer
1938 			 * negotiations without tagged messages getting
1939 			 * in the way.
1940 			 */
1941 			device->tag_delay_count = CAM_TAG_DELAY_COUNT;
1942 			device->flags |= CAM_DEV_TAG_AFTER_COUNT;
1943 		} else if (nowt && !newt)
1944 			xpt_stop_tags(path);
1945 	}
1946 
1947 	if (async_update == FALSE)
1948 		xpt_action_default((union ccb *)cts);
1949 }
1950 
1951 /*
1952  * Handle any per-device event notifications that require action by the XPT.
1953  */
1954 static void
1955 ata_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target,
1956 	      struct cam_ed *device, void *async_arg)
1957 {
1958 	cam_status status;
1959 	struct cam_path newpath;
1960 
1961 	/*
1962 	 * We only need to handle events for real devices.
1963 	 */
1964 	if (target->target_id == CAM_TARGET_WILDCARD
1965 	 || device->lun_id == CAM_LUN_WILDCARD)
1966 		return;
1967 
1968 	/*
1969 	 * We need our own path with wildcards expanded to
1970 	 * handle certain types of events.
1971 	 */
1972 	if ((async_code == AC_SENT_BDR)
1973 	 || (async_code == AC_BUS_RESET)
1974 	 || (async_code == AC_INQ_CHANGED))
1975 		status = xpt_compile_path(&newpath, NULL,
1976 					  bus->path_id,
1977 					  target->target_id,
1978 					  device->lun_id);
1979 	else
1980 		status = CAM_REQ_CMP_ERR;
1981 
1982 	if (status == CAM_REQ_CMP) {
1983 		if (async_code == AC_INQ_CHANGED) {
1984 			/*
1985 			 * We've sent a start unit command, or
1986 			 * something similar to a device that
1987 			 * may have caused its inquiry data to
1988 			 * change. So we re-scan the device to
1989 			 * refresh the inquiry data for it.
1990 			 */
1991 			ata_scan_lun(newpath.periph, &newpath,
1992 				     CAM_EXPECT_INQ_CHANGE, NULL);
1993 		} else {
1994 			/* We need to reinitialize device after reset. */
1995 			ata_scan_lun(newpath.periph, &newpath,
1996 				     0, NULL);
1997 		}
1998 		xpt_release_path(&newpath);
1999 	} else if (async_code == AC_LOST_DEVICE &&
2000 	    (device->flags & CAM_DEV_UNCONFIGURED) == 0) {
2001 		device->flags |= CAM_DEV_UNCONFIGURED;
2002 		xpt_release_device(device);
2003 	} else if (async_code == AC_TRANSFER_NEG) {
2004 		struct ccb_trans_settings *settings;
2005 		struct cam_path path;
2006 
2007 		settings = (struct ccb_trans_settings *)async_arg;
2008 		xpt_compile_path(&path, NULL, bus->path_id, target->target_id,
2009 				 device->lun_id);
2010 		ata_set_transfer_settings(settings, &path,
2011 					  /*async_update*/TRUE);
2012 		xpt_release_path(&path);
2013 	}
2014 }
2015 
2016 static void
2017 ata_announce_periph(struct cam_periph *periph)
2018 {
2019 	struct	ccb_pathinq cpi;
2020 	struct	ccb_trans_settings cts;
2021 	struct	cam_path *path = periph->path;
2022 	u_int	speed;
2023 	u_int	mb;
2024 
2025 	cam_periph_assert(periph, MA_OWNED);
2026 
2027 	xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
2028 	cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
2029 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
2030 	xpt_action((union ccb*)&cts);
2031 	if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
2032 		return;
2033 	/* Ask the SIM for its base transfer speed */
2034 	xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
2035 	cpi.ccb_h.func_code = XPT_PATH_INQ;
2036 	xpt_action((union ccb *)&cpi);
2037 	/* Report connection speed */
2038 	speed = cpi.base_transfer_speed;
2039 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) {
2040 		struct	ccb_trans_settings_pata *pata =
2041 		    &cts.xport_specific.ata;
2042 
2043 		if (pata->valid & CTS_ATA_VALID_MODE)
2044 			speed = ata_mode2speed(pata->mode);
2045 	}
2046 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
2047 		struct	ccb_trans_settings_sata *sata =
2048 		    &cts.xport_specific.sata;
2049 
2050 		if (sata->valid & CTS_SATA_VALID_REVISION)
2051 			speed = ata_revision2speed(sata->revision);
2052 	}
2053 	mb = speed / 1000;
2054 	if (mb > 0)
2055 		printf("%s%d: %d.%03dMB/s transfers",
2056 		       periph->periph_name, periph->unit_number,
2057 		       mb, speed % 1000);
2058 	else
2059 		printf("%s%d: %dKB/s transfers", periph->periph_name,
2060 		       periph->unit_number, speed);
2061 	/* Report additional information about connection */
2062 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) {
2063 		struct ccb_trans_settings_pata *pata =
2064 		    &cts.xport_specific.ata;
2065 
2066 		printf(" (");
2067 		if (pata->valid & CTS_ATA_VALID_MODE)
2068 			printf("%s, ", ata_mode2string(pata->mode));
2069 		if ((pata->valid & CTS_ATA_VALID_ATAPI) && pata->atapi != 0)
2070 			printf("ATAPI %dbytes, ", pata->atapi);
2071 		if (pata->valid & CTS_ATA_VALID_BYTECOUNT)
2072 			printf("PIO %dbytes", pata->bytecount);
2073 		printf(")");
2074 	}
2075 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
2076 		struct ccb_trans_settings_sata *sata =
2077 		    &cts.xport_specific.sata;
2078 
2079 		printf(" (");
2080 		if (sata->valid & CTS_SATA_VALID_REVISION)
2081 			printf("SATA %d.x, ", sata->revision);
2082 		else
2083 			printf("SATA, ");
2084 		if (sata->valid & CTS_SATA_VALID_MODE)
2085 			printf("%s, ", ata_mode2string(sata->mode));
2086 		if ((sata->valid & CTS_ATA_VALID_ATAPI) && sata->atapi != 0)
2087 			printf("ATAPI %dbytes, ", sata->atapi);
2088 		if (sata->valid & CTS_SATA_VALID_BYTECOUNT)
2089 			printf("PIO %dbytes", sata->bytecount);
2090 		printf(")");
2091 	}
2092 	printf("\n");
2093 }
2094