xref: /freebsd/sys/cam/ata/ata_da.c (revision 273c26a3c3bea87a241d6879abd4f991db180bf0)
1 /*-
2  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include "opt_ada.h"
31 
32 #include <sys/param.h>
33 
34 #ifdef _KERNEL
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/bio.h>
38 #include <sys/sysctl.h>
39 #include <sys/taskqueue.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/conf.h>
43 #include <sys/devicestat.h>
44 #include <sys/eventhandler.h>
45 #include <sys/malloc.h>
46 #include <sys/endian.h>
47 #include <sys/cons.h>
48 #include <sys/proc.h>
49 #include <sys/reboot.h>
50 #include <sys/sbuf.h>
51 #include <geom/geom_disk.h>
52 #endif /* _KERNEL */
53 
54 #ifndef _KERNEL
55 #include <stdio.h>
56 #include <string.h>
57 #endif /* _KERNEL */
58 
59 #include <cam/cam.h>
60 #include <cam/cam_ccb.h>
61 #include <cam/cam_periph.h>
62 #include <cam/cam_xpt_periph.h>
63 #include <cam/scsi/scsi_all.h>
64 #include <cam/scsi/scsi_da.h>
65 #include <cam/cam_sim.h>
66 #include <cam/cam_iosched.h>
67 
68 #include <cam/ata/ata_all.h>
69 
70 #include <machine/md_var.h>	/* geometry translation */
71 
72 #ifdef _KERNEL
73 
74 #define ATA_MAX_28BIT_LBA               268435455UL
75 
76 extern int iosched_debug;
77 
78 typedef enum {
79 	ADA_STATE_RAHEAD,
80 	ADA_STATE_WCACHE,
81 	ADA_STATE_LOGDIR,
82 	ADA_STATE_IDDIR,
83 	ADA_STATE_SUP_CAP,
84 	ADA_STATE_ZONE,
85 	ADA_STATE_NORMAL
86 } ada_state;
87 
88 typedef enum {
89 	ADA_FLAG_CAN_48BIT	= 0x00000002,
90 	ADA_FLAG_CAN_FLUSHCACHE	= 0x00000004,
91 	ADA_FLAG_CAN_NCQ	= 0x00000008,
92 	ADA_FLAG_CAN_DMA	= 0x00000010,
93 	ADA_FLAG_NEED_OTAG	= 0x00000020,
94 	ADA_FLAG_WAS_OTAG	= 0x00000040,
95 	ADA_FLAG_CAN_TRIM	= 0x00000080,
96 	ADA_FLAG_OPEN		= 0x00000100,
97 	ADA_FLAG_SCTX_INIT	= 0x00000200,
98 	ADA_FLAG_CAN_CFA        = 0x00000400,
99 	ADA_FLAG_CAN_POWERMGT   = 0x00000800,
100 	ADA_FLAG_CAN_DMA48	= 0x00001000,
101 	ADA_FLAG_CAN_LOG	= 0x00002000,
102 	ADA_FLAG_CAN_IDLOG	= 0x00004000,
103 	ADA_FLAG_CAN_SUPCAP	= 0x00008000,
104 	ADA_FLAG_CAN_ZONE	= 0x00010000,
105 	ADA_FLAG_CAN_WCACHE	= 0x00020000,
106 	ADA_FLAG_CAN_RAHEAD	= 0x00040000,
107 	ADA_FLAG_PROBED		= 0x00080000,
108 	ADA_FLAG_ANNOUNCED	= 0x00100000,
109 	ADA_FLAG_DIRTY		= 0x00200000,
110 	ADA_FLAG_CAN_NCQ_TRIM	= 0x00400000,	/* CAN_TRIM also set */
111 	ADA_FLAG_PIM_ATA_EXT	= 0x00800000
112 } ada_flags;
113 
114 typedef enum {
115 	ADA_Q_NONE		= 0x00,
116 	ADA_Q_4K		= 0x01,
117 	ADA_Q_NCQ_TRIM_BROKEN	= 0x02,
118 	ADA_Q_LOG_BROKEN	= 0x04,
119 	ADA_Q_SMR_DM		= 0x08
120 } ada_quirks;
121 
122 #define ADA_Q_BIT_STRING	\
123 	"\020"			\
124 	"\0014K"		\
125 	"\002NCQ_TRIM_BROKEN"	\
126 	"\003LOG_BROKEN"	\
127 	"\004SMR_DM"
128 
129 typedef enum {
130 	ADA_CCB_RAHEAD		= 0x01,
131 	ADA_CCB_WCACHE		= 0x02,
132 	ADA_CCB_BUFFER_IO	= 0x03,
133 	ADA_CCB_DUMP		= 0x05,
134 	ADA_CCB_TRIM		= 0x06,
135 	ADA_CCB_LOGDIR		= 0x07,
136 	ADA_CCB_IDDIR		= 0x08,
137 	ADA_CCB_SUP_CAP		= 0x09,
138 	ADA_CCB_ZONE		= 0x0a,
139 	ADA_CCB_TYPE_MASK	= 0x0F,
140 } ada_ccb_state;
141 
142 typedef enum {
143 	ADA_ZONE_NONE		= 0x00,
144 	ADA_ZONE_DRIVE_MANAGED	= 0x01,
145 	ADA_ZONE_HOST_AWARE	= 0x02,
146 	ADA_ZONE_HOST_MANAGED	= 0x03
147 } ada_zone_mode;
148 
149 typedef enum {
150 	ADA_ZONE_FLAG_RZ_SUP		= 0x0001,
151 	ADA_ZONE_FLAG_OPEN_SUP		= 0x0002,
152 	ADA_ZONE_FLAG_CLOSE_SUP		= 0x0004,
153 	ADA_ZONE_FLAG_FINISH_SUP	= 0x0008,
154 	ADA_ZONE_FLAG_RWP_SUP		= 0x0010,
155 	ADA_ZONE_FLAG_SUP_MASK		= (ADA_ZONE_FLAG_RZ_SUP |
156 					   ADA_ZONE_FLAG_OPEN_SUP |
157 					   ADA_ZONE_FLAG_CLOSE_SUP |
158 					   ADA_ZONE_FLAG_FINISH_SUP |
159 					   ADA_ZONE_FLAG_RWP_SUP),
160 	ADA_ZONE_FLAG_URSWRZ		= 0x0020,
161 	ADA_ZONE_FLAG_OPT_SEQ_SET	= 0x0040,
162 	ADA_ZONE_FLAG_OPT_NONSEQ_SET	= 0x0080,
163 	ADA_ZONE_FLAG_MAX_SEQ_SET	= 0x0100,
164 	ADA_ZONE_FLAG_SET_MASK		= (ADA_ZONE_FLAG_OPT_SEQ_SET |
165 					   ADA_ZONE_FLAG_OPT_NONSEQ_SET |
166 					   ADA_ZONE_FLAG_MAX_SEQ_SET)
167 } ada_zone_flags;
168 
169 static struct ada_zone_desc {
170 	ada_zone_flags value;
171 	const char *desc;
172 } ada_zone_desc_table[] = {
173 	{ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
174 	{ADA_ZONE_FLAG_OPEN_SUP, "Open" },
175 	{ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
176 	{ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
177 	{ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
178 };
179 
180 
181 /* Offsets into our private area for storing information */
182 #define ccb_state	ppriv_field0
183 #define ccb_bp		ppriv_ptr1
184 
185 typedef enum {
186 	ADA_DELETE_NONE,
187 	ADA_DELETE_DISABLE,
188 	ADA_DELETE_CFA_ERASE,
189 	ADA_DELETE_DSM_TRIM,
190 	ADA_DELETE_NCQ_DSM_TRIM,
191 	ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
192 	ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
193 } ada_delete_methods;
194 
195 static const char *ada_delete_method_names[] =
196     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
197 #if 0
198 static const char *ada_delete_method_desc[] =
199     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
200 #endif
201 
202 struct disk_params {
203 	u_int8_t  heads;
204 	u_int8_t  secs_per_track;
205 	u_int32_t cylinders;
206 	u_int32_t secsize;	/* Number of bytes/logical sector */
207 	u_int64_t sectors;	/* Total number sectors */
208 };
209 
210 #define TRIM_MAX_BLOCKS	8
211 #define TRIM_MAX_RANGES	(TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
212 struct trim_request {
213 	uint8_t		data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
214 	TAILQ_HEAD(, bio) bps;
215 };
216 
217 struct ada_softc {
218 	struct   cam_iosched_softc *cam_iosched;
219 	int	 outstanding_cmds;	/* Number of active commands */
220 	int	 refcount;		/* Active xpt_action() calls */
221 	ada_state state;
222 	ada_flags flags;
223 	ada_zone_mode zone_mode;
224 	ada_zone_flags zone_flags;
225 	struct ata_gp_log_dir ata_logdir;
226 	int valid_logdir_len;
227 	struct ata_identify_log_pages ata_iddir;
228 	int valid_iddir_len;
229 	uint64_t optimal_seq_zones;
230 	uint64_t optimal_nonseq_zones;
231 	uint64_t max_seq_zones;
232 	ada_quirks quirks;
233 	ada_delete_methods delete_method;
234 	int	 trim_max_ranges;
235 	int	 read_ahead;
236 	int	 write_cache;
237 	int	 unmappedio;
238 	int	 rotating;
239 #ifdef ADA_TEST_FAILURE
240 	int      force_read_error;
241 	int      force_write_error;
242 	int      periodic_read_error;
243 	int      periodic_read_count;
244 #endif
245 	struct	 disk_params params;
246 	struct	 disk *disk;
247 	struct task		sysctl_task;
248 	struct sysctl_ctx_list	sysctl_ctx;
249 	struct sysctl_oid	*sysctl_tree;
250 	struct callout		sendordered_c;
251 	struct trim_request	trim_req;
252 #ifdef CAM_IO_STATS
253 	struct sysctl_ctx_list	sysctl_stats_ctx;
254 	struct sysctl_oid	*sysctl_stats_tree;
255 	u_int	timeouts;
256 	u_int	errors;
257 	u_int	invalidations;
258 #endif
259 };
260 
261 struct ada_quirk_entry {
262 	struct scsi_inquiry_pattern inq_pat;
263 	ada_quirks quirks;
264 };
265 
266 static struct ada_quirk_entry ada_quirk_table[] =
267 {
268 	{
269 		/* Hitachi Advanced Format (4k) drives */
270 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
271 		/*quirks*/ADA_Q_4K
272 	},
273 	{
274 		/* Samsung Advanced Format (4k) drives */
275 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
276 		/*quirks*/ADA_Q_4K
277 	},
278 	{
279 		/* Samsung Advanced Format (4k) drives */
280 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
281 		/*quirks*/ADA_Q_4K
282 	},
283 	{
284 		/* Seagate Barracuda Green Advanced Format (4k) drives */
285 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
286 		/*quirks*/ADA_Q_4K
287 	},
288 	{
289 		/* Seagate Barracuda Advanced Format (4k) drives */
290 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
291 		/*quirks*/ADA_Q_4K
292 	},
293 	{
294 		/* Seagate Barracuda Advanced Format (4k) drives */
295 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
296 		/*quirks*/ADA_Q_4K
297 	},
298 	{
299 		/* Seagate Momentus Advanced Format (4k) drives */
300 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
301 		/*quirks*/ADA_Q_4K
302 	},
303 	{
304 		/* Seagate Momentus Advanced Format (4k) drives */
305 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
306 		/*quirks*/ADA_Q_4K
307 	},
308 	{
309 		/* Seagate Momentus Advanced Format (4k) drives */
310 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
311 		/*quirks*/ADA_Q_4K
312 	},
313 	{
314 		/* Seagate Momentus Advanced Format (4k) drives */
315 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
316 		/*quirks*/ADA_Q_4K
317 	},
318 	{
319 		/* Seagate Momentus Advanced Format (4k) drives */
320 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
321 		/*quirks*/ADA_Q_4K
322 	},
323 	{
324 		/* Seagate Momentus Advanced Format (4k) drives */
325 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
326 		/*quirks*/ADA_Q_4K
327 	},
328 	{
329 		/* Seagate Momentus Advanced Format (4k) drives */
330 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
331 		/*quirks*/ADA_Q_4K
332 	},
333 	{
334 		/* Seagate Momentus Thin Advanced Format (4k) drives */
335 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
336 		/*quirks*/ADA_Q_4K
337 	},
338 	{
339 		/* WDC Caviar Red Advanced Format (4k) drives */
340 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
341 		/*quirks*/ADA_Q_4K
342 	},
343 	{
344 		/* WDC Caviar Green Advanced Format (4k) drives */
345 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
346 		/*quirks*/ADA_Q_4K
347 	},
348 	{
349 		/* WDC Caviar Green/Red Advanced Format (4k) drives */
350 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
351 		/*quirks*/ADA_Q_4K
352 	},
353 	{
354 		/* WDC Caviar Red Advanced Format (4k) drives */
355 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
356 		/*quirks*/ADA_Q_4K
357 	},
358 	{
359 		/* WDC Caviar Black Advanced Format (4k) drives */
360 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????EX*", "*" },
361 		/*quirks*/ADA_Q_4K
362 	},
363 	{
364 		/* WDC Caviar Green Advanced Format (4k) drives */
365 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
366 		/*quirks*/ADA_Q_4K
367 	},
368 	{
369 		/* WDC Caviar Green Advanced Format (4k) drives */
370 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
371 		/*quirks*/ADA_Q_4K
372 	},
373 	{
374 		/* WDC Scorpio Black Advanced Format (4k) drives */
375 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
376 		/*quirks*/ADA_Q_4K
377 	},
378 	{
379 		/* WDC Scorpio Black Advanced Format (4k) drives */
380 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
381 		/*quirks*/ADA_Q_4K
382 	},
383 	{
384 		/* WDC Scorpio Blue Advanced Format (4k) drives */
385 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
386 		/*quirks*/ADA_Q_4K
387 	},
388 	{
389 		/* WDC Scorpio Blue Advanced Format (4k) drives */
390 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
391 		/*quirks*/ADA_Q_4K
392 	},
393 	/* SSDs */
394 	{
395 		/*
396 		 * Corsair Force 2 SSDs
397 		 * 4k optimised & trim only works in 4k requests + 4k aligned
398 		 */
399 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
400 		/*quirks*/ADA_Q_4K
401 	},
402 	{
403 		/*
404 		 * Corsair Force 3 SSDs
405 		 * 4k optimised & trim only works in 4k requests + 4k aligned
406 		 */
407 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
408 		/*quirks*/ADA_Q_4K
409 	},
410 	{
411 		/*
412 		 * Corsair Neutron GTX SSDs
413 		 * 4k optimised & trim only works in 4k requests + 4k aligned
414 		 */
415 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
416 		/*quirks*/ADA_Q_4K
417 	},
418 	{
419 		/*
420 		 * Corsair Force GT & GS SSDs
421 		 * 4k optimised & trim only works in 4k requests + 4k aligned
422 		 */
423 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
424 		/*quirks*/ADA_Q_4K
425 	},
426 	{
427 		/*
428 		 * Crucial M4 SSDs
429 		 * 4k optimised & trim only works in 4k requests + 4k aligned
430 		 */
431 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
432 		/*quirks*/ADA_Q_4K
433 	},
434 	{
435 		/*
436 		 * Crucial M500 SSDs MU07 firmware
437 		 * NCQ Trim works
438 		 */
439 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
440 		/*quirks*/0
441 	},
442 	{
443 		/*
444 		 * Crucial M500 SSDs all other firmware
445 		 * NCQ Trim doesn't work
446 		 */
447 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
448 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
449 	},
450 	{
451 		/*
452 		 * Crucial M550 SSDs
453 		 * NCQ Trim doesn't work, but only on MU01 firmware
454 		 */
455 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
456 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
457 	},
458 	{
459 		/*
460 		 * Crucial MX100 SSDs
461 		 * NCQ Trim doesn't work, but only on MU01 firmware
462 		 */
463 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
464 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
465 	},
466 	{
467 		/*
468 		 * Crucial RealSSD C300 SSDs
469 		 * 4k optimised
470 		 */
471 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
472 		"*" }, /*quirks*/ADA_Q_4K
473 	},
474 	{
475 		/*
476 		 * FCCT M500 SSDs
477 		 * NCQ Trim doesn't work
478 		 */
479 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
480 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
481 	},
482 	{
483 		/*
484 		 * Intel 320 Series SSDs
485 		 * 4k optimised & trim only works in 4k requests + 4k aligned
486 		 */
487 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
488 		/*quirks*/ADA_Q_4K
489 	},
490 	{
491 		/*
492 		 * Intel 330 Series SSDs
493 		 * 4k optimised & trim only works in 4k requests + 4k aligned
494 		 */
495 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
496 		/*quirks*/ADA_Q_4K
497 	},
498 	{
499 		/*
500 		 * Intel 510 Series SSDs
501 		 * 4k optimised & trim only works in 4k requests + 4k aligned
502 		 */
503 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
504 		/*quirks*/ADA_Q_4K
505 	},
506 	{
507 		/*
508 		 * Intel 520 Series SSDs
509 		 * 4k optimised & trim only works in 4k requests + 4k aligned
510 		 */
511 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
512 		/*quirks*/ADA_Q_4K
513 	},
514 	{
515 		/*
516 		 * Intel X25-M Series SSDs
517 		 * 4k optimised & trim only works in 4k requests + 4k aligned
518 		 */
519 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
520 		/*quirks*/ADA_Q_4K
521 	},
522 	{
523 		/*
524 		 * Kingston E100 Series SSDs
525 		 * 4k optimised & trim only works in 4k requests + 4k aligned
526 		 */
527 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
528 		/*quirks*/ADA_Q_4K
529 	},
530 	{
531 		/*
532 		 * Kingston HyperX 3k SSDs
533 		 * 4k optimised & trim only works in 4k requests + 4k aligned
534 		 */
535 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
536 		/*quirks*/ADA_Q_4K
537 	},
538 	{
539 		/*
540 		 * Marvell SSDs (entry taken from OpenSolaris)
541 		 * 4k optimised & trim only works in 4k requests + 4k aligned
542 		 */
543 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
544 		/*quirks*/ADA_Q_4K
545 	},
546 	{
547 		/*
548 		 * Micron M500 SSDs firmware MU07
549 		 * NCQ Trim works?
550 		 */
551 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
552 		/*quirks*/0
553 	},
554 	{
555 		/*
556 		 * Micron M500 SSDs all other firmware
557 		 * NCQ Trim doesn't work
558 		 */
559 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
560 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
561 	},
562 	{
563 		/*
564 		 * Micron M5[15]0 SSDs
565 		 * NCQ Trim doesn't work, but only MU01 firmware
566 		 */
567 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
568 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
569 	},
570 	{
571 		/*
572 		 * OCZ Agility 2 SSDs
573 		 * 4k optimised & trim only works in 4k requests + 4k aligned
574 		 */
575 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
576 		/*quirks*/ADA_Q_4K
577 	},
578 	{
579 		/*
580 		 * OCZ Agility 3 SSDs
581 		 * 4k optimised & trim only works in 4k requests + 4k aligned
582 		 */
583 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
584 		/*quirks*/ADA_Q_4K
585 	},
586 	{
587 		/*
588 		 * OCZ Deneva R Series SSDs
589 		 * 4k optimised & trim only works in 4k requests + 4k aligned
590 		 */
591 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
592 		/*quirks*/ADA_Q_4K
593 	},
594 	{
595 		/*
596 		 * OCZ Vertex 2 SSDs (inc pro series)
597 		 * 4k optimised & trim only works in 4k requests + 4k aligned
598 		 */
599 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
600 		/*quirks*/ADA_Q_4K
601 	},
602 	{
603 		/*
604 		 * OCZ Vertex 3 SSDs
605 		 * 4k optimised & trim only works in 4k requests + 4k aligned
606 		 */
607 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
608 		/*quirks*/ADA_Q_4K
609 	},
610 	{
611 		/*
612 		 * OCZ Vertex 4 SSDs
613 		 * 4k optimised & trim only works in 4k requests + 4k aligned
614 		 */
615 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
616 		/*quirks*/ADA_Q_4K
617 	},
618 	{
619 		/*
620 		 * Samsung 830 Series SSDs
621 		 * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
622 		 */
623 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
624 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
625 	},
626 	{
627 		/*
628 		 * Samsung 840 SSDs
629 		 * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
630 		 */
631 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
632 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
633 	},
634 	{
635 		/*
636 		 * Samsung 850 SSDs
637 		 * 4k optimised, NCQ TRIM broken (normal TRIM fine)
638 		 */
639 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
640 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
641 	},
642 	{
643 		/*
644 		 * Samsung SM863 Series SSDs (MZ7KM*)
645 		 * 4k optimised, NCQ believed to be working
646 		 */
647 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
648 		/*quirks*/ADA_Q_4K
649 	},
650 	{
651 		/*
652 		 * Samsung 843T Series SSDs (MZ7WD*)
653 		 * Samsung PM851 Series SSDs (MZ7TE*)
654 		 * Samsung PM853T Series SSDs (MZ7GE*)
655 		 * 4k optimised, NCQ believed to be broken since these are
656 		 * appear to be built with the same controllers as the 840/850.
657 		 */
658 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
659 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
660 	},
661 	{
662 		/*
663 		 * Samsung PM851 Series SSDs Dell OEM
664 		 * device model          "SAMSUNG SSD PM851 mSATA 256GB"
665 		 * 4k optimised, NCQ broken
666 		 */
667 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
668 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
669 	},
670 	{
671 		/*
672 		 * SuperTalent TeraDrive CT SSDs
673 		 * 4k optimised & trim only works in 4k requests + 4k aligned
674 		 */
675 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
676 		/*quirks*/ADA_Q_4K
677 	},
678 	{
679 		/*
680 		 * XceedIOPS SATA SSDs
681 		 * 4k optimised
682 		 */
683 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
684 		/*quirks*/ADA_Q_4K
685 	},
686 	{
687 		/*
688 		 * Samsung drive that doesn't support READ LOG EXT or
689 		 * READ LOG DMA EXT, despite reporting that it does in
690 		 * ATA identify data:
691 		 * SAMSUNG HD200HJ KF100-06
692 		 */
693 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
694 		/*quirks*/ADA_Q_LOG_BROKEN
695 	},
696 	{
697 		/*
698 		 * Samsung drive that doesn't support READ LOG EXT or
699 		 * READ LOG DMA EXT, despite reporting that it does in
700 		 * ATA identify data:
701 		 * SAMSUNG HD501LJ CR100-10
702 		 */
703 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
704 		/*quirks*/ADA_Q_LOG_BROKEN
705 	},
706 	{
707 		/*
708 		 * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
709 		 * Drive Managed SATA hard drive.  This drive doesn't report
710 		 * in firmware that it is a drive managed SMR drive.
711 		 */
712 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS0002*", "*" },
713 		/*quirks*/ADA_Q_SMR_DM
714 	},
715 	{
716 		/* Default */
717 		{
718 		  T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
719 		  /*vendor*/"*", /*product*/"*", /*revision*/"*"
720 		},
721 		/*quirks*/0
722 	},
723 };
724 
725 static	disk_strategy_t	adastrategy;
726 static	dumper_t	adadump;
727 static	periph_init_t	adainit;
728 static	void		adadiskgonecb(struct disk *dp);
729 static	periph_oninv_t	adaoninvalidate;
730 static	periph_dtor_t	adacleanup;
731 static	void		adaasync(void *callback_arg, u_int32_t code,
732 				struct cam_path *path, void *arg);
733 static	int		adazonemodesysctl(SYSCTL_HANDLER_ARGS);
734 static	int		adazonesupsysctl(SYSCTL_HANDLER_ARGS);
735 static	void		adasysctlinit(void *context, int pending);
736 static	int		adagetattr(struct bio *bp);
737 static	void		adasetflags(struct ada_softc *softc,
738 				    struct ccb_getdev *cgd);
739 static	periph_ctor_t	adaregister;
740 static	void		ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
741 				    struct ccb_ataio *ataio);
742 static	void 		ada_cfaerase(struct ada_softc *softc, struct bio *bp,
743 				     struct ccb_ataio *ataio);
744 static	int		ada_zone_bio_to_ata(int disk_zone_cmd);
745 static	int		ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
746 				     struct bio *bp, int *queue_ccb);
747 static	periph_start_t	adastart;
748 static	void		adaprobedone(struct cam_periph *periph, union ccb *ccb);
749 static	void		adazonedone(struct cam_periph *periph, union ccb *ccb);
750 static	void		adadone(struct cam_periph *periph,
751 			       union ccb *done_ccb);
752 static  int		adaerror(union ccb *ccb, u_int32_t cam_flags,
753 				u_int32_t sense_flags);
754 static void		adagetparams(struct cam_periph *periph,
755 				struct ccb_getdev *cgd);
756 static timeout_t	adasendorderedtag;
757 static void		adashutdown(void *arg, int howto);
758 static void		adasuspend(void *arg);
759 static void		adaresume(void *arg);
760 
761 #ifndef	ADA_DEFAULT_LEGACY_ALIASES
762 #define	ADA_DEFAULT_LEGACY_ALIASES	1
763 #endif
764 
765 #ifndef ADA_DEFAULT_TIMEOUT
766 #define ADA_DEFAULT_TIMEOUT 30	/* Timeout in seconds */
767 #endif
768 
769 #ifndef	ADA_DEFAULT_RETRY
770 #define	ADA_DEFAULT_RETRY	4
771 #endif
772 
773 #ifndef	ADA_DEFAULT_SEND_ORDERED
774 #define	ADA_DEFAULT_SEND_ORDERED	1
775 #endif
776 
777 #ifndef	ADA_DEFAULT_SPINDOWN_SHUTDOWN
778 #define	ADA_DEFAULT_SPINDOWN_SHUTDOWN	1
779 #endif
780 
781 #ifndef	ADA_DEFAULT_SPINDOWN_SUSPEND
782 #define	ADA_DEFAULT_SPINDOWN_SUSPEND	1
783 #endif
784 
785 #ifndef	ADA_DEFAULT_READ_AHEAD
786 #define	ADA_DEFAULT_READ_AHEAD	1
787 #endif
788 
789 #ifndef	ADA_DEFAULT_WRITE_CACHE
790 #define	ADA_DEFAULT_WRITE_CACHE	1
791 #endif
792 
793 #define	ADA_RA	(softc->read_ahead >= 0 ? \
794 		 softc->read_ahead : ada_read_ahead)
795 #define	ADA_WC	(softc->write_cache >= 0 ? \
796 		 softc->write_cache : ada_write_cache)
797 
798 /*
799  * Most platforms map firmware geometry to actual, but some don't.  If
800  * not overridden, default to nothing.
801  */
802 #ifndef ata_disk_firmware_geom_adjust
803 #define	ata_disk_firmware_geom_adjust(disk)
804 #endif
805 
806 static int ada_retry_count = ADA_DEFAULT_RETRY;
807 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
808 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
809 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
810 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
811 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
812 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
813 
814 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
815             "CAM Direct Access Disk driver");
816 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
817            &ada_retry_count, 0, "Normal I/O retry count");
818 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
819            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
820 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
821            &ada_send_ordered, 0, "Send Ordered Tags");
822 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
823            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
824 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
825            &ada_spindown_suspend, 0, "Spin down upon suspend");
826 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
827            &ada_read_ahead, 0, "Enable disk read-ahead");
828 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
829            &ada_write_cache, 0, "Enable disk write cache");
830 
831 /*
832  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
833  * to the default timeout, we check to see whether an ordered
834  * tagged transaction is appropriate to prevent simple tag
835  * starvation.  Since we'd like to ensure that there is at least
836  * 1/2 of the timeout length left for a starved transaction to
837  * complete after we've sent an ordered tag, we must poll at least
838  * four times in every timeout period.  This takes care of the worst
839  * case where a starved transaction starts during an interval that
840  * meets the requirement "don't send an ordered tag" test so it takes
841  * us two intervals to determine that a tag must be sent.
842  */
843 #ifndef ADA_ORDEREDTAG_INTERVAL
844 #define ADA_ORDEREDTAG_INTERVAL 4
845 #endif
846 
847 static struct periph_driver adadriver =
848 {
849 	adainit, "ada",
850 	TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
851 };
852 
853 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
854 
855 PERIPHDRIVER_DECLARE(ada, adadriver);
856 
857 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
858 
859 static int
860 adaopen(struct disk *dp)
861 {
862 	struct cam_periph *periph;
863 	struct ada_softc *softc;
864 	int error;
865 
866 	periph = (struct cam_periph *)dp->d_drv1;
867 	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
868 		return(ENXIO);
869 	}
870 
871 	cam_periph_lock(periph);
872 	if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
873 		cam_periph_unlock(periph);
874 		cam_periph_release(periph);
875 		return (error);
876 	}
877 
878 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
879 	    ("adaopen\n"));
880 
881 	softc = (struct ada_softc *)periph->softc;
882 	softc->flags |= ADA_FLAG_OPEN;
883 
884 	cam_periph_unhold(periph);
885 	cam_periph_unlock(periph);
886 	return (0);
887 }
888 
889 static int
890 adaclose(struct disk *dp)
891 {
892 	struct	cam_periph *periph;
893 	struct	ada_softc *softc;
894 	union ccb *ccb;
895 	int error;
896 
897 	periph = (struct cam_periph *)dp->d_drv1;
898 	softc = (struct ada_softc *)periph->softc;
899 	cam_periph_lock(periph);
900 
901 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
902 	    ("adaclose\n"));
903 
904 	/* We only sync the cache if the drive is capable of it. */
905 	if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
906 	    (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
907 	    (periph->flags & CAM_PERIPH_INVALID) == 0 &&
908 	    cam_periph_hold(periph, PRIBIO) == 0) {
909 
910 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
911 		cam_fill_ataio(&ccb->ataio,
912 				    1,
913 				    adadone,
914 				    CAM_DIR_NONE,
915 				    0,
916 				    NULL,
917 				    0,
918 				    ada_default_timeout*1000);
919 
920 		if (softc->flags & ADA_FLAG_CAN_48BIT)
921 			ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
922 		else
923 			ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
924 		error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
925 		    /*sense_flags*/0, softc->disk->d_devstat);
926 
927 		if (error != 0)
928 			xpt_print(periph->path, "Synchronize cache failed\n");
929 		else
930 			softc->flags &= ~ADA_FLAG_DIRTY;
931 		xpt_release_ccb(ccb);
932 		cam_periph_unhold(periph);
933 	}
934 
935 	softc->flags &= ~ADA_FLAG_OPEN;
936 
937 	while (softc->refcount != 0)
938 		cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
939 	cam_periph_unlock(periph);
940 	cam_periph_release(periph);
941 	return (0);
942 }
943 
944 static void
945 adaschedule(struct cam_periph *periph)
946 {
947 	struct ada_softc *softc = (struct ada_softc *)periph->softc;
948 
949 	if (softc->state != ADA_STATE_NORMAL)
950 		return;
951 
952 	cam_iosched_schedule(softc->cam_iosched, periph);
953 }
954 
955 /*
956  * Actually translate the requested transfer into one the physical driver
957  * can understand.  The transfer is described by a buf and will include
958  * only one physical transfer.
959  */
960 static void
961 adastrategy(struct bio *bp)
962 {
963 	struct cam_periph *periph;
964 	struct ada_softc *softc;
965 
966 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
967 	softc = (struct ada_softc *)periph->softc;
968 
969 	cam_periph_lock(periph);
970 
971 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
972 
973 	/*
974 	 * If the device has been made invalid, error out
975 	 */
976 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
977 		cam_periph_unlock(periph);
978 		biofinish(bp, NULL, ENXIO);
979 		return;
980 	}
981 
982 	/*
983 	 * Zone commands must be ordered, because they can depend on the
984 	 * effects of previously issued commands, and they may affect
985 	 * commands after them.
986 	 */
987 	if (bp->bio_cmd == BIO_ZONE)
988 		bp->bio_flags |= BIO_ORDERED;
989 
990 	/*
991 	 * Place it in the queue of disk activities for this disk
992 	 */
993 	cam_iosched_queue_work(softc->cam_iosched, bp);
994 
995 	/*
996 	 * Schedule ourselves for performing the work.
997 	 */
998 	adaschedule(periph);
999 	cam_periph_unlock(periph);
1000 
1001 	return;
1002 }
1003 
1004 static int
1005 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1006 {
1007 	struct	    cam_periph *periph;
1008 	struct	    ada_softc *softc;
1009 	u_int	    secsize;
1010 	union	    ccb ccb;
1011 	struct	    disk *dp;
1012 	uint64_t    lba;
1013 	uint16_t    count;
1014 	int	    error = 0;
1015 
1016 	dp = arg;
1017 	periph = dp->d_drv1;
1018 	softc = (struct ada_softc *)periph->softc;
1019 	cam_periph_lock(periph);
1020 	secsize = softc->params.secsize;
1021 	lba = offset / secsize;
1022 	count = length / secsize;
1023 
1024 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1025 		cam_periph_unlock(periph);
1026 		return (ENXIO);
1027 	}
1028 
1029 	if (length > 0) {
1030 		xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1031 		ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
1032 		cam_fill_ataio(&ccb.ataio,
1033 		    0,
1034 		    adadone,
1035 		    CAM_DIR_OUT,
1036 		    0,
1037 		    (u_int8_t *) virtual,
1038 		    length,
1039 		    ada_default_timeout*1000);
1040 		if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1041 		    (lba + count >= ATA_MAX_28BIT_LBA ||
1042 		    count >= 256)) {
1043 			ata_48bit_cmd(&ccb.ataio, ATA_WRITE_DMA48,
1044 			    0, lba, count);
1045 		} else {
1046 			ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA,
1047 			    0, lba, count);
1048 		}
1049 		xpt_polled_action(&ccb);
1050 
1051 		error = cam_periph_error(&ccb,
1052 		    0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1053 		if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
1054 			cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
1055 			    /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
1056 		if (error != 0)
1057 			printf("Aborting dump due to I/O error.\n");
1058 
1059 		cam_periph_unlock(periph);
1060 		return (error);
1061 	}
1062 
1063 	if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1064 		xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1065 
1066 		/*
1067 		 * Tell the drive to flush its internal cache. if we
1068 		 * can't flush in 5s we have big problems. No need to
1069 		 * wait the default 60s to detect problems.
1070 		 */
1071 		ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
1072 		cam_fill_ataio(&ccb.ataio,
1073 				    0,
1074 				    adadone,
1075 				    CAM_DIR_NONE,
1076 				    0,
1077 				    NULL,
1078 				    0,
1079 				    5*1000);
1080 
1081 		if (softc->flags & ADA_FLAG_CAN_48BIT)
1082 			ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1083 		else
1084 			ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0);
1085 		xpt_polled_action(&ccb);
1086 
1087 		error = cam_periph_error(&ccb,
1088 		    0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1089 		if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
1090 			cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
1091 			    /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
1092 		if (error != 0)
1093 			xpt_print(periph->path, "Synchronize cache failed\n");
1094 	}
1095 	cam_periph_unlock(periph);
1096 	return (error);
1097 }
1098 
1099 static void
1100 adainit(void)
1101 {
1102 	cam_status status;
1103 
1104 	/*
1105 	 * Install a global async callback.  This callback will
1106 	 * receive async callbacks like "new device found".
1107 	 */
1108 	status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1109 
1110 	if (status != CAM_REQ_CMP) {
1111 		printf("ada: Failed to attach master async callback "
1112 		       "due to status 0x%x!\n", status);
1113 	} else if (ada_send_ordered) {
1114 
1115 		/* Register our event handlers */
1116 		if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1117 					   NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1118 		    printf("adainit: power event registration failed!\n");
1119 		if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1120 					   NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1121 		    printf("adainit: power event registration failed!\n");
1122 		if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1123 					   NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1124 		    printf("adainit: shutdown event registration failed!\n");
1125 	}
1126 }
1127 
1128 /*
1129  * Callback from GEOM, called when it has finished cleaning up its
1130  * resources.
1131  */
1132 static void
1133 adadiskgonecb(struct disk *dp)
1134 {
1135 	struct cam_periph *periph;
1136 
1137 	periph = (struct cam_periph *)dp->d_drv1;
1138 
1139 	cam_periph_release(periph);
1140 }
1141 
1142 static void
1143 adaoninvalidate(struct cam_periph *periph)
1144 {
1145 	struct ada_softc *softc;
1146 
1147 	softc = (struct ada_softc *)periph->softc;
1148 
1149 	/*
1150 	 * De-register any async callbacks.
1151 	 */
1152 	xpt_register_async(0, adaasync, periph, periph->path);
1153 #ifdef CAM_IO_STATS
1154 	softc->invalidations++;
1155 #endif
1156 
1157 	/*
1158 	 * Return all queued I/O with ENXIO.
1159 	 * XXX Handle any transactions queued to the card
1160 	 *     with XPT_ABORT_CCB.
1161 	 */
1162 	cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1163 
1164 	disk_gone(softc->disk);
1165 }
1166 
1167 static void
1168 adacleanup(struct cam_periph *periph)
1169 {
1170 	struct ada_softc *softc;
1171 
1172 	softc = (struct ada_softc *)periph->softc;
1173 
1174 	cam_periph_unlock(periph);
1175 
1176 	cam_iosched_fini(softc->cam_iosched);
1177 
1178 	/*
1179 	 * If we can't free the sysctl tree, oh well...
1180 	 */
1181 	if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1182 #ifdef CAM_IO_STATS
1183 		if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1184 			xpt_print(periph->path,
1185 			    "can't remove sysctl stats context\n");
1186 #endif
1187 		if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1188 			xpt_print(periph->path,
1189 			    "can't remove sysctl context\n");
1190 	}
1191 
1192 	disk_destroy(softc->disk);
1193 	callout_drain(&softc->sendordered_c);
1194 	free(softc, M_DEVBUF);
1195 	cam_periph_lock(periph);
1196 }
1197 
1198 static void
1199 adasetdeletemethod(struct ada_softc *softc)
1200 {
1201 
1202 	if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1203 		softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1204 	else if (softc->flags & ADA_FLAG_CAN_TRIM)
1205 		softc->delete_method = ADA_DELETE_DSM_TRIM;
1206 	else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1207 		softc->delete_method = ADA_DELETE_CFA_ERASE;
1208 	else
1209 		softc->delete_method = ADA_DELETE_NONE;
1210 }
1211 
1212 static void
1213 adaasync(void *callback_arg, u_int32_t code,
1214 	struct cam_path *path, void *arg)
1215 {
1216 	struct ccb_getdev cgd;
1217 	struct cam_periph *periph;
1218 	struct ada_softc *softc;
1219 
1220 	periph = (struct cam_periph *)callback_arg;
1221 	switch (code) {
1222 	case AC_FOUND_DEVICE:
1223 	{
1224 		struct ccb_getdev *cgd;
1225 		cam_status status;
1226 
1227 		cgd = (struct ccb_getdev *)arg;
1228 		if (cgd == NULL)
1229 			break;
1230 
1231 		if (cgd->protocol != PROTO_ATA)
1232 			break;
1233 
1234 		/*
1235 		 * Allocate a peripheral instance for
1236 		 * this device and start the probe
1237 		 * process.
1238 		 */
1239 		status = cam_periph_alloc(adaregister, adaoninvalidate,
1240 					  adacleanup, adastart,
1241 					  "ada", CAM_PERIPH_BIO,
1242 					  path, adaasync,
1243 					  AC_FOUND_DEVICE, cgd);
1244 
1245 		if (status != CAM_REQ_CMP
1246 		 && status != CAM_REQ_INPROG)
1247 			printf("adaasync: Unable to attach to new device "
1248 				"due to status 0x%x\n", status);
1249 		break;
1250 	}
1251 	case AC_GETDEV_CHANGED:
1252 	{
1253 		softc = (struct ada_softc *)periph->softc;
1254 		xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1255 		cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1256 		xpt_action((union ccb *)&cgd);
1257 
1258 		/*
1259 		 * Set/clear support flags based on the new Identify data.
1260 		 */
1261 		adasetflags(softc, &cgd);
1262 
1263 		cam_periph_async(periph, code, path, arg);
1264 		break;
1265 	}
1266 	case AC_ADVINFO_CHANGED:
1267 	{
1268 		uintptr_t buftype;
1269 
1270 		buftype = (uintptr_t)arg;
1271 		if (buftype == CDAI_TYPE_PHYS_PATH) {
1272 			struct ada_softc *softc;
1273 
1274 			softc = periph->softc;
1275 			disk_attr_changed(softc->disk, "GEOM::physpath",
1276 					  M_NOWAIT);
1277 		}
1278 		break;
1279 	}
1280 	case AC_SENT_BDR:
1281 	case AC_BUS_RESET:
1282 	{
1283 		softc = (struct ada_softc *)periph->softc;
1284 		cam_periph_async(periph, code, path, arg);
1285 		if (softc->state != ADA_STATE_NORMAL)
1286 			break;
1287 		xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1288 		cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1289 		xpt_action((union ccb *)&cgd);
1290 		if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1291 			softc->state = ADA_STATE_RAHEAD;
1292 		else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1293 			softc->state = ADA_STATE_WCACHE;
1294 		else if ((softc->flags & ADA_FLAG_CAN_LOG)
1295 		      && (softc->zone_mode != ADA_ZONE_NONE))
1296 			softc->state = ADA_STATE_LOGDIR;
1297 		else
1298 		    break;
1299 		if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1300 			softc->state = ADA_STATE_NORMAL;
1301 		else
1302 			xpt_schedule(periph, CAM_PRIORITY_DEV);
1303 	}
1304 	default:
1305 		cam_periph_async(periph, code, path, arg);
1306 		break;
1307 	}
1308 }
1309 
1310 static int
1311 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1312 {
1313 	char tmpbuf[40];
1314 	struct ada_softc *softc;
1315 	int error;
1316 
1317 	softc = (struct ada_softc *)arg1;
1318 
1319 	switch (softc->zone_mode) {
1320 	case ADA_ZONE_DRIVE_MANAGED:
1321 		snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1322 		break;
1323 	case ADA_ZONE_HOST_AWARE:
1324 		snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1325 		break;
1326 	case ADA_ZONE_HOST_MANAGED:
1327 		snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1328 		break;
1329 	case ADA_ZONE_NONE:
1330 	default:
1331 		snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1332 		break;
1333 	}
1334 
1335 	error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1336 
1337 	return (error);
1338 }
1339 
1340 static int
1341 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1342 {
1343 	char tmpbuf[180];
1344 	struct ada_softc *softc;
1345 	struct sbuf sb;
1346 	int error, first;
1347 	unsigned int i;
1348 
1349 	softc = (struct ada_softc *)arg1;
1350 
1351 	error = 0;
1352 	first = 1;
1353 	sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1354 
1355 	for (i = 0; i < sizeof(ada_zone_desc_table) /
1356 	     sizeof(ada_zone_desc_table[0]); i++) {
1357 		if (softc->zone_flags & ada_zone_desc_table[i].value) {
1358 			if (first == 0)
1359 				sbuf_printf(&sb, ", ");
1360 			else
1361 				first = 0;
1362 			sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1363 		}
1364 	}
1365 
1366 	if (first == 1)
1367 		sbuf_printf(&sb, "None");
1368 
1369 	sbuf_finish(&sb);
1370 
1371 	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1372 
1373 	return (error);
1374 }
1375 
1376 
1377 static void
1378 adasysctlinit(void *context, int pending)
1379 {
1380 	struct cam_periph *periph;
1381 	struct ada_softc *softc;
1382 	char tmpstr[80], tmpstr2[80];
1383 
1384 	periph = (struct cam_periph *)context;
1385 
1386 	/* periph was held for us when this task was enqueued */
1387 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1388 		cam_periph_release(periph);
1389 		return;
1390 	}
1391 
1392 	softc = (struct ada_softc *)periph->softc;
1393 	snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1394 	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1395 
1396 	sysctl_ctx_init(&softc->sysctl_ctx);
1397 	softc->flags |= ADA_FLAG_SCTX_INIT;
1398 	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1399 		SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1400 		CTLFLAG_RD, 0, tmpstr);
1401 	if (softc->sysctl_tree == NULL) {
1402 		printf("adasysctlinit: unable to allocate sysctl tree\n");
1403 		cam_periph_release(periph);
1404 		return;
1405 	}
1406 
1407 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1408 		OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1409 		softc, 0, adadeletemethodsysctl, "A",
1410 		"BIO_DELETE execution method");
1411 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1412 		OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1413 		&softc->read_ahead, 0, "Enable disk read ahead.");
1414 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1415 		OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1416 		&softc->write_cache, 0, "Enable disk write cache.");
1417 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1418 		OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1419 		&softc->unmappedio, 0, "Unmapped I/O leaf");
1420 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1421 		OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1422 		&softc->rotating, 0, "Rotating media");
1423 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1424 		OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1425 		softc, 0, adazonemodesysctl, "A",
1426 		"Zone Mode");
1427 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1428 		OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1429 		softc, 0, adazonesupsysctl, "A",
1430 		"Zone Support");
1431 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1432 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1433 		"optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1434 		"Optimal Number of Open Sequential Write Preferred Zones");
1435 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1436 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1437 		"optimal_nonseq_zones", CTLFLAG_RD,
1438 		&softc->optimal_nonseq_zones,
1439 		"Optimal Number of Non-Sequentially Written Sequential Write "
1440 		"Preferred Zones");
1441 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1442 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1443 		"max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1444 		"Maximum Number of Open Sequential Write Required Zones");
1445 
1446 #ifdef ADA_TEST_FAILURE
1447 	/*
1448 	 * Add a 'door bell' sysctl which allows one to set it from userland
1449 	 * and cause something bad to happen.  For the moment, we only allow
1450 	 * whacking the next read or write.
1451 	 */
1452 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1453 		OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1454 		&softc->force_read_error, 0,
1455 		"Force a read error for the next N reads.");
1456 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1457 		OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1458 		&softc->force_write_error, 0,
1459 		"Force a write error for the next N writes.");
1460 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1461 		OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1462 		&softc->periodic_read_error, 0,
1463 		"Force a read error every N reads (don't set too low).");
1464 #endif
1465 
1466 #ifdef CAM_IO_STATS
1467 	softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1468 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1469 		CTLFLAG_RD, 0, "Statistics");
1470 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1471 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1472 		OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1473 		&softc->timeouts, 0,
1474 		"Device timeouts reported by the SIM");
1475 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1476 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1477 		OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1478 		&softc->errors, 0,
1479 		"Transport errors reported by the SIM.");
1480 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1481 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1482 		OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1483 		&softc->invalidations, 0,
1484 		"Device pack invalidations.");
1485 #endif
1486 
1487 	cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1488 	    softc->sysctl_tree);
1489 
1490 	cam_periph_release(periph);
1491 }
1492 
1493 static int
1494 adagetattr(struct bio *bp)
1495 {
1496 	int ret;
1497 	struct cam_periph *periph;
1498 
1499 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1500 	cam_periph_lock(periph);
1501 	ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1502 	    periph->path);
1503 	cam_periph_unlock(periph);
1504 	if (ret == 0)
1505 		bp->bio_completed = bp->bio_length;
1506 	return ret;
1507 }
1508 
1509 static int
1510 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1511 {
1512 	char buf[16];
1513 	const char *p;
1514 	struct ada_softc *softc;
1515 	int i, error, value, methods;
1516 
1517 	softc = (struct ada_softc *)arg1;
1518 
1519 	value = softc->delete_method;
1520 	if (value < 0 || value > ADA_DELETE_MAX)
1521 		p = "UNKNOWN";
1522 	else
1523 		p = ada_delete_method_names[value];
1524 	strncpy(buf, p, sizeof(buf));
1525 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1526 	if (error != 0 || req->newptr == NULL)
1527 		return (error);
1528 	methods = 1 << ADA_DELETE_DISABLE;
1529 	if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1530 	    !(softc->flags & ADA_FLAG_CAN_48BIT))
1531 		methods |= 1 << ADA_DELETE_CFA_ERASE;
1532 	if (softc->flags & ADA_FLAG_CAN_TRIM)
1533 		methods |= 1 << ADA_DELETE_DSM_TRIM;
1534 	if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1535 		methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1536 	for (i = 0; i <= ADA_DELETE_MAX; i++) {
1537 		if (!(methods & (1 << i)) ||
1538 		    strcmp(buf, ada_delete_method_names[i]) != 0)
1539 			continue;
1540 		softc->delete_method = i;
1541 		return (0);
1542 	}
1543 	return (EINVAL);
1544 }
1545 
1546 static void
1547 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1548 {
1549 	if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1550 	    (cgd->inq_flags & SID_DMA))
1551 		softc->flags |= ADA_FLAG_CAN_DMA;
1552 	else
1553 		softc->flags &= ~ADA_FLAG_CAN_DMA;
1554 
1555 	if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1556 		softc->flags |= ADA_FLAG_CAN_48BIT;
1557 		if (cgd->inq_flags & SID_DMA48)
1558 			softc->flags |= ADA_FLAG_CAN_DMA48;
1559 		else
1560 			softc->flags &= ~ADA_FLAG_CAN_DMA48;
1561 	} else
1562 		softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1563 
1564 	if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1565 		softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1566 	else
1567 		softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1568 
1569 	if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1570 		softc->flags |= ADA_FLAG_CAN_POWERMGT;
1571 	else
1572 		softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1573 
1574 	if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1575 	    (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1576 		softc->flags |= ADA_FLAG_CAN_NCQ;
1577 	else
1578 		softc->flags &= ~ADA_FLAG_CAN_NCQ;
1579 
1580 	if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1581 	    (cgd->inq_flags & SID_DMA)) {
1582 		softc->flags |= ADA_FLAG_CAN_TRIM;
1583 		softc->trim_max_ranges = TRIM_MAX_RANGES;
1584 		if (cgd->ident_data.max_dsm_blocks != 0) {
1585 			softc->trim_max_ranges =
1586 			    min(cgd->ident_data.max_dsm_blocks *
1587 				ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1588 		}
1589 		/*
1590 		 * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1591 		 * to do NCQ trims, if we support trims at all. We also need
1592 		 * support from the SIM to do things properly. Perhaps we
1593 		 * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1594 		 * set too...
1595 		 */
1596 		if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1597 		    (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1598 		    (cgd->ident_data.satacapabilities2 &
1599 		     ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1600 		    (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1601 			softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1602 		else
1603 			softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1604 	} else
1605 		softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1606 
1607 	if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1608 		softc->flags |= ADA_FLAG_CAN_CFA;
1609 	else
1610 		softc->flags &= ~ADA_FLAG_CAN_CFA;
1611 
1612 	/*
1613 	 * Now that we've set the appropriate flags, setup the delete
1614 	 * method.
1615 	 */
1616 	adasetdeletemethod(softc);
1617 
1618 	if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1619 	 && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1620 		softc->flags |= ADA_FLAG_CAN_LOG;
1621 	else
1622 		softc->flags &= ~ADA_FLAG_CAN_LOG;
1623 
1624 	if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1625 	     ATA_SUPPORT_ZONE_HOST_AWARE)
1626 		softc->zone_mode = ADA_ZONE_HOST_AWARE;
1627 	else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1628 		   ATA_SUPPORT_ZONE_DEV_MANAGED)
1629 	      || (softc->quirks & ADA_Q_SMR_DM))
1630 		softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1631 	else
1632 		softc->zone_mode = ADA_ZONE_NONE;
1633 
1634 	if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1635 		softc->flags |= ADA_FLAG_CAN_RAHEAD;
1636 	else
1637 		softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1638 
1639 	if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1640 		softc->flags |= ADA_FLAG_CAN_WCACHE;
1641 	else
1642 		softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1643 }
1644 
1645 static cam_status
1646 adaregister(struct cam_periph *periph, void *arg)
1647 {
1648 	struct ada_softc *softc;
1649 	struct ccb_pathinq cpi;
1650 	struct ccb_getdev *cgd;
1651 	char   announce_buf[80];
1652 	struct disk_params *dp;
1653 	caddr_t match;
1654 	u_int maxio;
1655 	int quirks;
1656 
1657 	cgd = (struct ccb_getdev *)arg;
1658 	if (cgd == NULL) {
1659 		printf("adaregister: no getdev CCB, can't register device\n");
1660 		return(CAM_REQ_CMP_ERR);
1661 	}
1662 
1663 	softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1664 	    M_NOWAIT|M_ZERO);
1665 
1666 	if (softc == NULL) {
1667 		printf("adaregister: Unable to probe new device. "
1668 		    "Unable to allocate softc\n");
1669 		return(CAM_REQ_CMP_ERR);
1670 	}
1671 
1672 	if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1673 		printf("adaregister: Unable to probe new device. "
1674 		       "Unable to allocate iosched memory\n");
1675 		free(softc, M_DEVBUF);
1676 		return(CAM_REQ_CMP_ERR);
1677 	}
1678 
1679 	periph->softc = softc;
1680 
1681 	/*
1682 	 * See if this device has any quirks.
1683 	 */
1684 	match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1685 			       (caddr_t)ada_quirk_table,
1686 			       nitems(ada_quirk_table),
1687 			       sizeof(*ada_quirk_table), ata_identify_match);
1688 	if (match != NULL)
1689 		softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1690 	else
1691 		softc->quirks = ADA_Q_NONE;
1692 
1693 	bzero(&cpi, sizeof(cpi));
1694 	xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
1695 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1696 	xpt_action((union ccb *)&cpi);
1697 
1698 	TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1699 
1700 	/*
1701 	 * Register this media as a disk
1702 	 */
1703 	(void)cam_periph_hold(periph, PRIBIO);
1704 	cam_periph_unlock(periph);
1705 	snprintf(announce_buf, sizeof(announce_buf),
1706 	    "kern.cam.ada.%d.quirks", periph->unit_number);
1707 	quirks = softc->quirks;
1708 	TUNABLE_INT_FETCH(announce_buf, &quirks);
1709 	softc->quirks = quirks;
1710 	softc->read_ahead = -1;
1711 	snprintf(announce_buf, sizeof(announce_buf),
1712 	    "kern.cam.ada.%d.read_ahead", periph->unit_number);
1713 	TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1714 	softc->write_cache = -1;
1715 	snprintf(announce_buf, sizeof(announce_buf),
1716 	    "kern.cam.ada.%d.write_cache", periph->unit_number);
1717 	TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1718 
1719 	/*
1720 	 * Set support flags based on the Identify data and quirks.
1721 	 */
1722 	adasetflags(softc, cgd);
1723 
1724 	/* Disable queue sorting for non-rotational media by default. */
1725 	if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1726 		softc->rotating = 0;
1727 	} else {
1728 		softc->rotating = 1;
1729 	}
1730 	cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1731 	adagetparams(periph, cgd);
1732 	softc->disk = disk_alloc();
1733 	softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1734 	softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1735 			  periph->unit_number, softc->params.secsize,
1736 			  DEVSTAT_ALL_SUPPORTED,
1737 			  DEVSTAT_TYPE_DIRECT |
1738 			  XPORT_DEVSTAT_TYPE(cpi.transport),
1739 			  DEVSTAT_PRIORITY_DISK);
1740 	softc->disk->d_open = adaopen;
1741 	softc->disk->d_close = adaclose;
1742 	softc->disk->d_strategy = adastrategy;
1743 	softc->disk->d_getattr = adagetattr;
1744 	softc->disk->d_dump = adadump;
1745 	softc->disk->d_gone = adadiskgonecb;
1746 	softc->disk->d_name = "ada";
1747 	softc->disk->d_drv1 = periph;
1748 	maxio = cpi.maxio;		/* Honor max I/O size of SIM */
1749 	if (maxio == 0)
1750 		maxio = DFLTPHYS;	/* traditional default */
1751 	else if (maxio > MAXPHYS)
1752 		maxio = MAXPHYS;	/* for safety */
1753 	if (softc->flags & ADA_FLAG_CAN_48BIT)
1754 		maxio = min(maxio, 65536 * softc->params.secsize);
1755 	else					/* 28bit ATA command limit */
1756 		maxio = min(maxio, 256 * softc->params.secsize);
1757 	softc->disk->d_maxsize = maxio;
1758 	softc->disk->d_unit = periph->unit_number;
1759 	softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1760 	if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1761 		softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1762 	if (softc->flags & ADA_FLAG_CAN_TRIM) {
1763 		softc->disk->d_flags |= DISKFLAG_CANDELETE;
1764 		softc->disk->d_delmaxsize = softc->params.secsize *
1765 					    ATA_DSM_RANGE_MAX *
1766 					    softc->trim_max_ranges;
1767 	} else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1768 	    !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1769 		softc->disk->d_flags |= DISKFLAG_CANDELETE;
1770 		softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1771 	} else
1772 		softc->disk->d_delmaxsize = maxio;
1773 	if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1774 		softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1775 		softc->unmappedio = 1;
1776 	}
1777 	if (cpi.hba_misc & PIM_ATA_EXT)
1778 		softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1779 	strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1780 	    MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1781 	strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1782 	    MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1783 	softc->disk->d_hba_vendor = cpi.hba_vendor;
1784 	softc->disk->d_hba_device = cpi.hba_device;
1785 	softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1786 	softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1787 
1788 	softc->disk->d_sectorsize = softc->params.secsize;
1789 	softc->disk->d_mediasize = (off_t)softc->params.sectors *
1790 	    softc->params.secsize;
1791 	if (ata_physical_sector_size(&cgd->ident_data) !=
1792 	    softc->params.secsize) {
1793 		softc->disk->d_stripesize =
1794 		    ata_physical_sector_size(&cgd->ident_data);
1795 		softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1796 		    ata_logical_sector_offset(&cgd->ident_data)) %
1797 		    softc->disk->d_stripesize;
1798 	} else if (softc->quirks & ADA_Q_4K) {
1799 		softc->disk->d_stripesize = 4096;
1800 		softc->disk->d_stripeoffset = 0;
1801 	}
1802 	softc->disk->d_fwsectors = softc->params.secs_per_track;
1803 	softc->disk->d_fwheads = softc->params.heads;
1804 	ata_disk_firmware_geom_adjust(softc->disk);
1805 
1806 	/*
1807 	 * Acquire a reference to the periph before we register with GEOM.
1808 	 * We'll release this reference once GEOM calls us back (via
1809 	 * adadiskgonecb()) telling us that our provider has been freed.
1810 	 */
1811 	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
1812 		xpt_print(periph->path, "%s: lost periph during "
1813 			  "registration!\n", __func__);
1814 		cam_periph_lock(periph);
1815 		return (CAM_REQ_CMP_ERR);
1816 	}
1817 	disk_create(softc->disk, DISK_VERSION);
1818 	cam_periph_lock(periph);
1819 
1820 	dp = &softc->params;
1821 	snprintf(announce_buf, sizeof(announce_buf),
1822 	    "%juMB (%ju %u byte sectors)",
1823 	    ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1824 	    (uintmax_t)dp->sectors, dp->secsize);
1825 	xpt_announce_periph(periph, announce_buf);
1826 	xpt_announce_quirks(periph, softc->quirks, ADA_Q_BIT_STRING);
1827 
1828 	/*
1829 	 * Create our sysctl variables, now that we know
1830 	 * we have successfully attached.
1831 	 */
1832 	if (cam_periph_acquire(periph) == CAM_REQ_CMP)
1833 		taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1834 
1835 	/*
1836 	 * Add async callbacks for bus reset and
1837 	 * bus device reset calls.  I don't bother
1838 	 * checking if this fails as, in most cases,
1839 	 * the system will function just fine without
1840 	 * them and the only alternative would be to
1841 	 * not attach the device on failure.
1842 	 */
1843 	xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1844 	    AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1845 	    adaasync, periph, periph->path);
1846 
1847 	/*
1848 	 * Schedule a periodic event to occasionally send an
1849 	 * ordered tag to a device.
1850 	 */
1851 	callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1852 	callout_reset(&softc->sendordered_c,
1853 	    (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1854 	    adasendorderedtag, softc);
1855 
1856 	if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1857 		softc->state = ADA_STATE_RAHEAD;
1858 	} else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1859 		softc->state = ADA_STATE_WCACHE;
1860 	} else if ((softc->flags & ADA_FLAG_CAN_LOG)
1861 		&& (softc->zone_mode != ADA_ZONE_NONE)) {
1862 		softc->state = ADA_STATE_LOGDIR;
1863 	} else {
1864 		/*
1865 		 * Nothing to probe, so we can just transition to the
1866 		 * normal state.
1867 		 */
1868 		adaprobedone(periph, NULL);
1869 		return(CAM_REQ_CMP);
1870 	}
1871 
1872 	xpt_schedule(periph, CAM_PRIORITY_DEV);
1873 
1874 	return(CAM_REQ_CMP);
1875 }
1876 
1877 static int
1878 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1879 {
1880 	uint64_t lastlba = (uint64_t)-1;
1881 	int c, lastcount = 0, off, ranges = 0;
1882 
1883 	bzero(req, sizeof(*req));
1884 	TAILQ_INIT(&req->bps);
1885 	do {
1886 		uint64_t lba = bp->bio_pblkno;
1887 		int count = bp->bio_bcount / softc->params.secsize;
1888 
1889 		/* Try to extend the previous range. */
1890 		if (lba == lastlba) {
1891 			c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1892 			lastcount += c;
1893 			off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1894 			req->data[off + 6] = lastcount & 0xff;
1895 			req->data[off + 7] =
1896 				(lastcount >> 8) & 0xff;
1897 			count -= c;
1898 			lba += c;
1899 		}
1900 
1901 		while (count > 0) {
1902 			c = min(count, ATA_DSM_RANGE_MAX);
1903 			off = ranges * ATA_DSM_RANGE_SIZE;
1904 			req->data[off + 0] = lba & 0xff;
1905 			req->data[off + 1] = (lba >> 8) & 0xff;
1906 			req->data[off + 2] = (lba >> 16) & 0xff;
1907 			req->data[off + 3] = (lba >> 24) & 0xff;
1908 			req->data[off + 4] = (lba >> 32) & 0xff;
1909 			req->data[off + 5] = (lba >> 40) & 0xff;
1910 			req->data[off + 6] = c & 0xff;
1911 			req->data[off + 7] = (c >> 8) & 0xff;
1912 			lba += c;
1913 			count -= c;
1914 			lastcount = c;
1915 			ranges++;
1916 			/*
1917 			 * Its the caller's responsibility to ensure the
1918 			 * request will fit so we don't need to check for
1919 			 * overrun here
1920 			 */
1921 		}
1922 		lastlba = lba;
1923 		TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1924 
1925 		bp = cam_iosched_next_trim(softc->cam_iosched);
1926 		if (bp == NULL)
1927 			break;
1928 		if (bp->bio_bcount / softc->params.secsize >
1929 		    (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1930 			cam_iosched_put_back_trim(softc->cam_iosched, bp);
1931 			break;
1932 		}
1933 	} while (1);
1934 
1935 	return (ranges);
1936 }
1937 
1938 static void
1939 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1940 {
1941 	struct trim_request *req = &softc->trim_req;
1942 	int ranges;
1943 
1944 	ranges = ada_dsmtrim_req_create(softc, bp, req);
1945 	cam_fill_ataio(ataio,
1946 	    ada_retry_count,
1947 	    adadone,
1948 	    CAM_DIR_OUT,
1949 	    0,
1950 	    req->data,
1951 	    howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1952 	    ada_default_timeout * 1000);
1953 	ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1954 	    ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
1955 }
1956 
1957 static void
1958 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1959 {
1960 	struct trim_request *req = &softc->trim_req;
1961 	int ranges;
1962 
1963 	ranges = ada_dsmtrim_req_create(softc, bp, req);
1964 	cam_fill_ataio(ataio,
1965 	    ada_retry_count,
1966 	    adadone,
1967 	    CAM_DIR_OUT,
1968 	    0,
1969 	    req->data,
1970 	    howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1971 	    ada_default_timeout * 1000);
1972 	ata_ncq_cmd(ataio,
1973 	    ATA_SEND_FPDMA_QUEUED,
1974 	    0,
1975 	    howmany(ranges, ATA_DSM_BLK_RANGES));
1976 	ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
1977 	ataio->ata_flags |= ATA_FLAG_AUX;
1978 	ataio->aux = 1;
1979 }
1980 
1981 static void
1982 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1983 {
1984 	struct trim_request *req = &softc->trim_req;
1985 	uint64_t lba = bp->bio_pblkno;
1986 	uint16_t count = bp->bio_bcount / softc->params.secsize;
1987 
1988 	bzero(req, sizeof(*req));
1989 	TAILQ_INIT(&req->bps);
1990 	TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1991 
1992 	cam_fill_ataio(ataio,
1993 	    ada_retry_count,
1994 	    adadone,
1995 	    CAM_DIR_NONE,
1996 	    0,
1997 	    NULL,
1998 	    0,
1999 	    ada_default_timeout*1000);
2000 
2001 	if (count >= 256)
2002 		count = 0;
2003 	ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2004 }
2005 
2006 static int
2007 ada_zone_bio_to_ata(int disk_zone_cmd)
2008 {
2009 	switch (disk_zone_cmd) {
2010 	case DISK_ZONE_OPEN:
2011 		return ATA_ZM_OPEN_ZONE;
2012 	case DISK_ZONE_CLOSE:
2013 		return ATA_ZM_CLOSE_ZONE;
2014 	case DISK_ZONE_FINISH:
2015 		return ATA_ZM_FINISH_ZONE;
2016 	case DISK_ZONE_RWP:
2017 		return ATA_ZM_RWP;
2018 	}
2019 
2020 	return -1;
2021 }
2022 
2023 static int
2024 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2025 	     int *queue_ccb)
2026 {
2027 	struct ada_softc *softc;
2028 	int error;
2029 
2030 	error = 0;
2031 
2032 	if (bp->bio_cmd != BIO_ZONE) {
2033 		error = EINVAL;
2034 		goto bailout;
2035 	}
2036 
2037 	softc = periph->softc;
2038 
2039 	switch (bp->bio_zone.zone_cmd) {
2040 	case DISK_ZONE_OPEN:
2041 	case DISK_ZONE_CLOSE:
2042 	case DISK_ZONE_FINISH:
2043 	case DISK_ZONE_RWP: {
2044 		int zone_flags;
2045 		int zone_sa;
2046 		uint64_t lba;
2047 
2048 		zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2049 		if (zone_sa == -1) {
2050 			xpt_print(periph->path, "Cannot translate zone "
2051 			    "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2052 			error = EINVAL;
2053 			goto bailout;
2054 		}
2055 
2056 		zone_flags = 0;
2057 		lba = bp->bio_zone.zone_params.rwp.id;
2058 
2059 		if (bp->bio_zone.zone_params.rwp.flags &
2060 		    DISK_ZONE_RWP_FLAG_ALL)
2061 			zone_flags |= ZBC_OUT_ALL;
2062 
2063 		ata_zac_mgmt_out(&ccb->ataio,
2064 				 /*retries*/ ada_retry_count,
2065 				 /*cbfcnp*/ adadone,
2066 				 /*use_ncq*/ (softc->flags &
2067 					      ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2068 				 /*zm_action*/ zone_sa,
2069 				 /*zone_id*/ lba,
2070 				 /*zone_flags*/ zone_flags,
2071 				 /*sector_count*/ 0,
2072 				 /*data_ptr*/ NULL,
2073 				 /*dxfer_len*/ 0,
2074 				 /*timeout*/ ada_default_timeout * 1000);
2075 		*queue_ccb = 1;
2076 
2077 		break;
2078 	}
2079 	case DISK_ZONE_REPORT_ZONES: {
2080 		uint8_t *rz_ptr;
2081 		uint32_t num_entries, alloc_size;
2082 		struct disk_zone_report *rep;
2083 
2084 		rep = &bp->bio_zone.zone_params.report;
2085 
2086 		num_entries = rep->entries_allocated;
2087 		if (num_entries == 0) {
2088 			xpt_print(periph->path, "No entries allocated for "
2089 			    "Report Zones request\n");
2090 			error = EINVAL;
2091 			goto bailout;
2092 		}
2093 		alloc_size = sizeof(struct scsi_report_zones_hdr) +
2094 		    (sizeof(struct scsi_report_zones_desc) * num_entries);
2095 		alloc_size = min(alloc_size, softc->disk->d_maxsize);
2096 		rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2097 		if (rz_ptr == NULL) {
2098 			xpt_print(periph->path, "Unable to allocate memory "
2099 			   "for Report Zones request\n");
2100 			error = ENOMEM;
2101 			goto bailout;
2102 		}
2103 
2104 		ata_zac_mgmt_in(&ccb->ataio,
2105 				/*retries*/ ada_retry_count,
2106 				/*cbcfnp*/ adadone,
2107 				/*use_ncq*/ (softc->flags &
2108 					     ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2109 				/*zm_action*/ ATA_ZM_REPORT_ZONES,
2110 				/*zone_id*/ rep->starting_id,
2111 				/*zone_flags*/ rep->rep_options,
2112 				/*data_ptr*/ rz_ptr,
2113 				/*dxfer_len*/ alloc_size,
2114 				/*timeout*/ ada_default_timeout * 1000);
2115 
2116 		/*
2117 		 * For BIO_ZONE, this isn't normally needed.  However, it
2118 		 * is used by devstat_end_transaction_bio() to determine
2119 		 * how much data was transferred.
2120 		 */
2121 		/*
2122 		 * XXX KDM we have a problem.  But I'm not sure how to fix
2123 		 * it.  devstat uses bio_bcount - bio_resid to calculate
2124 		 * the amount of data transferred.   The GEOM disk code
2125 		 * uses bio_length - bio_resid to calculate the amount of
2126 		 * data in bio_completed.  We have different structure
2127 		 * sizes above and below the ada(4) driver.  So, if we
2128 		 * use the sizes above, the amount transferred won't be
2129 		 * quite accurate for devstat.  If we use different sizes
2130 		 * for bio_bcount and bio_length (above and below
2131 		 * respectively), then the residual needs to match one or
2132 		 * the other.  Everything is calculated after the bio
2133 		 * leaves the driver, so changing the values around isn't
2134 		 * really an option.  For now, just set the count to the
2135 		 * passed in length.  This means that the calculations
2136 		 * above (e.g. bio_completed) will be correct, but the
2137 		 * amount of data reported to devstat will be slightly
2138 		 * under or overstated.
2139 		 */
2140 		bp->bio_bcount = bp->bio_length;
2141 
2142 		*queue_ccb = 1;
2143 
2144 		break;
2145 	}
2146 	case DISK_ZONE_GET_PARAMS: {
2147 		struct disk_zone_disk_params *params;
2148 
2149 		params = &bp->bio_zone.zone_params.disk_params;
2150 		bzero(params, sizeof(*params));
2151 
2152 		switch (softc->zone_mode) {
2153 		case ADA_ZONE_DRIVE_MANAGED:
2154 			params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2155 			break;
2156 		case ADA_ZONE_HOST_AWARE:
2157 			params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2158 			break;
2159 		case ADA_ZONE_HOST_MANAGED:
2160 			params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2161 			break;
2162 		default:
2163 		case ADA_ZONE_NONE:
2164 			params->zone_mode = DISK_ZONE_MODE_NONE;
2165 			break;
2166 		}
2167 
2168 		if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2169 			params->flags |= DISK_ZONE_DISK_URSWRZ;
2170 
2171 		if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2172 			params->optimal_seq_zones = softc->optimal_seq_zones;
2173 			params->flags |= DISK_ZONE_OPT_SEQ_SET;
2174 		}
2175 
2176 		if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2177 			params->optimal_nonseq_zones =
2178 			    softc->optimal_nonseq_zones;
2179 			params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2180 		}
2181 
2182 		if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2183 			params->max_seq_zones = softc->max_seq_zones;
2184 			params->flags |= DISK_ZONE_MAX_SEQ_SET;
2185 		}
2186 		if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2187 			params->flags |= DISK_ZONE_RZ_SUP;
2188 
2189 		if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2190 			params->flags |= DISK_ZONE_OPEN_SUP;
2191 
2192 		if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2193 			params->flags |= DISK_ZONE_CLOSE_SUP;
2194 
2195 		if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2196 			params->flags |= DISK_ZONE_FINISH_SUP;
2197 
2198 		if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2199 			params->flags |= DISK_ZONE_RWP_SUP;
2200 		break;
2201 	}
2202 	default:
2203 		break;
2204 	}
2205 bailout:
2206 	return (error);
2207 }
2208 
2209 static void
2210 adastart(struct cam_periph *periph, union ccb *start_ccb)
2211 {
2212 	struct ada_softc *softc = (struct ada_softc *)periph->softc;
2213 	struct ccb_ataio *ataio = &start_ccb->ataio;
2214 
2215 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2216 
2217 	switch (softc->state) {
2218 	case ADA_STATE_NORMAL:
2219 	{
2220 		struct bio *bp;
2221 		u_int8_t tag_code;
2222 
2223 		bp = cam_iosched_next_bio(softc->cam_iosched);
2224 		if (bp == NULL) {
2225 			xpt_release_ccb(start_ccb);
2226 			break;
2227 		}
2228 
2229 		if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2230 		    (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2231 			softc->flags &= ~ADA_FLAG_NEED_OTAG;
2232 			softc->flags |= ADA_FLAG_WAS_OTAG;
2233 			tag_code = 0;
2234 		} else {
2235 			tag_code = 1;
2236 		}
2237 		switch (bp->bio_cmd) {
2238 		case BIO_WRITE:
2239 		case BIO_READ:
2240 		{
2241 			uint64_t lba = bp->bio_pblkno;
2242 			uint16_t count = bp->bio_bcount / softc->params.secsize;
2243 			void *data_ptr;
2244 			int rw_op;
2245 
2246 			if (bp->bio_cmd == BIO_WRITE) {
2247 				softc->flags |= ADA_FLAG_DIRTY;
2248 				rw_op = CAM_DIR_OUT;
2249 			} else {
2250 				rw_op = CAM_DIR_IN;
2251 			}
2252 
2253 			data_ptr = bp->bio_data;
2254 			if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2255 				rw_op |= CAM_DATA_BIO;
2256 				data_ptr = bp;
2257 			}
2258 
2259 #ifdef ADA_TEST_FAILURE
2260 			int fail = 0;
2261 
2262 			/*
2263 			 * Support the failure ioctls.  If the command is a
2264 			 * read, and there are pending forced read errors, or
2265 			 * if a write and pending write errors, then fail this
2266 			 * operation with EIO.  This is useful for testing
2267 			 * purposes.  Also, support having every Nth read fail.
2268 			 *
2269 			 * This is a rather blunt tool.
2270 			 */
2271 			if (bp->bio_cmd == BIO_READ) {
2272 				if (softc->force_read_error) {
2273 					softc->force_read_error--;
2274 					fail = 1;
2275 				}
2276 				if (softc->periodic_read_error > 0) {
2277 					if (++softc->periodic_read_count >=
2278 					    softc->periodic_read_error) {
2279 						softc->periodic_read_count = 0;
2280 						fail = 1;
2281 					}
2282 				}
2283 			} else {
2284 				if (softc->force_write_error) {
2285 					softc->force_write_error--;
2286 					fail = 1;
2287 				}
2288 			}
2289 			if (fail) {
2290 				biofinish(bp, NULL, EIO);
2291 				xpt_release_ccb(start_ccb);
2292 				adaschedule(periph);
2293 				return;
2294 			}
2295 #endif
2296 			KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2297 			    round_page(bp->bio_bcount + bp->bio_ma_offset) /
2298 			    PAGE_SIZE == bp->bio_ma_n,
2299 			    ("Short bio %p", bp));
2300 			cam_fill_ataio(ataio,
2301 			    ada_retry_count,
2302 			    adadone,
2303 			    rw_op,
2304 			    0,
2305 			    data_ptr,
2306 			    bp->bio_bcount,
2307 			    ada_default_timeout*1000);
2308 
2309 			if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2310 				if (bp->bio_cmd == BIO_READ) {
2311 					ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2312 					    lba, count);
2313 				} else {
2314 					ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2315 					    lba, count);
2316 				}
2317 			} else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2318 			    (lba + count >= ATA_MAX_28BIT_LBA ||
2319 			    count > 256)) {
2320 				if (softc->flags & ADA_FLAG_CAN_DMA48) {
2321 					if (bp->bio_cmd == BIO_READ) {
2322 						ata_48bit_cmd(ataio, ATA_READ_DMA48,
2323 						    0, lba, count);
2324 					} else {
2325 						ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2326 						    0, lba, count);
2327 					}
2328 				} else {
2329 					if (bp->bio_cmd == BIO_READ) {
2330 						ata_48bit_cmd(ataio, ATA_READ_MUL48,
2331 						    0, lba, count);
2332 					} else {
2333 						ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2334 						    0, lba, count);
2335 					}
2336 				}
2337 			} else {
2338 				if (count == 256)
2339 					count = 0;
2340 				if (softc->flags & ADA_FLAG_CAN_DMA) {
2341 					if (bp->bio_cmd == BIO_READ) {
2342 						ata_28bit_cmd(ataio, ATA_READ_DMA,
2343 						    0, lba, count);
2344 					} else {
2345 						ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2346 						    0, lba, count);
2347 					}
2348 				} else {
2349 					if (bp->bio_cmd == BIO_READ) {
2350 						ata_28bit_cmd(ataio, ATA_READ_MUL,
2351 						    0, lba, count);
2352 					} else {
2353 						ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2354 						    0, lba, count);
2355 					}
2356 				}
2357 			}
2358 			break;
2359 		}
2360 		case BIO_DELETE:
2361 			switch (softc->delete_method) {
2362 			case ADA_DELETE_NCQ_DSM_TRIM:
2363 				ada_ncq_dsmtrim(softc, bp, ataio);
2364 				break;
2365 			case ADA_DELETE_DSM_TRIM:
2366 				ada_dsmtrim(softc, bp, ataio);
2367 				break;
2368 			case ADA_DELETE_CFA_ERASE:
2369 				ada_cfaerase(softc, bp, ataio);
2370 				break;
2371 			default:
2372 				biofinish(bp, NULL, EOPNOTSUPP);
2373 				xpt_release_ccb(start_ccb);
2374 				adaschedule(periph);
2375 				return;
2376 			}
2377 			start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2378 			start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2379 			cam_iosched_submit_trim(softc->cam_iosched);
2380 			goto out;
2381 		case BIO_FLUSH:
2382 			cam_fill_ataio(ataio,
2383 			    1,
2384 			    adadone,
2385 			    CAM_DIR_NONE,
2386 			    0,
2387 			    NULL,
2388 			    0,
2389 			    ada_default_timeout*1000);
2390 
2391 			if (softc->flags & ADA_FLAG_CAN_48BIT)
2392 				ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2393 			else
2394 				ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2395 			break;
2396 		case BIO_ZONE: {
2397 			int error, queue_ccb;
2398 
2399 			queue_ccb = 0;
2400 
2401 			error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2402 			if ((error != 0)
2403 			 || (queue_ccb == 0)) {
2404 				biofinish(bp, NULL, error);
2405 				xpt_release_ccb(start_ccb);
2406 				return;
2407 			}
2408 			break;
2409 		}
2410 		}
2411 		start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2412 		start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2413 out:
2414 		start_ccb->ccb_h.ccb_bp = bp;
2415 		softc->outstanding_cmds++;
2416 		softc->refcount++;
2417 		cam_periph_unlock(periph);
2418 		xpt_action(start_ccb);
2419 		cam_periph_lock(periph);
2420 		softc->refcount--;
2421 
2422 		/* May have more work to do, so ensure we stay scheduled */
2423 		adaschedule(periph);
2424 		break;
2425 	}
2426 	case ADA_STATE_RAHEAD:
2427 	case ADA_STATE_WCACHE:
2428 	{
2429 		cam_fill_ataio(ataio,
2430 		    1,
2431 		    adadone,
2432 		    CAM_DIR_NONE,
2433 		    0,
2434 		    NULL,
2435 		    0,
2436 		    ada_default_timeout*1000);
2437 
2438 		if (softc->state == ADA_STATE_RAHEAD) {
2439 			ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2440 			    ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2441 			start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2442 		} else {
2443 			ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2444 			    ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2445 			start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2446 		}
2447 		start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2448 		xpt_action(start_ccb);
2449 		break;
2450 	}
2451 	case ADA_STATE_LOGDIR:
2452 	{
2453 		struct ata_gp_log_dir *log_dir;
2454 
2455 		if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2456 			adaprobedone(periph, start_ccb);
2457 			break;
2458 		}
2459 
2460 		log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2461 		if (log_dir == NULL) {
2462 			xpt_print(periph->path, "Couldn't malloc log_dir "
2463 			    "data\n");
2464 			softc->state = ADA_STATE_NORMAL;
2465 			xpt_release_ccb(start_ccb);
2466 			break;
2467 		}
2468 
2469 
2470 		ata_read_log(ataio,
2471 		    /*retries*/1,
2472 		    /*cbfcnp*/adadone,
2473 		    /*log_address*/ ATA_LOG_DIRECTORY,
2474 		    /*page_number*/ 0,
2475 		    /*block_count*/ 1,
2476 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2477 				 CAM_ATAIO_DMA : 0,
2478 		    /*data_ptr*/ (uint8_t *)log_dir,
2479 		    /*dxfer_len*/sizeof(*log_dir),
2480 		    /*timeout*/ada_default_timeout*1000);
2481 
2482 		start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2483 		xpt_action(start_ccb);
2484 		break;
2485 	}
2486 	case ADA_STATE_IDDIR:
2487 	{
2488 		struct ata_identify_log_pages *id_dir;
2489 
2490 		id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2491 		if (id_dir == NULL) {
2492 			xpt_print(periph->path, "Couldn't malloc id_dir "
2493 			    "data\n");
2494 			adaprobedone(periph, start_ccb);
2495 			break;
2496 		}
2497 
2498 		ata_read_log(ataio,
2499 		    /*retries*/1,
2500 		    /*cbfcnp*/adadone,
2501 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2502 		    /*page_number*/ ATA_IDL_PAGE_LIST,
2503 		    /*block_count*/ 1,
2504 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2505 				 CAM_ATAIO_DMA : 0,
2506 		    /*data_ptr*/ (uint8_t *)id_dir,
2507 		    /*dxfer_len*/ sizeof(*id_dir),
2508 		    /*timeout*/ada_default_timeout*1000);
2509 
2510 		start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2511 		xpt_action(start_ccb);
2512 		break;
2513 	}
2514 	case ADA_STATE_SUP_CAP:
2515 	{
2516 		struct ata_identify_log_sup_cap *sup_cap;
2517 
2518 		sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2519 		if (sup_cap == NULL) {
2520 			xpt_print(periph->path, "Couldn't malloc sup_cap "
2521 			    "data\n");
2522 			adaprobedone(periph, start_ccb);
2523 			break;
2524 		}
2525 
2526 		ata_read_log(ataio,
2527 		    /*retries*/1,
2528 		    /*cbfcnp*/adadone,
2529 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2530 		    /*page_number*/ ATA_IDL_SUP_CAP,
2531 		    /*block_count*/ 1,
2532 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2533 				 CAM_ATAIO_DMA : 0,
2534 		    /*data_ptr*/ (uint8_t *)sup_cap,
2535 		    /*dxfer_len*/ sizeof(*sup_cap),
2536 		    /*timeout*/ada_default_timeout*1000);
2537 
2538 		start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2539 		xpt_action(start_ccb);
2540 		break;
2541 	}
2542 	case ADA_STATE_ZONE:
2543 	{
2544 		struct ata_zoned_info_log *ata_zone;
2545 
2546 		ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2547 		if (ata_zone == NULL) {
2548 			xpt_print(periph->path, "Couldn't malloc ata_zone "
2549 			    "data\n");
2550 			adaprobedone(periph, start_ccb);
2551 			break;
2552 		}
2553 
2554 		ata_read_log(ataio,
2555 		    /*retries*/1,
2556 		    /*cbfcnp*/adadone,
2557 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2558 		    /*page_number*/ ATA_IDL_ZDI,
2559 		    /*block_count*/ 1,
2560 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2561 				 CAM_ATAIO_DMA : 0,
2562 		    /*data_ptr*/ (uint8_t *)ata_zone,
2563 		    /*dxfer_len*/ sizeof(*ata_zone),
2564 		    /*timeout*/ada_default_timeout*1000);
2565 
2566 		start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2567 		xpt_action(start_ccb);
2568 		break;
2569 	}
2570 	}
2571 }
2572 
2573 static void
2574 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2575 {
2576 	struct ada_softc *softc;
2577 
2578 	softc = (struct ada_softc *)periph->softc;
2579 
2580 	if (ccb != NULL)
2581 		xpt_release_ccb(ccb);
2582 
2583 	softc->state = ADA_STATE_NORMAL;
2584 	softc->flags |= ADA_FLAG_PROBED;
2585 	adaschedule(periph);
2586 	if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2587 		softc->flags |= ADA_FLAG_ANNOUNCED;
2588 		cam_periph_unhold(periph);
2589 	} else {
2590 		cam_periph_release_locked(periph);
2591 	}
2592 }
2593 
2594 static void
2595 adazonedone(struct cam_periph *periph, union ccb *ccb)
2596 {
2597 	struct ada_softc *softc;
2598 	struct bio *bp;
2599 
2600 	softc = periph->softc;
2601 	bp = (struct bio *)ccb->ccb_h.ccb_bp;
2602 
2603 	switch (bp->bio_zone.zone_cmd) {
2604 	case DISK_ZONE_OPEN:
2605 	case DISK_ZONE_CLOSE:
2606 	case DISK_ZONE_FINISH:
2607 	case DISK_ZONE_RWP:
2608 		break;
2609 	case DISK_ZONE_REPORT_ZONES: {
2610 		uint32_t avail_len;
2611 		struct disk_zone_report *rep;
2612 		struct scsi_report_zones_hdr *hdr;
2613 		struct scsi_report_zones_desc *desc;
2614 		struct disk_zone_rep_entry *entry;
2615 		uint32_t num_alloced, hdr_len, num_avail;
2616 		uint32_t num_to_fill, i;
2617 
2618 		rep = &bp->bio_zone.zone_params.report;
2619 		avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2620 		/*
2621 		 * Note that bio_resid isn't normally used for zone
2622 		 * commands, but it is used by devstat_end_transaction_bio()
2623 		 * to determine how much data was transferred.  Because
2624 		 * the size of the SCSI/ATA data structures is different
2625 		 * than the size of the BIO interface structures, the
2626 		 * amount of data actually transferred from the drive will
2627 		 * be different than the amount of data transferred to
2628 		 * the user.
2629 		 */
2630 		num_alloced = rep->entries_allocated;
2631 		hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2632 		if (avail_len < sizeof(*hdr)) {
2633 			/*
2634 			 * Is there a better error than EIO here?  We asked
2635 			 * for at least the header, and we got less than
2636 			 * that.
2637 			 */
2638 			bp->bio_error = EIO;
2639 			bp->bio_flags |= BIO_ERROR;
2640 			bp->bio_resid = bp->bio_bcount;
2641 			break;
2642 		}
2643 
2644 		hdr_len = le32dec(hdr->length);
2645 		if (hdr_len > 0)
2646 			rep->entries_available = hdr_len / sizeof(*desc);
2647 		else
2648 			rep->entries_available = 0;
2649 		/*
2650 		 * NOTE: using the same values for the BIO version of the
2651 		 * same field as the SCSI/ATA values.  This means we could
2652 		 * get some additional values that aren't defined in bio.h
2653 		 * if more values of the same field are defined later.
2654 		 */
2655 		rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2656 		rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2657 		/*
2658 		 * If the drive reports no entries that match the query,
2659 		 * we're done.
2660 		 */
2661 		if (hdr_len == 0) {
2662 			rep->entries_filled = 0;
2663 			bp->bio_resid = bp->bio_bcount;
2664 			break;
2665 		}
2666 
2667 		num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2668 				hdr_len / sizeof(*desc));
2669 		/*
2670 		 * If the drive didn't return any data, then we're done.
2671 		 */
2672 		if (num_avail == 0) {
2673 			rep->entries_filled = 0;
2674 			bp->bio_resid = bp->bio_bcount;
2675 			break;
2676 		}
2677 
2678 		num_to_fill = min(num_avail, rep->entries_allocated);
2679 		/*
2680 		 * If the user didn't allocate any entries for us to fill,
2681 		 * we're done.
2682 		 */
2683 		if (num_to_fill == 0) {
2684 			rep->entries_filled = 0;
2685 			bp->bio_resid = bp->bio_bcount;
2686 			break;
2687 		}
2688 
2689 		for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2690 		     i < num_to_fill; i++, desc++, entry++) {
2691 			/*
2692 			 * NOTE: we're mapping the values here directly
2693 			 * from the SCSI/ATA bit definitions to the bio.h
2694 			 * definitions.  There is also a warning in
2695 			 * disk_zone.h, but the impact is that if
2696 			 * additional values are added in the SCSI/ATA
2697 			 * specs these will be visible to consumers of
2698 			 * this interface.
2699 			 */
2700 			entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2701 			entry->zone_condition =
2702 			    (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2703 			    SRZ_ZONE_COND_SHIFT;
2704 			entry->zone_flags |= desc->zone_flags &
2705 			    (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2706 			entry->zone_length = le64dec(desc->zone_length);
2707 			entry->zone_start_lba = le64dec(desc->zone_start_lba);
2708 			entry->write_pointer_lba =
2709 			    le64dec(desc->write_pointer_lba);
2710 		}
2711 		rep->entries_filled = num_to_fill;
2712 		/*
2713 		 * Note that this residual is accurate from the user's
2714 		 * standpoint, but the amount transferred isn't accurate
2715 		 * from the standpoint of what actually came back from the
2716 		 * drive.
2717 		 */
2718 		bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2719 		break;
2720 	}
2721 	case DISK_ZONE_GET_PARAMS:
2722 	default:
2723 		/*
2724 		 * In theory we should not get a GET_PARAMS bio, since it
2725 		 * should be handled without queueing the command to the
2726 		 * drive.
2727 		 */
2728 		panic("%s: Invalid zone command %d", __func__,
2729 		    bp->bio_zone.zone_cmd);
2730 		break;
2731 	}
2732 
2733 	if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2734 		free(ccb->ataio.data_ptr, M_ATADA);
2735 }
2736 
2737 
2738 static void
2739 adadone(struct cam_periph *periph, union ccb *done_ccb)
2740 {
2741 	struct ada_softc *softc;
2742 	struct ccb_ataio *ataio;
2743 	struct cam_path *path;
2744 	uint32_t priority;
2745 	int state;
2746 
2747 	softc = (struct ada_softc *)periph->softc;
2748 	ataio = &done_ccb->ataio;
2749 	path = done_ccb->ccb_h.path;
2750 	priority = done_ccb->ccb_h.pinfo.priority;
2751 
2752 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2753 
2754 	state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2755 	switch (state) {
2756 	case ADA_CCB_BUFFER_IO:
2757 	case ADA_CCB_TRIM:
2758 	{
2759 		struct bio *bp;
2760 		int error;
2761 
2762 		cam_periph_lock(periph);
2763 		bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2764 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2765 			error = adaerror(done_ccb, 0, 0);
2766 			if (error == ERESTART) {
2767 				/* A retry was scheduled, so just return. */
2768 				cam_periph_unlock(periph);
2769 				return;
2770 			}
2771 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2772 				cam_release_devq(path,
2773 						 /*relsim_flags*/0,
2774 						 /*reduction*/0,
2775 						 /*timeout*/0,
2776 						 /*getcount_only*/0);
2777 			/*
2778 			 * If we get an error on an NCQ DSM TRIM, fall back
2779 			 * to a non-NCQ DSM TRIM forever. Please note that if
2780 			 * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2781 			 * However, for this one trim, we treat it as advisory
2782 			 * and return success up the stack.
2783 			 */
2784 			if (state == ADA_CCB_TRIM &&
2785 			    error != 0 &&
2786 			    (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2787 				softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2788 				error = 0;
2789 				adasetdeletemethod(softc);
2790 			}
2791 		} else {
2792 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2793 				panic("REQ_CMP with QFRZN");
2794 
2795 			error = 0;
2796 		}
2797 		bp->bio_error = error;
2798 		if (error != 0) {
2799 			bp->bio_resid = bp->bio_bcount;
2800 			bp->bio_flags |= BIO_ERROR;
2801 		} else {
2802 			if (bp->bio_cmd == BIO_ZONE)
2803 				adazonedone(periph, done_ccb);
2804 			else if (state == ADA_CCB_TRIM)
2805 				bp->bio_resid = 0;
2806 			else
2807 				bp->bio_resid = ataio->resid;
2808 
2809 			if ((bp->bio_resid > 0)
2810 			 && (bp->bio_cmd != BIO_ZONE))
2811 				bp->bio_flags |= BIO_ERROR;
2812 		}
2813 		softc->outstanding_cmds--;
2814 		if (softc->outstanding_cmds == 0)
2815 			softc->flags |= ADA_FLAG_WAS_OTAG;
2816 
2817 		cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2818 		xpt_release_ccb(done_ccb);
2819 		if (state == ADA_CCB_TRIM) {
2820 			TAILQ_HEAD(, bio) queue;
2821 			struct bio *bp1;
2822 
2823 			TAILQ_INIT(&queue);
2824 			TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2825 			/*
2826 			 * Normally, the xpt_release_ccb() above would make sure
2827 			 * that when we have more work to do, that work would
2828 			 * get kicked off. However, we specifically keep
2829 			 * trim_running set to 0 before the call above to allow
2830 			 * other I/O to progress when many BIO_DELETE requests
2831 			 * are pushed down. We set trim_running to 0 and call
2832 			 * daschedule again so that we don't stall if there are
2833 			 * no other I/Os pending apart from BIO_DELETEs.
2834 			 */
2835 			cam_iosched_trim_done(softc->cam_iosched);
2836 			adaschedule(periph);
2837 			cam_periph_unlock(periph);
2838 			while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2839 				TAILQ_REMOVE(&queue, bp1, bio_queue);
2840 				bp1->bio_error = error;
2841 				if (error != 0) {
2842 					bp1->bio_flags |= BIO_ERROR;
2843 					bp1->bio_resid = bp1->bio_bcount;
2844 				} else
2845 					bp1->bio_resid = 0;
2846 				biodone(bp1);
2847 			}
2848 		} else {
2849 			adaschedule(periph);
2850 			cam_periph_unlock(periph);
2851 			biodone(bp);
2852 		}
2853 		return;
2854 	}
2855 	case ADA_CCB_RAHEAD:
2856 	{
2857 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2858 			if (adaerror(done_ccb, 0, 0) == ERESTART) {
2859 				/* Drop freeze taken due to CAM_DEV_QFREEZE */
2860 				cam_release_devq(path, 0, 0, 0, FALSE);
2861 				return;
2862 			} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2863 				cam_release_devq(path,
2864 				    /*relsim_flags*/0,
2865 				    /*reduction*/0,
2866 				    /*timeout*/0,
2867 				    /*getcount_only*/0);
2868 			}
2869 		}
2870 
2871 		/*
2872 		 * Since our peripheral may be invalidated by an error
2873 		 * above or an external event, we must release our CCB
2874 		 * before releasing the reference on the peripheral.
2875 		 * The peripheral will only go away once the last reference
2876 		 * is removed, and we need it around for the CCB release
2877 		 * operation.
2878 		 */
2879 
2880 		xpt_release_ccb(done_ccb);
2881 		softc->state = ADA_STATE_WCACHE;
2882 		xpt_schedule(periph, priority);
2883 		/* Drop freeze taken due to CAM_DEV_QFREEZE */
2884 		cam_release_devq(path, 0, 0, 0, FALSE);
2885 		return;
2886 	}
2887 	case ADA_CCB_WCACHE:
2888 	{
2889 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2890 			if (adaerror(done_ccb, 0, 0) == ERESTART) {
2891 				/* Drop freeze taken due to CAM_DEV_QFREEZE */
2892 				cam_release_devq(path, 0, 0, 0, FALSE);
2893 				return;
2894 			} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2895 				cam_release_devq(path,
2896 				    /*relsim_flags*/0,
2897 				    /*reduction*/0,
2898 				    /*timeout*/0,
2899 				    /*getcount_only*/0);
2900 			}
2901 		}
2902 
2903 		/* Drop freeze taken due to CAM_DEV_QFREEZE */
2904 		cam_release_devq(path, 0, 0, 0, FALSE);
2905 
2906 		if ((softc->flags & ADA_FLAG_CAN_LOG)
2907 		 && (softc->zone_mode != ADA_ZONE_NONE)) {
2908 			xpt_release_ccb(done_ccb);
2909 			softc->state = ADA_STATE_LOGDIR;
2910 			xpt_schedule(periph, priority);
2911 		} else {
2912 			adaprobedone(periph, done_ccb);
2913 		}
2914 		return;
2915 	}
2916 	case ADA_CCB_LOGDIR:
2917 	{
2918 		int error;
2919 
2920 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2921 			error = 0;
2922 			softc->valid_logdir_len = 0;
2923 			bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2924 			softc->valid_logdir_len =
2925 				ataio->dxfer_len - ataio->resid;
2926 			if (softc->valid_logdir_len > 0)
2927 				bcopy(ataio->data_ptr, &softc->ata_logdir,
2928 				    min(softc->valid_logdir_len,
2929 					sizeof(softc->ata_logdir)));
2930 			/*
2931 			 * Figure out whether the Identify Device log is
2932 			 * supported.  The General Purpose log directory
2933 			 * has a header, and lists the number of pages
2934 			 * available for each GP log identified by the
2935 			 * offset into the list.
2936 			 */
2937 			if ((softc->valid_logdir_len >=
2938 			    ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
2939 			 && (le16dec(softc->ata_logdir.header) ==
2940 			     ATA_GP_LOG_DIR_VERSION)
2941 			 && (le16dec(&softc->ata_logdir.num_pages[
2942 			     (ATA_IDENTIFY_DATA_LOG *
2943 			     sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
2944 				softc->flags |= ADA_FLAG_CAN_IDLOG;
2945 			} else {
2946 				softc->flags &= ~ADA_FLAG_CAN_IDLOG;
2947 			}
2948 		} else {
2949 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
2950 					 SF_RETRY_UA|SF_NO_PRINT);
2951 			if (error == ERESTART)
2952 				return;
2953 			else if (error != 0) {
2954 				/*
2955 				 * If we can't get the ATA log directory,
2956 				 * then ATA logs are effectively not
2957 				 * supported even if the bit is set in the
2958 				 * identify data.
2959 				 */
2960 				softc->flags &= ~(ADA_FLAG_CAN_LOG |
2961 						  ADA_FLAG_CAN_IDLOG);
2962 				if ((done_ccb->ccb_h.status &
2963 				     CAM_DEV_QFRZN) != 0) {
2964 					/* Don't wedge this device's queue */
2965 					cam_release_devq(done_ccb->ccb_h.path,
2966 							 /*relsim_flags*/0,
2967 							 /*reduction*/0,
2968 							 /*timeout*/0,
2969 							 /*getcount_only*/0);
2970 				}
2971 			}
2972 
2973 
2974 		}
2975 
2976 		free(ataio->data_ptr, M_ATADA);
2977 
2978 		if ((error == 0)
2979 		 && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
2980 			softc->state = ADA_STATE_IDDIR;
2981 			xpt_release_ccb(done_ccb);
2982 			xpt_schedule(periph, priority);
2983 		} else
2984 			adaprobedone(periph, done_ccb);
2985 
2986 		return;
2987 	}
2988 	case ADA_CCB_IDDIR: {
2989 		int error;
2990 
2991 		if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2992 			off_t entries_offset, max_entries;
2993 			error = 0;
2994 
2995 			softc->valid_iddir_len = 0;
2996 			bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
2997 			softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
2998 					  ADA_FLAG_CAN_ZONE);
2999 			softc->valid_iddir_len =
3000 				ataio->dxfer_len - ataio->resid;
3001 			if (softc->valid_iddir_len > 0)
3002 				bcopy(ataio->data_ptr, &softc->ata_iddir,
3003 				    min(softc->valid_iddir_len,
3004 					sizeof(softc->ata_iddir)));
3005 
3006 			entries_offset =
3007 			    __offsetof(struct ata_identify_log_pages,entries);
3008 			max_entries = softc->valid_iddir_len - entries_offset;
3009 			if ((softc->valid_iddir_len > (entries_offset + 1))
3010 			 && (le64dec(softc->ata_iddir.header) ==
3011 			     ATA_IDLOG_REVISION)
3012 			 && (softc->ata_iddir.entry_count > 0)) {
3013 				int num_entries, i;
3014 
3015 				num_entries = softc->ata_iddir.entry_count;
3016 				num_entries = min(num_entries,
3017 				   softc->valid_iddir_len - entries_offset);
3018 				for (i = 0; i < num_entries &&
3019 				     i < max_entries; i++) {
3020 					if (softc->ata_iddir.entries[i] ==
3021 					    ATA_IDL_SUP_CAP)
3022 						softc->flags |=
3023 						    ADA_FLAG_CAN_SUPCAP;
3024 					else if (softc->ata_iddir.entries[i]==
3025 						 ATA_IDL_ZDI)
3026 						softc->flags |=
3027 						    ADA_FLAG_CAN_ZONE;
3028 
3029 					if ((softc->flags &
3030 					     ADA_FLAG_CAN_SUPCAP)
3031 					 && (softc->flags &
3032 					     ADA_FLAG_CAN_ZONE))
3033 						break;
3034 				}
3035 			}
3036 		} else {
3037 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3038 					 SF_RETRY_UA|SF_NO_PRINT);
3039 			if (error == ERESTART)
3040 				return;
3041 			else if (error != 0) {
3042 				/*
3043 				 * If we can't get the ATA Identify Data log
3044 				 * directory, then it effectively isn't
3045 				 * supported even if the ATA Log directory
3046 				 * a non-zero number of pages present for
3047 				 * this log.
3048 				 */
3049 				softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3050 				if ((done_ccb->ccb_h.status &
3051 				     CAM_DEV_QFRZN) != 0) {
3052 					/* Don't wedge this device's queue */
3053 					cam_release_devq(done_ccb->ccb_h.path,
3054 							 /*relsim_flags*/0,
3055 							 /*reduction*/0,
3056 							 /*timeout*/0,
3057 							 /*getcount_only*/0);
3058 				}
3059 			}
3060 		}
3061 
3062 		free(ataio->data_ptr, M_ATADA);
3063 
3064 		if ((error == 0)
3065 		 && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3066 			softc->state = ADA_STATE_SUP_CAP;
3067 			xpt_release_ccb(done_ccb);
3068 			xpt_schedule(periph, priority);
3069 		} else
3070 			adaprobedone(periph, done_ccb);
3071 		return;
3072 	}
3073 	case ADA_CCB_SUP_CAP: {
3074 		int error;
3075 
3076 		if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3077 			uint32_t valid_len;
3078 			size_t needed_size;
3079 			struct ata_identify_log_sup_cap *sup_cap;
3080 			error = 0;
3081 
3082 			sup_cap = (struct ata_identify_log_sup_cap *)
3083 			    ataio->data_ptr;
3084 			valid_len = ataio->dxfer_len - ataio->resid;
3085 			needed_size =
3086 			    __offsetof(struct ata_identify_log_sup_cap,
3087 			    sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3088 			if (valid_len >= needed_size) {
3089 				uint64_t zoned, zac_cap;
3090 
3091 				zoned = le64dec(sup_cap->zoned_cap);
3092 				if (zoned & ATA_ZONED_VALID) {
3093 					/*
3094 					 * This should have already been
3095 					 * set, because this is also in the
3096 					 * ATA identify data.
3097 					 */
3098 					if ((zoned & ATA_ZONED_MASK) ==
3099 					    ATA_SUPPORT_ZONE_HOST_AWARE)
3100 						softc->zone_mode =
3101 						    ADA_ZONE_HOST_AWARE;
3102 					else if ((zoned & ATA_ZONED_MASK) ==
3103 					    ATA_SUPPORT_ZONE_DEV_MANAGED)
3104 						softc->zone_mode =
3105 						    ADA_ZONE_DRIVE_MANAGED;
3106 				}
3107 
3108 				zac_cap = le64dec(sup_cap->sup_zac_cap);
3109 				if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3110 					if (zac_cap & ATA_REPORT_ZONES_SUP)
3111 						softc->zone_flags |=
3112 						    ADA_ZONE_FLAG_RZ_SUP;
3113 					if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3114 						softc->zone_flags |=
3115 						    ADA_ZONE_FLAG_OPEN_SUP;
3116 					if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3117 						softc->zone_flags |=
3118 						    ADA_ZONE_FLAG_CLOSE_SUP;
3119 					if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3120 						softc->zone_flags |=
3121 						    ADA_ZONE_FLAG_FINISH_SUP;
3122 					if (zac_cap & ATA_ND_RWP_SUP)
3123 						softc->zone_flags |=
3124 						    ADA_ZONE_FLAG_RWP_SUP;
3125 				} else {
3126 					/*
3127 					 * This field was introduced in
3128 					 * ACS-4, r08 on April 28th, 2015.
3129 					 * If the drive firmware was written
3130 					 * to an earlier spec, it won't have
3131 					 * the field.  So, assume all
3132 					 * commands are supported.
3133 					 */
3134 					softc->zone_flags |=
3135 					    ADA_ZONE_FLAG_SUP_MASK;
3136 				}
3137 
3138 			}
3139 		} else {
3140 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3141 					 SF_RETRY_UA|SF_NO_PRINT);
3142 			if (error == ERESTART)
3143 				return;
3144 			else if (error != 0) {
3145 				/*
3146 				 * If we can't get the ATA Identify Data
3147 				 * Supported Capabilities page, clear the
3148 				 * flag...
3149 				 */
3150 				softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3151 				/*
3152 				 * And clear zone capabilities.
3153 				 */
3154 				softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3155 				if ((done_ccb->ccb_h.status &
3156 				     CAM_DEV_QFRZN) != 0) {
3157 					/* Don't wedge this device's queue */
3158 					cam_release_devq(done_ccb->ccb_h.path,
3159 							 /*relsim_flags*/0,
3160 							 /*reduction*/0,
3161 							 /*timeout*/0,
3162 							 /*getcount_only*/0);
3163 				}
3164 			}
3165 		}
3166 
3167 		free(ataio->data_ptr, M_ATADA);
3168 
3169 		if ((error == 0)
3170 		 && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3171 			softc->state = ADA_STATE_ZONE;
3172 			xpt_release_ccb(done_ccb);
3173 			xpt_schedule(periph, priority);
3174 		} else
3175 			adaprobedone(periph, done_ccb);
3176 		return;
3177 	}
3178 	case ADA_CCB_ZONE: {
3179 		int error;
3180 
3181 		if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3182 			struct ata_zoned_info_log *zi_log;
3183 			uint32_t valid_len;
3184 			size_t needed_size;
3185 
3186 			zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3187 
3188 			valid_len = ataio->dxfer_len - ataio->resid;
3189 			needed_size = __offsetof(struct ata_zoned_info_log,
3190 			    version_info) + 1 + sizeof(zi_log->version_info);
3191 			if (valid_len >= needed_size) {
3192 				uint64_t tmpvar;
3193 
3194 				tmpvar = le64dec(zi_log->zoned_cap);
3195 				if (tmpvar & ATA_ZDI_CAP_VALID) {
3196 					if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3197 						softc->zone_flags |=
3198 						    ADA_ZONE_FLAG_URSWRZ;
3199 					else
3200 						softc->zone_flags &=
3201 						    ~ADA_ZONE_FLAG_URSWRZ;
3202 				}
3203 				tmpvar = le64dec(zi_log->optimal_seq_zones);
3204 				if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3205 					softc->zone_flags |=
3206 					    ADA_ZONE_FLAG_OPT_SEQ_SET;
3207 					softc->optimal_seq_zones = (tmpvar &
3208 					    ATA_ZDI_OPT_SEQ_MASK);
3209 				} else {
3210 					softc->zone_flags &=
3211 					    ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3212 					softc->optimal_seq_zones = 0;
3213 				}
3214 
3215 				tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3216 				if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3217 					softc->zone_flags |=
3218 					    ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3219 					softc->optimal_nonseq_zones =
3220 					    (tmpvar & ATA_ZDI_OPT_NS_MASK);
3221 				} else {
3222 					softc->zone_flags &=
3223 					    ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3224 					softc->optimal_nonseq_zones = 0;
3225 				}
3226 
3227 				tmpvar = le64dec(zi_log->max_seq_req_zones);
3228 				if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3229 					softc->zone_flags |=
3230 					    ADA_ZONE_FLAG_MAX_SEQ_SET;
3231 					softc->max_seq_zones =
3232 					    (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3233 				} else {
3234 					softc->zone_flags &=
3235 					    ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3236 					softc->max_seq_zones = 0;
3237 				}
3238 			}
3239 		} else {
3240 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3241 					 SF_RETRY_UA|SF_NO_PRINT);
3242 			if (error == ERESTART)
3243 				return;
3244 			else if (error != 0) {
3245 				softc->flags &= ~ADA_FLAG_CAN_ZONE;
3246 				softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3247 
3248 				if ((done_ccb->ccb_h.status &
3249 				     CAM_DEV_QFRZN) != 0) {
3250 					/* Don't wedge this device's queue */
3251 					cam_release_devq(done_ccb->ccb_h.path,
3252 							 /*relsim_flags*/0,
3253 							 /*reduction*/0,
3254 							 /*timeout*/0,
3255 							 /*getcount_only*/0);
3256 				}
3257 			}
3258 
3259 		}
3260 		free(ataio->data_ptr, M_ATADA);
3261 
3262 		adaprobedone(periph, done_ccb);
3263 		return;
3264 	}
3265 	case ADA_CCB_DUMP:
3266 		/* No-op.  We're polling */
3267 		return;
3268 	default:
3269 		break;
3270 	}
3271 	xpt_release_ccb(done_ccb);
3272 }
3273 
3274 static int
3275 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3276 {
3277 #ifdef CAM_IO_STATS
3278 	struct ada_softc *softc;
3279 	struct cam_periph *periph;
3280 
3281 	periph = xpt_path_periph(ccb->ccb_h.path);
3282 	softc = (struct ada_softc *)periph->softc;
3283 
3284 	switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3285 	case CAM_CMD_TIMEOUT:
3286 		softc->timeouts++;
3287 		break;
3288 	case CAM_REQ_ABORTED:
3289 	case CAM_REQ_CMP_ERR:
3290 	case CAM_REQ_TERMIO:
3291 	case CAM_UNREC_HBA_ERROR:
3292 	case CAM_DATA_RUN_ERR:
3293 	case CAM_ATA_STATUS_ERROR:
3294 		softc->errors++;
3295 		break;
3296 	default:
3297 		break;
3298 	}
3299 #endif
3300 
3301 	return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
3302 }
3303 
3304 static void
3305 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3306 {
3307 	struct ada_softc *softc = (struct ada_softc *)periph->softc;
3308 	struct disk_params *dp = &softc->params;
3309 	u_int64_t lbasize48;
3310 	u_int32_t lbasize;
3311 
3312 	dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3313 	if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3314 		cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3315 		dp->heads = cgd->ident_data.current_heads;
3316 		dp->secs_per_track = cgd->ident_data.current_sectors;
3317 		dp->cylinders = cgd->ident_data.cylinders;
3318 		dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3319 			  ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3320 	} else {
3321 		dp->heads = cgd->ident_data.heads;
3322 		dp->secs_per_track = cgd->ident_data.sectors;
3323 		dp->cylinders = cgd->ident_data.cylinders;
3324 		dp->sectors = cgd->ident_data.cylinders * dp->heads * dp->secs_per_track;
3325 	}
3326 	lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3327 		  ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3328 
3329 	/* use the 28bit LBA size if valid or bigger than the CHS mapping */
3330 	if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3331 		dp->sectors = lbasize;
3332 
3333 	/* use the 48bit LBA size if valid */
3334 	lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3335 		    ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3336 		    ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3337 		    ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3338 	if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3339 	    lbasize48 > ATA_MAX_28BIT_LBA)
3340 		dp->sectors = lbasize48;
3341 }
3342 
3343 static void
3344 adasendorderedtag(void *arg)
3345 {
3346 	struct ada_softc *softc = arg;
3347 
3348 	if (ada_send_ordered) {
3349 		if (softc->outstanding_cmds > 0) {
3350 			if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3351 				softc->flags |= ADA_FLAG_NEED_OTAG;
3352 			softc->flags &= ~ADA_FLAG_WAS_OTAG;
3353 		}
3354 	}
3355 	/* Queue us up again */
3356 	callout_reset(&softc->sendordered_c,
3357 	    (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3358 	    adasendorderedtag, softc);
3359 }
3360 
3361 /*
3362  * Step through all ADA peripheral drivers, and if the device is still open,
3363  * sync the disk cache to physical media.
3364  */
3365 static void
3366 adaflush(void)
3367 {
3368 	struct cam_periph *periph;
3369 	struct ada_softc *softc;
3370 	union ccb *ccb;
3371 	int error;
3372 
3373 	CAM_PERIPH_FOREACH(periph, &adadriver) {
3374 		softc = (struct ada_softc *)periph->softc;
3375 		if (SCHEDULER_STOPPED()) {
3376 			/* If we paniced with the lock held, do not recurse. */
3377 			if (!cam_periph_owned(periph) &&
3378 			    (softc->flags & ADA_FLAG_OPEN)) {
3379 				adadump(softc->disk, NULL, 0, 0, 0);
3380 			}
3381 			continue;
3382 		}
3383 		cam_periph_lock(periph);
3384 		/*
3385 		 * We only sync the cache if the drive is still open, and
3386 		 * if the drive is capable of it..
3387 		 */
3388 		if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3389 		    (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3390 			cam_periph_unlock(periph);
3391 			continue;
3392 		}
3393 
3394 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3395 		cam_fill_ataio(&ccb->ataio,
3396 				    0,
3397 				    adadone,
3398 				    CAM_DIR_NONE,
3399 				    0,
3400 				    NULL,
3401 				    0,
3402 				    ada_default_timeout*1000);
3403 		if (softc->flags & ADA_FLAG_CAN_48BIT)
3404 			ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3405 		else
3406 			ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3407 
3408 		error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3409 		    /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3410 		    softc->disk->d_devstat);
3411 		if (error != 0)
3412 			xpt_print(periph->path, "Synchronize cache failed\n");
3413 		xpt_release_ccb(ccb);
3414 		cam_periph_unlock(periph);
3415 	}
3416 }
3417 
3418 static void
3419 adaspindown(uint8_t cmd, int flags)
3420 {
3421 	struct cam_periph *periph;
3422 	struct ada_softc *softc;
3423 	union ccb *ccb;
3424 	int error;
3425 
3426 	CAM_PERIPH_FOREACH(periph, &adadriver) {
3427 		/* If we paniced with lock held - not recurse here. */
3428 		if (cam_periph_owned(periph))
3429 			continue;
3430 		cam_periph_lock(periph);
3431 		softc = (struct ada_softc *)periph->softc;
3432 		/*
3433 		 * We only spin-down the drive if it is capable of it..
3434 		 */
3435 		if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3436 			cam_periph_unlock(periph);
3437 			continue;
3438 		}
3439 
3440 		if (bootverbose)
3441 			xpt_print(periph->path, "spin-down\n");
3442 
3443 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3444 		cam_fill_ataio(&ccb->ataio,
3445 				    0,
3446 				    adadone,
3447 				    CAM_DIR_NONE | flags,
3448 				    0,
3449 				    NULL,
3450 				    0,
3451 				    ada_default_timeout*1000);
3452 		ata_28bit_cmd(&ccb->ataio, cmd, 0, 0, 0);
3453 
3454 		error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3455 		    /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3456 		    softc->disk->d_devstat);
3457 		if (error != 0)
3458 			xpt_print(periph->path, "Spin-down disk failed\n");
3459 		xpt_release_ccb(ccb);
3460 		cam_periph_unlock(periph);
3461 	}
3462 }
3463 
3464 static void
3465 adashutdown(void *arg, int howto)
3466 {
3467 
3468 	adaflush();
3469 	if (ada_spindown_shutdown != 0 &&
3470 	    (howto & (RB_HALT | RB_POWEROFF)) != 0)
3471 		adaspindown(ATA_STANDBY_IMMEDIATE, 0);
3472 }
3473 
3474 static void
3475 adasuspend(void *arg)
3476 {
3477 
3478 	adaflush();
3479 	if (ada_spindown_suspend != 0)
3480 		adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3481 }
3482 
3483 static void
3484 adaresume(void *arg)
3485 {
3486 	struct cam_periph *periph;
3487 	struct ada_softc *softc;
3488 
3489 	if (ada_spindown_suspend == 0)
3490 		return;
3491 
3492 	CAM_PERIPH_FOREACH(periph, &adadriver) {
3493 		cam_periph_lock(periph);
3494 		softc = (struct ada_softc *)periph->softc;
3495 		/*
3496 		 * We only spin-down the drive if it is capable of it..
3497 		 */
3498 		if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3499 			cam_periph_unlock(periph);
3500 			continue;
3501 		}
3502 
3503 		if (bootverbose)
3504 			xpt_print(periph->path, "resume\n");
3505 
3506 		/*
3507 		 * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3508 		 * sleep request.
3509 		 */
3510 		cam_release_devq(periph->path,
3511 			 /*relsim_flags*/0,
3512 			 /*openings*/0,
3513 			 /*timeout*/0,
3514 			 /*getcount_only*/0);
3515 
3516 		cam_periph_unlock(periph);
3517 	}
3518 }
3519 
3520 #endif /* _KERNEL */
3521