xref: /freebsd/sys/cam/ata/ata_da.c (revision e4f6a1bfa31a2299612bbf9dae402bb38d38b1df)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer,
12  *    without modification, immediately at the beginning of the file.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include "opt_ada.h"
33 
34 #include <sys/param.h>
35 
36 #ifdef _KERNEL
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/bio.h>
40 #include <sys/sysctl.h>
41 #include <sys/taskqueue.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/conf.h>
45 #include <sys/devicestat.h>
46 #include <sys/eventhandler.h>
47 #include <sys/malloc.h>
48 #include <sys/endian.h>
49 #include <sys/cons.h>
50 #include <sys/proc.h>
51 #include <sys/reboot.h>
52 #include <sys/sbuf.h>
53 #include <geom/geom_disk.h>
54 #endif /* _KERNEL */
55 
56 #ifndef _KERNEL
57 #include <stdio.h>
58 #include <string.h>
59 #endif /* _KERNEL */
60 
61 #include <cam/cam.h>
62 #include <cam/cam_ccb.h>
63 #include <cam/cam_periph.h>
64 #include <cam/cam_xpt_periph.h>
65 #include <cam/scsi/scsi_all.h>
66 #include <cam/scsi/scsi_da.h>
67 #include <cam/cam_sim.h>
68 #include <cam/cam_iosched.h>
69 
70 #include <cam/ata/ata_all.h>
71 
72 #include <machine/md_var.h>	/* geometry translation */
73 
74 #ifdef _KERNEL
75 
76 #define ATA_MAX_28BIT_LBA               268435455UL
77 
78 extern int iosched_debug;
79 
80 typedef enum {
81 	ADA_STATE_RAHEAD,
82 	ADA_STATE_WCACHE,
83 	ADA_STATE_LOGDIR,
84 	ADA_STATE_IDDIR,
85 	ADA_STATE_SUP_CAP,
86 	ADA_STATE_ZONE,
87 	ADA_STATE_NORMAL
88 } ada_state;
89 
90 typedef enum {
91 	ADA_FLAG_CAN_48BIT	= 0x00000002,
92 	ADA_FLAG_CAN_FLUSHCACHE	= 0x00000004,
93 	ADA_FLAG_CAN_NCQ	= 0x00000008,
94 	ADA_FLAG_CAN_DMA	= 0x00000010,
95 	ADA_FLAG_NEED_OTAG	= 0x00000020,
96 	ADA_FLAG_WAS_OTAG	= 0x00000040,
97 	ADA_FLAG_CAN_TRIM	= 0x00000080,
98 	ADA_FLAG_OPEN		= 0x00000100,
99 	ADA_FLAG_SCTX_INIT	= 0x00000200,
100 	ADA_FLAG_CAN_CFA        = 0x00000400,
101 	ADA_FLAG_CAN_POWERMGT   = 0x00000800,
102 	ADA_FLAG_CAN_DMA48	= 0x00001000,
103 	ADA_FLAG_CAN_LOG	= 0x00002000,
104 	ADA_FLAG_CAN_IDLOG	= 0x00004000,
105 	ADA_FLAG_CAN_SUPCAP	= 0x00008000,
106 	ADA_FLAG_CAN_ZONE	= 0x00010000,
107 	ADA_FLAG_CAN_WCACHE	= 0x00020000,
108 	ADA_FLAG_CAN_RAHEAD	= 0x00040000,
109 	ADA_FLAG_PROBED		= 0x00080000,
110 	ADA_FLAG_ANNOUNCED	= 0x00100000,
111 	ADA_FLAG_DIRTY		= 0x00200000,
112 	ADA_FLAG_CAN_NCQ_TRIM	= 0x00400000,	/* CAN_TRIM also set */
113 	ADA_FLAG_PIM_ATA_EXT	= 0x00800000
114 } ada_flags;
115 
116 typedef enum {
117 	ADA_Q_NONE		= 0x00,
118 	ADA_Q_4K		= 0x01,
119 	ADA_Q_NCQ_TRIM_BROKEN	= 0x02,
120 	ADA_Q_LOG_BROKEN	= 0x04,
121 	ADA_Q_SMR_DM		= 0x08
122 } ada_quirks;
123 
124 #define ADA_Q_BIT_STRING	\
125 	"\020"			\
126 	"\0014K"		\
127 	"\002NCQ_TRIM_BROKEN"	\
128 	"\003LOG_BROKEN"	\
129 	"\004SMR_DM"
130 
131 typedef enum {
132 	ADA_CCB_RAHEAD		= 0x01,
133 	ADA_CCB_WCACHE		= 0x02,
134 	ADA_CCB_BUFFER_IO	= 0x03,
135 	ADA_CCB_DUMP		= 0x05,
136 	ADA_CCB_TRIM		= 0x06,
137 	ADA_CCB_LOGDIR		= 0x07,
138 	ADA_CCB_IDDIR		= 0x08,
139 	ADA_CCB_SUP_CAP		= 0x09,
140 	ADA_CCB_ZONE		= 0x0a,
141 	ADA_CCB_TYPE_MASK	= 0x0F,
142 } ada_ccb_state;
143 
144 typedef enum {
145 	ADA_ZONE_NONE		= 0x00,
146 	ADA_ZONE_DRIVE_MANAGED	= 0x01,
147 	ADA_ZONE_HOST_AWARE	= 0x02,
148 	ADA_ZONE_HOST_MANAGED	= 0x03
149 } ada_zone_mode;
150 
151 typedef enum {
152 	ADA_ZONE_FLAG_RZ_SUP		= 0x0001,
153 	ADA_ZONE_FLAG_OPEN_SUP		= 0x0002,
154 	ADA_ZONE_FLAG_CLOSE_SUP		= 0x0004,
155 	ADA_ZONE_FLAG_FINISH_SUP	= 0x0008,
156 	ADA_ZONE_FLAG_RWP_SUP		= 0x0010,
157 	ADA_ZONE_FLAG_SUP_MASK		= (ADA_ZONE_FLAG_RZ_SUP |
158 					   ADA_ZONE_FLAG_OPEN_SUP |
159 					   ADA_ZONE_FLAG_CLOSE_SUP |
160 					   ADA_ZONE_FLAG_FINISH_SUP |
161 					   ADA_ZONE_FLAG_RWP_SUP),
162 	ADA_ZONE_FLAG_URSWRZ		= 0x0020,
163 	ADA_ZONE_FLAG_OPT_SEQ_SET	= 0x0040,
164 	ADA_ZONE_FLAG_OPT_NONSEQ_SET	= 0x0080,
165 	ADA_ZONE_FLAG_MAX_SEQ_SET	= 0x0100,
166 	ADA_ZONE_FLAG_SET_MASK		= (ADA_ZONE_FLAG_OPT_SEQ_SET |
167 					   ADA_ZONE_FLAG_OPT_NONSEQ_SET |
168 					   ADA_ZONE_FLAG_MAX_SEQ_SET)
169 } ada_zone_flags;
170 
171 static struct ada_zone_desc {
172 	ada_zone_flags value;
173 	const char *desc;
174 } ada_zone_desc_table[] = {
175 	{ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
176 	{ADA_ZONE_FLAG_OPEN_SUP, "Open" },
177 	{ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
178 	{ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
179 	{ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
180 };
181 
182 
183 /* Offsets into our private area for storing information */
184 #define ccb_state	ppriv_field0
185 #define ccb_bp		ppriv_ptr1
186 
187 typedef enum {
188 	ADA_DELETE_NONE,
189 	ADA_DELETE_DISABLE,
190 	ADA_DELETE_CFA_ERASE,
191 	ADA_DELETE_DSM_TRIM,
192 	ADA_DELETE_NCQ_DSM_TRIM,
193 	ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
194 	ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
195 } ada_delete_methods;
196 
197 static const char *ada_delete_method_names[] =
198     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
199 #if 0
200 static const char *ada_delete_method_desc[] =
201     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
202 #endif
203 
204 struct disk_params {
205 	u_int8_t  heads;
206 	u_int8_t  secs_per_track;
207 	u_int32_t cylinders;
208 	u_int32_t secsize;	/* Number of bytes/logical sector */
209 	u_int64_t sectors;	/* Total number sectors */
210 };
211 
212 #define TRIM_MAX_BLOCKS	8
213 #define TRIM_MAX_RANGES	(TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
214 struct trim_request {
215 	uint8_t		data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
216 	TAILQ_HEAD(, bio) bps;
217 };
218 
219 struct ada_softc {
220 	struct   cam_iosched_softc *cam_iosched;
221 	int	 outstanding_cmds;	/* Number of active commands */
222 	int	 refcount;		/* Active xpt_action() calls */
223 	ada_state state;
224 	ada_flags flags;
225 	ada_zone_mode zone_mode;
226 	ada_zone_flags zone_flags;
227 	struct ata_gp_log_dir ata_logdir;
228 	int valid_logdir_len;
229 	struct ata_identify_log_pages ata_iddir;
230 	int valid_iddir_len;
231 	uint64_t optimal_seq_zones;
232 	uint64_t optimal_nonseq_zones;
233 	uint64_t max_seq_zones;
234 	ada_quirks quirks;
235 	ada_delete_methods delete_method;
236 	int	 trim_max_ranges;
237 	int	 read_ahead;
238 	int	 write_cache;
239 	int	 unmappedio;
240 	int	 rotating;
241 #ifdef ADA_TEST_FAILURE
242 	int      force_read_error;
243 	int      force_write_error;
244 	int      periodic_read_error;
245 	int      periodic_read_count;
246 #endif
247 	struct	 disk_params params;
248 	struct	 disk *disk;
249 	struct task		sysctl_task;
250 	struct sysctl_ctx_list	sysctl_ctx;
251 	struct sysctl_oid	*sysctl_tree;
252 	struct callout		sendordered_c;
253 	struct trim_request	trim_req;
254 #ifdef CAM_IO_STATS
255 	struct sysctl_ctx_list	sysctl_stats_ctx;
256 	struct sysctl_oid	*sysctl_stats_tree;
257 	u_int	timeouts;
258 	u_int	errors;
259 	u_int	invalidations;
260 #endif
261 #define ADA_ANNOUNCETMP_SZ 80
262 	char	announce_temp[ADA_ANNOUNCETMP_SZ];
263 #define ADA_ANNOUNCE_SZ 400
264 	char	announce_buffer[ADA_ANNOUNCE_SZ];
265 };
266 
267 struct ada_quirk_entry {
268 	struct scsi_inquiry_pattern inq_pat;
269 	ada_quirks quirks;
270 };
271 
272 static struct ada_quirk_entry ada_quirk_table[] =
273 {
274 	{
275 		/* Hitachi Advanced Format (4k) drives */
276 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
277 		/*quirks*/ADA_Q_4K
278 	},
279 	{
280 		/* Samsung Advanced Format (4k) drives */
281 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
282 		/*quirks*/ADA_Q_4K
283 	},
284 	{
285 		/* Samsung Advanced Format (4k) drives */
286 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
287 		/*quirks*/ADA_Q_4K
288 	},
289 	{
290 		/* Seagate Barracuda Green Advanced Format (4k) drives */
291 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
292 		/*quirks*/ADA_Q_4K
293 	},
294 	{
295 		/* Seagate Barracuda Advanced Format (4k) drives */
296 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
297 		/*quirks*/ADA_Q_4K
298 	},
299 	{
300 		/* Seagate Barracuda Advanced Format (4k) drives */
301 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
302 		/*quirks*/ADA_Q_4K
303 	},
304 	{
305 		/* Seagate Momentus Advanced Format (4k) drives */
306 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
307 		/*quirks*/ADA_Q_4K
308 	},
309 	{
310 		/* Seagate Momentus Advanced Format (4k) drives */
311 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
312 		/*quirks*/ADA_Q_4K
313 	},
314 	{
315 		/* Seagate Momentus Advanced Format (4k) drives */
316 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
317 		/*quirks*/ADA_Q_4K
318 	},
319 	{
320 		/* Seagate Momentus Advanced Format (4k) drives */
321 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
322 		/*quirks*/ADA_Q_4K
323 	},
324 	{
325 		/* Seagate Momentus Advanced Format (4k) drives */
326 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
327 		/*quirks*/ADA_Q_4K
328 	},
329 	{
330 		/* Seagate Momentus Advanced Format (4k) drives */
331 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
332 		/*quirks*/ADA_Q_4K
333 	},
334 	{
335 		/* Seagate Momentus Advanced Format (4k) drives */
336 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
337 		/*quirks*/ADA_Q_4K
338 	},
339 	{
340 		/* Seagate Momentus Thin Advanced Format (4k) drives */
341 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
342 		/*quirks*/ADA_Q_4K
343 	},
344 	{
345 		/* WDC Caviar Red Advanced Format (4k) drives */
346 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
347 		/*quirks*/ADA_Q_4K
348 	},
349 	{
350 		/* WDC Caviar Green Advanced Format (4k) drives */
351 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
352 		/*quirks*/ADA_Q_4K
353 	},
354 	{
355 		/* WDC Caviar Green/Red Advanced Format (4k) drives */
356 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
357 		/*quirks*/ADA_Q_4K
358 	},
359 	{
360 		/* WDC Caviar Red Advanced Format (4k) drives */
361 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
362 		/*quirks*/ADA_Q_4K
363 	},
364 	{
365 		/* WDC Caviar Black Advanced Format (4k) drives */
366 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????EX*", "*" },
367 		/*quirks*/ADA_Q_4K
368 	},
369 	{
370 		/* WDC Caviar Green Advanced Format (4k) drives */
371 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
372 		/*quirks*/ADA_Q_4K
373 	},
374 	{
375 		/* WDC Caviar Green Advanced Format (4k) drives */
376 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
377 		/*quirks*/ADA_Q_4K
378 	},
379 	{
380 		/* WDC Scorpio Black Advanced Format (4k) drives */
381 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
382 		/*quirks*/ADA_Q_4K
383 	},
384 	{
385 		/* WDC Scorpio Black Advanced Format (4k) drives */
386 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
387 		/*quirks*/ADA_Q_4K
388 	},
389 	{
390 		/* WDC Scorpio Blue Advanced Format (4k) drives */
391 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
392 		/*quirks*/ADA_Q_4K
393 	},
394 	{
395 		/* WDC Scorpio Blue Advanced Format (4k) drives */
396 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
397 		/*quirks*/ADA_Q_4K
398 	},
399 	/* SSDs */
400 	{
401 		/*
402 		 * Corsair Force 2 SSDs
403 		 * 4k optimised & trim only works in 4k requests + 4k aligned
404 		 */
405 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
406 		/*quirks*/ADA_Q_4K
407 	},
408 	{
409 		/*
410 		 * Corsair Force 3 SSDs
411 		 * 4k optimised & trim only works in 4k requests + 4k aligned
412 		 */
413 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
414 		/*quirks*/ADA_Q_4K
415 	},
416 	{
417 		/*
418 		 * Corsair Neutron GTX SSDs
419 		 * 4k optimised & trim only works in 4k requests + 4k aligned
420 		 */
421 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
422 		/*quirks*/ADA_Q_4K
423 	},
424 	{
425 		/*
426 		 * Corsair Force GT & GS SSDs
427 		 * 4k optimised & trim only works in 4k requests + 4k aligned
428 		 */
429 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
430 		/*quirks*/ADA_Q_4K
431 	},
432 	{
433 		/*
434 		 * Crucial M4 SSDs
435 		 * 4k optimised & trim only works in 4k requests + 4k aligned
436 		 */
437 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
438 		/*quirks*/ADA_Q_4K
439 	},
440 	{
441 		/*
442 		 * Crucial M500 SSDs MU07 firmware
443 		 * NCQ Trim works
444 		 */
445 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
446 		/*quirks*/0
447 	},
448 	{
449 		/*
450 		 * Crucial M500 SSDs all other firmware
451 		 * NCQ Trim doesn't work
452 		 */
453 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
454 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
455 	},
456 	{
457 		/*
458 		 * Crucial M550 SSDs
459 		 * NCQ Trim doesn't work, but only on MU01 firmware
460 		 */
461 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
462 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
463 	},
464 	{
465 		/*
466 		 * Crucial MX100 SSDs
467 		 * NCQ Trim doesn't work, but only on MU01 firmware
468 		 */
469 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
470 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
471 	},
472 	{
473 		/*
474 		 * Crucial RealSSD C300 SSDs
475 		 * 4k optimised
476 		 */
477 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
478 		"*" }, /*quirks*/ADA_Q_4K
479 	},
480 	{
481 		/*
482 		 * FCCT M500 SSDs
483 		 * NCQ Trim doesn't work
484 		 */
485 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
486 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
487 	},
488 	{
489 		/*
490 		 * Intel 320 Series SSDs
491 		 * 4k optimised & trim only works in 4k requests + 4k aligned
492 		 */
493 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
494 		/*quirks*/ADA_Q_4K
495 	},
496 	{
497 		/*
498 		 * Intel 330 Series SSDs
499 		 * 4k optimised & trim only works in 4k requests + 4k aligned
500 		 */
501 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
502 		/*quirks*/ADA_Q_4K
503 	},
504 	{
505 		/*
506 		 * Intel 510 Series SSDs
507 		 * 4k optimised & trim only works in 4k requests + 4k aligned
508 		 */
509 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
510 		/*quirks*/ADA_Q_4K
511 	},
512 	{
513 		/*
514 		 * Intel 520 Series SSDs
515 		 * 4k optimised & trim only works in 4k requests + 4k aligned
516 		 */
517 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
518 		/*quirks*/ADA_Q_4K
519 	},
520 	{
521 		/*
522 		 * Intel S3610 Series SSDs
523 		 * 4k optimised & trim only works in 4k requests + 4k aligned
524 		 */
525 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
526 		/*quirks*/ADA_Q_4K
527 	},
528 	{
529 		/*
530 		 * Intel X25-M Series SSDs
531 		 * 4k optimised & trim only works in 4k requests + 4k aligned
532 		 */
533 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
534 		/*quirks*/ADA_Q_4K
535 	},
536 	{
537 		/*
538 		 * Kingston E100 Series SSDs
539 		 * 4k optimised & trim only works in 4k requests + 4k aligned
540 		 */
541 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
542 		/*quirks*/ADA_Q_4K
543 	},
544 	{
545 		/*
546 		 * Kingston HyperX 3k SSDs
547 		 * 4k optimised & trim only works in 4k requests + 4k aligned
548 		 */
549 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
550 		/*quirks*/ADA_Q_4K
551 	},
552 	{
553 		/*
554 		 * Marvell SSDs (entry taken from OpenSolaris)
555 		 * 4k optimised & trim only works in 4k requests + 4k aligned
556 		 */
557 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
558 		/*quirks*/ADA_Q_4K
559 	},
560 	{
561 		/*
562 		 * Micron M500 SSDs firmware MU07
563 		 * NCQ Trim works?
564 		 */
565 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
566 		/*quirks*/0
567 	},
568 	{
569 		/*
570 		 * Micron M500 SSDs all other firmware
571 		 * NCQ Trim doesn't work
572 		 */
573 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
574 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
575 	},
576 	{
577 		/*
578 		 * Micron M5[15]0 SSDs
579 		 * NCQ Trim doesn't work, but only MU01 firmware
580 		 */
581 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
582 		/*quirks*/ADA_Q_NCQ_TRIM_BROKEN
583 	},
584 	{
585 		/*
586 		 * Micron 5100 SSDs
587 		 * 4k optimised & trim only works in 4k requests + 4k aligned
588 		 */
589 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
590 		/*quirks*/ADA_Q_4K
591 	},
592 	{
593 		/*
594 		 * OCZ Agility 2 SSDs
595 		 * 4k optimised & trim only works in 4k requests + 4k aligned
596 		 */
597 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
598 		/*quirks*/ADA_Q_4K
599 	},
600 	{
601 		/*
602 		 * OCZ Agility 3 SSDs
603 		 * 4k optimised & trim only works in 4k requests + 4k aligned
604 		 */
605 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
606 		/*quirks*/ADA_Q_4K
607 	},
608 	{
609 		/*
610 		 * OCZ Deneva R Series SSDs
611 		 * 4k optimised & trim only works in 4k requests + 4k aligned
612 		 */
613 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
614 		/*quirks*/ADA_Q_4K
615 	},
616 	{
617 		/*
618 		 * OCZ Vertex 2 SSDs (inc pro series)
619 		 * 4k optimised & trim only works in 4k requests + 4k aligned
620 		 */
621 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
622 		/*quirks*/ADA_Q_4K
623 	},
624 	{
625 		/*
626 		 * OCZ Vertex 3 SSDs
627 		 * 4k optimised & trim only works in 4k requests + 4k aligned
628 		 */
629 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
630 		/*quirks*/ADA_Q_4K
631 	},
632 	{
633 		/*
634 		 * OCZ Vertex 4 SSDs
635 		 * 4k optimised & trim only works in 4k requests + 4k aligned
636 		 */
637 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
638 		/*quirks*/ADA_Q_4K
639 	},
640 	{
641 		/*
642 		 * Samsung 750 SSDs
643 		 * 4k optimised, NCQ TRIM seems to work
644 		 */
645 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
646 		/*quirks*/ADA_Q_4K
647 	},
648 	{
649 		/*
650 		 * Samsung 830 Series SSDs
651 		 * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
652 		 */
653 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
654 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
655 	},
656 	{
657 		/*
658 		 * Samsung 840 SSDs
659 		 * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
660 		 */
661 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
662 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
663 	},
664 	{
665 		/*
666 		 * Samsung 845 SSDs
667 		 * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
668 		 */
669 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
670 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
671 	},
672 	{
673 		/*
674 		 * Samsung 850 SSDs
675 		 * 4k optimised, NCQ TRIM broken (normal TRIM fine)
676 		 */
677 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
678 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
679 	},
680 	{
681 		/*
682 		 * Samsung SM863 Series SSDs (MZ7KM*)
683 		 * 4k optimised, NCQ believed to be working
684 		 */
685 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
686 		/*quirks*/ADA_Q_4K
687 	},
688 	{
689 		/*
690 		 * Samsung 843T Series SSDs (MZ7WD*)
691 		 * Samsung PM851 Series SSDs (MZ7TE*)
692 		 * Samsung PM853T Series SSDs (MZ7GE*)
693 		 * 4k optimised, NCQ believed to be broken since these are
694 		 * appear to be built with the same controllers as the 840/850.
695 		 */
696 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
697 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
698 	},
699 	{
700 		/*
701 		 * Same as for SAMSUNG MZ7* but enable the quirks for SSD
702 		 * starting with MZ7* too
703 		 */
704 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
705 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
706 	},
707 	{
708 		/*
709 		 * Samsung PM851 Series SSDs Dell OEM
710 		 * device model          "SAMSUNG SSD PM851 mSATA 256GB"
711 		 * 4k optimised, NCQ broken
712 		 */
713 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
714 		/*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
715 	},
716 	{
717 		/*
718 		 * SuperTalent TeraDrive CT SSDs
719 		 * 4k optimised & trim only works in 4k requests + 4k aligned
720 		 */
721 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
722 		/*quirks*/ADA_Q_4K
723 	},
724 	{
725 		/*
726 		 * XceedIOPS SATA SSDs
727 		 * 4k optimised
728 		 */
729 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
730 		/*quirks*/ADA_Q_4K
731 	},
732 	{
733 		/*
734 		 * Samsung drive that doesn't support READ LOG EXT or
735 		 * READ LOG DMA EXT, despite reporting that it does in
736 		 * ATA identify data:
737 		 * SAMSUNG HD200HJ KF100-06
738 		 */
739 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
740 		/*quirks*/ADA_Q_LOG_BROKEN
741 	},
742 	{
743 		/*
744 		 * Samsung drive that doesn't support READ LOG EXT or
745 		 * READ LOG DMA EXT, despite reporting that it does in
746 		 * ATA identify data:
747 		 * SAMSUNG HD501LJ CR100-10
748 		 */
749 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
750 		/*quirks*/ADA_Q_LOG_BROKEN
751 	},
752 	{
753 		/*
754 		 * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
755 		 * Drive Managed SATA hard drive.  This drive doesn't report
756 		 * in firmware that it is a drive managed SMR drive.
757 		 */
758 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
759 		/*quirks*/ADA_Q_SMR_DM
760 	},
761 	{
762 		/* Default */
763 		{
764 		  T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
765 		  /*vendor*/"*", /*product*/"*", /*revision*/"*"
766 		},
767 		/*quirks*/0
768 	},
769 };
770 
771 static	disk_strategy_t	adastrategy;
772 static	dumper_t	adadump;
773 static	periph_init_t	adainit;
774 static	void		adadiskgonecb(struct disk *dp);
775 static	periph_oninv_t	adaoninvalidate;
776 static	periph_dtor_t	adacleanup;
777 static	void		adaasync(void *callback_arg, u_int32_t code,
778 				struct cam_path *path, void *arg);
779 static	int		adazonemodesysctl(SYSCTL_HANDLER_ARGS);
780 static	int		adazonesupsysctl(SYSCTL_HANDLER_ARGS);
781 static	void		adasysctlinit(void *context, int pending);
782 static	int		adagetattr(struct bio *bp);
783 static	void		adasetflags(struct ada_softc *softc,
784 				    struct ccb_getdev *cgd);
785 static	periph_ctor_t	adaregister;
786 static	void		ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
787 				    struct ccb_ataio *ataio);
788 static	void 		ada_cfaerase(struct ada_softc *softc, struct bio *bp,
789 				     struct ccb_ataio *ataio);
790 static	int		ada_zone_bio_to_ata(int disk_zone_cmd);
791 static	int		ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
792 				     struct bio *bp, int *queue_ccb);
793 static	periph_start_t	adastart;
794 static	void		adaprobedone(struct cam_periph *periph, union ccb *ccb);
795 static	void		adazonedone(struct cam_periph *periph, union ccb *ccb);
796 static	void		adadone(struct cam_periph *periph,
797 			       union ccb *done_ccb);
798 static  int		adaerror(union ccb *ccb, u_int32_t cam_flags,
799 				u_int32_t sense_flags);
800 static void		adagetparams(struct cam_periph *periph,
801 				struct ccb_getdev *cgd);
802 static timeout_t	adasendorderedtag;
803 static void		adashutdown(void *arg, int howto);
804 static void		adasuspend(void *arg);
805 static void		adaresume(void *arg);
806 
807 #ifndef ADA_DEFAULT_TIMEOUT
808 #define ADA_DEFAULT_TIMEOUT 30	/* Timeout in seconds */
809 #endif
810 
811 #ifndef	ADA_DEFAULT_RETRY
812 #define	ADA_DEFAULT_RETRY	4
813 #endif
814 
815 #ifndef	ADA_DEFAULT_SEND_ORDERED
816 #define	ADA_DEFAULT_SEND_ORDERED	1
817 #endif
818 
819 #ifndef	ADA_DEFAULT_SPINDOWN_SHUTDOWN
820 #define	ADA_DEFAULT_SPINDOWN_SHUTDOWN	1
821 #endif
822 
823 #ifndef	ADA_DEFAULT_SPINDOWN_SUSPEND
824 #define	ADA_DEFAULT_SPINDOWN_SUSPEND	1
825 #endif
826 
827 #ifndef	ADA_DEFAULT_READ_AHEAD
828 #define	ADA_DEFAULT_READ_AHEAD	1
829 #endif
830 
831 #ifndef	ADA_DEFAULT_WRITE_CACHE
832 #define	ADA_DEFAULT_WRITE_CACHE	1
833 #endif
834 
835 #define	ADA_RA	(softc->read_ahead >= 0 ? \
836 		 softc->read_ahead : ada_read_ahead)
837 #define	ADA_WC	(softc->write_cache >= 0 ? \
838 		 softc->write_cache : ada_write_cache)
839 
840 /*
841  * Most platforms map firmware geometry to actual, but some don't.  If
842  * not overridden, default to nothing.
843  */
844 #ifndef ata_disk_firmware_geom_adjust
845 #define	ata_disk_firmware_geom_adjust(disk)
846 #endif
847 
848 static int ada_retry_count = ADA_DEFAULT_RETRY;
849 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
850 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
851 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
852 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
853 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
854 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
855 
856 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
857             "CAM Direct Access Disk driver");
858 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
859            &ada_retry_count, 0, "Normal I/O retry count");
860 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
861            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
862 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
863            &ada_send_ordered, 0, "Send Ordered Tags");
864 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
865            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
866 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
867            &ada_spindown_suspend, 0, "Spin down upon suspend");
868 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
869            &ada_read_ahead, 0, "Enable disk read-ahead");
870 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
871            &ada_write_cache, 0, "Enable disk write cache");
872 
873 /*
874  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
875  * to the default timeout, we check to see whether an ordered
876  * tagged transaction is appropriate to prevent simple tag
877  * starvation.  Since we'd like to ensure that there is at least
878  * 1/2 of the timeout length left for a starved transaction to
879  * complete after we've sent an ordered tag, we must poll at least
880  * four times in every timeout period.  This takes care of the worst
881  * case where a starved transaction starts during an interval that
882  * meets the requirement "don't send an ordered tag" test so it takes
883  * us two intervals to determine that a tag must be sent.
884  */
885 #ifndef ADA_ORDEREDTAG_INTERVAL
886 #define ADA_ORDEREDTAG_INTERVAL 4
887 #endif
888 
889 static struct periph_driver adadriver =
890 {
891 	adainit, "ada",
892 	TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
893 };
894 
895 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
896 
897 PERIPHDRIVER_DECLARE(ada, adadriver);
898 
899 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
900 
901 static int
902 adaopen(struct disk *dp)
903 {
904 	struct cam_periph *periph;
905 	struct ada_softc *softc;
906 	int error;
907 
908 	periph = (struct cam_periph *)dp->d_drv1;
909 	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
910 		return(ENXIO);
911 	}
912 
913 	cam_periph_lock(periph);
914 	if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
915 		cam_periph_unlock(periph);
916 		cam_periph_release(periph);
917 		return (error);
918 	}
919 
920 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
921 	    ("adaopen\n"));
922 
923 	softc = (struct ada_softc *)periph->softc;
924 	softc->flags |= ADA_FLAG_OPEN;
925 
926 	cam_periph_unhold(periph);
927 	cam_periph_unlock(periph);
928 	return (0);
929 }
930 
931 static int
932 adaclose(struct disk *dp)
933 {
934 	struct	cam_periph *periph;
935 	struct	ada_softc *softc;
936 	union ccb *ccb;
937 	int error;
938 
939 	periph = (struct cam_periph *)dp->d_drv1;
940 	softc = (struct ada_softc *)periph->softc;
941 	cam_periph_lock(periph);
942 
943 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
944 	    ("adaclose\n"));
945 
946 	/* We only sync the cache if the drive is capable of it. */
947 	if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
948 	    (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
949 	    (periph->flags & CAM_PERIPH_INVALID) == 0 &&
950 	    cam_periph_hold(periph, PRIBIO) == 0) {
951 
952 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
953 		cam_fill_ataio(&ccb->ataio,
954 				    1,
955 				    adadone,
956 				    CAM_DIR_NONE,
957 				    0,
958 				    NULL,
959 				    0,
960 				    ada_default_timeout*1000);
961 
962 		if (softc->flags & ADA_FLAG_CAN_48BIT)
963 			ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
964 		else
965 			ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
966 		error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
967 		    /*sense_flags*/0, softc->disk->d_devstat);
968 
969 		if (error != 0)
970 			xpt_print(periph->path, "Synchronize cache failed\n");
971 		softc->flags &= ~ADA_FLAG_DIRTY;
972 		xpt_release_ccb(ccb);
973 		cam_periph_unhold(periph);
974 	}
975 
976 	softc->flags &= ~ADA_FLAG_OPEN;
977 
978 	while (softc->refcount != 0)
979 		cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
980 	cam_periph_unlock(periph);
981 	cam_periph_release(periph);
982 	return (0);
983 }
984 
985 static void
986 adaschedule(struct cam_periph *periph)
987 {
988 	struct ada_softc *softc = (struct ada_softc *)periph->softc;
989 
990 	if (softc->state != ADA_STATE_NORMAL)
991 		return;
992 
993 	cam_iosched_schedule(softc->cam_iosched, periph);
994 }
995 
996 /*
997  * Actually translate the requested transfer into one the physical driver
998  * can understand.  The transfer is described by a buf and will include
999  * only one physical transfer.
1000  */
1001 static void
1002 adastrategy(struct bio *bp)
1003 {
1004 	struct cam_periph *periph;
1005 	struct ada_softc *softc;
1006 
1007 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1008 	softc = (struct ada_softc *)periph->softc;
1009 
1010 	cam_periph_lock(periph);
1011 
1012 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1013 
1014 	/*
1015 	 * If the device has been made invalid, error out
1016 	 */
1017 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1018 		cam_periph_unlock(periph);
1019 		biofinish(bp, NULL, ENXIO);
1020 		return;
1021 	}
1022 
1023 	/*
1024 	 * Zone commands must be ordered, because they can depend on the
1025 	 * effects of previously issued commands, and they may affect
1026 	 * commands after them.
1027 	 */
1028 	if (bp->bio_cmd == BIO_ZONE)
1029 		bp->bio_flags |= BIO_ORDERED;
1030 
1031 	/*
1032 	 * Place it in the queue of disk activities for this disk
1033 	 */
1034 	cam_iosched_queue_work(softc->cam_iosched, bp);
1035 
1036 	/*
1037 	 * Schedule ourselves for performing the work.
1038 	 */
1039 	adaschedule(periph);
1040 	cam_periph_unlock(periph);
1041 
1042 	return;
1043 }
1044 
1045 static int
1046 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1047 {
1048 	struct	    cam_periph *periph;
1049 	struct	    ada_softc *softc;
1050 	u_int	    secsize;
1051 	union	    ccb ccb;
1052 	struct	    disk *dp;
1053 	uint64_t    lba;
1054 	uint16_t    count;
1055 	int	    error = 0;
1056 
1057 	dp = arg;
1058 	periph = dp->d_drv1;
1059 	softc = (struct ada_softc *)periph->softc;
1060 	cam_periph_lock(periph);
1061 	secsize = softc->params.secsize;
1062 	lba = offset / secsize;
1063 	count = length / secsize;
1064 
1065 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1066 		cam_periph_unlock(periph);
1067 		return (ENXIO);
1068 	}
1069 
1070 	memset(&ccb, 0, sizeof(ccb));
1071 	if (length > 0) {
1072 		xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1073 		ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
1074 		cam_fill_ataio(&ccb.ataio,
1075 		    0,
1076 		    adadone,
1077 		    CAM_DIR_OUT,
1078 		    0,
1079 		    (u_int8_t *) virtual,
1080 		    length,
1081 		    ada_default_timeout*1000);
1082 		if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1083 		    (lba + count >= ATA_MAX_28BIT_LBA ||
1084 		    count >= 256)) {
1085 			ata_48bit_cmd(&ccb.ataio, ATA_WRITE_DMA48,
1086 			    0, lba, count);
1087 		} else {
1088 			ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA,
1089 			    0, lba, count);
1090 		}
1091 		xpt_polled_action(&ccb);
1092 
1093 		error = adaerror(&ccb,
1094 		    0, SF_NO_RECOVERY | SF_NO_RETRY);
1095 		if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
1096 			cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
1097 			    /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
1098 		if (error != 0)
1099 			printf("Aborting dump due to I/O error.\n");
1100 
1101 		cam_periph_unlock(periph);
1102 		return (error);
1103 	}
1104 
1105 	if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1106 		xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1107 
1108 		/*
1109 		 * Tell the drive to flush its internal cache. if we
1110 		 * can't flush in 5s we have big problems. No need to
1111 		 * wait the default 60s to detect problems.
1112 		 */
1113 		ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
1114 		cam_fill_ataio(&ccb.ataio,
1115 				    0,
1116 				    adadone,
1117 				    CAM_DIR_NONE,
1118 				    0,
1119 				    NULL,
1120 				    0,
1121 				    5*1000);
1122 
1123 		if (softc->flags & ADA_FLAG_CAN_48BIT)
1124 			ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1125 		else
1126 			ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0);
1127 		xpt_polled_action(&ccb);
1128 
1129 		error = adaerror(&ccb,
1130 		    0, SF_NO_RECOVERY | SF_NO_RETRY);
1131 		if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
1132 			cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
1133 			    /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
1134 		if (error != 0)
1135 			xpt_print(periph->path, "Synchronize cache failed\n");
1136 	}
1137 	cam_periph_unlock(periph);
1138 	return (error);
1139 }
1140 
1141 static void
1142 adainit(void)
1143 {
1144 	cam_status status;
1145 
1146 	/*
1147 	 * Install a global async callback.  This callback will
1148 	 * receive async callbacks like "new device found".
1149 	 */
1150 	status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1151 
1152 	if (status != CAM_REQ_CMP) {
1153 		printf("ada: Failed to attach master async callback "
1154 		       "due to status 0x%x!\n", status);
1155 	} else if (ada_send_ordered) {
1156 
1157 		/* Register our event handlers */
1158 		if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1159 					   NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1160 		    printf("adainit: power event registration failed!\n");
1161 		if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1162 					   NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1163 		    printf("adainit: power event registration failed!\n");
1164 		if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1165 					   NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1166 		    printf("adainit: shutdown event registration failed!\n");
1167 	}
1168 }
1169 
1170 /*
1171  * Callback from GEOM, called when it has finished cleaning up its
1172  * resources.
1173  */
1174 static void
1175 adadiskgonecb(struct disk *dp)
1176 {
1177 	struct cam_periph *periph;
1178 
1179 	periph = (struct cam_periph *)dp->d_drv1;
1180 
1181 	cam_periph_release(periph);
1182 }
1183 
1184 static void
1185 adaoninvalidate(struct cam_periph *periph)
1186 {
1187 	struct ada_softc *softc;
1188 
1189 	softc = (struct ada_softc *)periph->softc;
1190 
1191 	/*
1192 	 * De-register any async callbacks.
1193 	 */
1194 	xpt_register_async(0, adaasync, periph, periph->path);
1195 #ifdef CAM_IO_STATS
1196 	softc->invalidations++;
1197 #endif
1198 
1199 	/*
1200 	 * Return all queued I/O with ENXIO.
1201 	 * XXX Handle any transactions queued to the card
1202 	 *     with XPT_ABORT_CCB.
1203 	 */
1204 	cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1205 
1206 	disk_gone(softc->disk);
1207 }
1208 
1209 static void
1210 adacleanup(struct cam_periph *periph)
1211 {
1212 	struct ada_softc *softc;
1213 
1214 	softc = (struct ada_softc *)periph->softc;
1215 
1216 	cam_periph_unlock(periph);
1217 
1218 	cam_iosched_fini(softc->cam_iosched);
1219 
1220 	/*
1221 	 * If we can't free the sysctl tree, oh well...
1222 	 */
1223 	if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1224 #ifdef CAM_IO_STATS
1225 		if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1226 			xpt_print(periph->path,
1227 			    "can't remove sysctl stats context\n");
1228 #endif
1229 		if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1230 			xpt_print(periph->path,
1231 			    "can't remove sysctl context\n");
1232 	}
1233 
1234 	disk_destroy(softc->disk);
1235 	callout_drain(&softc->sendordered_c);
1236 	free(softc, M_DEVBUF);
1237 	cam_periph_lock(periph);
1238 }
1239 
1240 static void
1241 adasetdeletemethod(struct ada_softc *softc)
1242 {
1243 
1244 	if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1245 		softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1246 	else if (softc->flags & ADA_FLAG_CAN_TRIM)
1247 		softc->delete_method = ADA_DELETE_DSM_TRIM;
1248 	else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1249 		softc->delete_method = ADA_DELETE_CFA_ERASE;
1250 	else
1251 		softc->delete_method = ADA_DELETE_NONE;
1252 }
1253 
1254 static void
1255 adaasync(void *callback_arg, u_int32_t code,
1256 	struct cam_path *path, void *arg)
1257 {
1258 	struct ccb_getdev cgd;
1259 	struct cam_periph *periph;
1260 	struct ada_softc *softc;
1261 
1262 	periph = (struct cam_periph *)callback_arg;
1263 	switch (code) {
1264 	case AC_FOUND_DEVICE:
1265 	{
1266 		struct ccb_getdev *cgd;
1267 		cam_status status;
1268 
1269 		cgd = (struct ccb_getdev *)arg;
1270 		if (cgd == NULL)
1271 			break;
1272 
1273 		if (cgd->protocol != PROTO_ATA)
1274 			break;
1275 
1276 		/*
1277 		 * Allocate a peripheral instance for
1278 		 * this device and start the probe
1279 		 * process.
1280 		 */
1281 		status = cam_periph_alloc(adaregister, adaoninvalidate,
1282 					  adacleanup, adastart,
1283 					  "ada", CAM_PERIPH_BIO,
1284 					  path, adaasync,
1285 					  AC_FOUND_DEVICE, cgd);
1286 
1287 		if (status != CAM_REQ_CMP
1288 		 && status != CAM_REQ_INPROG)
1289 			printf("adaasync: Unable to attach to new device "
1290 				"due to status 0x%x\n", status);
1291 		break;
1292 	}
1293 	case AC_GETDEV_CHANGED:
1294 	{
1295 		softc = (struct ada_softc *)periph->softc;
1296 		xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1297 		cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1298 		xpt_action((union ccb *)&cgd);
1299 
1300 		/*
1301 		 * Set/clear support flags based on the new Identify data.
1302 		 */
1303 		adasetflags(softc, &cgd);
1304 
1305 		cam_periph_async(periph, code, path, arg);
1306 		break;
1307 	}
1308 	case AC_ADVINFO_CHANGED:
1309 	{
1310 		uintptr_t buftype;
1311 
1312 		buftype = (uintptr_t)arg;
1313 		if (buftype == CDAI_TYPE_PHYS_PATH) {
1314 			struct ada_softc *softc;
1315 
1316 			softc = periph->softc;
1317 			disk_attr_changed(softc->disk, "GEOM::physpath",
1318 					  M_NOWAIT);
1319 		}
1320 		break;
1321 	}
1322 	case AC_SENT_BDR:
1323 	case AC_BUS_RESET:
1324 	{
1325 		softc = (struct ada_softc *)periph->softc;
1326 		cam_periph_async(periph, code, path, arg);
1327 		if (softc->state != ADA_STATE_NORMAL)
1328 			break;
1329 		xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1330 		cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1331 		xpt_action((union ccb *)&cgd);
1332 		if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1333 			softc->state = ADA_STATE_RAHEAD;
1334 		else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1335 			softc->state = ADA_STATE_WCACHE;
1336 		else if ((softc->flags & ADA_FLAG_CAN_LOG)
1337 		      && (softc->zone_mode != ADA_ZONE_NONE))
1338 			softc->state = ADA_STATE_LOGDIR;
1339 		else
1340 		    break;
1341 		if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1342 			softc->state = ADA_STATE_NORMAL;
1343 		else
1344 			xpt_schedule(periph, CAM_PRIORITY_DEV);
1345 	}
1346 	default:
1347 		cam_periph_async(periph, code, path, arg);
1348 		break;
1349 	}
1350 }
1351 
1352 static int
1353 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1354 {
1355 	char tmpbuf[40];
1356 	struct ada_softc *softc;
1357 	int error;
1358 
1359 	softc = (struct ada_softc *)arg1;
1360 
1361 	switch (softc->zone_mode) {
1362 	case ADA_ZONE_DRIVE_MANAGED:
1363 		snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1364 		break;
1365 	case ADA_ZONE_HOST_AWARE:
1366 		snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1367 		break;
1368 	case ADA_ZONE_HOST_MANAGED:
1369 		snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1370 		break;
1371 	case ADA_ZONE_NONE:
1372 	default:
1373 		snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1374 		break;
1375 	}
1376 
1377 	error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1378 
1379 	return (error);
1380 }
1381 
1382 static int
1383 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1384 {
1385 	char tmpbuf[180];
1386 	struct ada_softc *softc;
1387 	struct sbuf sb;
1388 	int error, first;
1389 	unsigned int i;
1390 
1391 	softc = (struct ada_softc *)arg1;
1392 
1393 	error = 0;
1394 	first = 1;
1395 	sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1396 
1397 	for (i = 0; i < sizeof(ada_zone_desc_table) /
1398 	     sizeof(ada_zone_desc_table[0]); i++) {
1399 		if (softc->zone_flags & ada_zone_desc_table[i].value) {
1400 			if (first == 0)
1401 				sbuf_printf(&sb, ", ");
1402 			else
1403 				first = 0;
1404 			sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1405 		}
1406 	}
1407 
1408 	if (first == 1)
1409 		sbuf_printf(&sb, "None");
1410 
1411 	sbuf_finish(&sb);
1412 
1413 	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1414 
1415 	return (error);
1416 }
1417 
1418 
1419 static void
1420 adasysctlinit(void *context, int pending)
1421 {
1422 	struct cam_periph *periph;
1423 	struct ada_softc *softc;
1424 	char tmpstr[80], tmpstr2[80];
1425 
1426 	periph = (struct cam_periph *)context;
1427 
1428 	/* periph was held for us when this task was enqueued */
1429 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1430 		cam_periph_release(periph);
1431 		return;
1432 	}
1433 
1434 	softc = (struct ada_softc *)periph->softc;
1435 	snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1436 	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1437 
1438 	sysctl_ctx_init(&softc->sysctl_ctx);
1439 	softc->flags |= ADA_FLAG_SCTX_INIT;
1440 	softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1441 		SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1442 		CTLFLAG_RD, 0, tmpstr, "device_index");
1443 	if (softc->sysctl_tree == NULL) {
1444 		printf("adasysctlinit: unable to allocate sysctl tree\n");
1445 		cam_periph_release(periph);
1446 		return;
1447 	}
1448 
1449 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1450 		OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1451 		softc, 0, adadeletemethodsysctl, "A",
1452 		"BIO_DELETE execution method");
1453 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1454 		OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1455 		&softc->read_ahead, 0, "Enable disk read ahead.");
1456 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1457 		OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1458 		&softc->write_cache, 0, "Enable disk write cache.");
1459 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1460 		OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1461 		&softc->unmappedio, 0, "Unmapped I/O leaf");
1462 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1463 		OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1464 		&softc->rotating, 0, "Rotating media");
1465 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1466 		OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1467 		softc, 0, adazonemodesysctl, "A",
1468 		"Zone Mode");
1469 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1470 		OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1471 		softc, 0, adazonesupsysctl, "A",
1472 		"Zone Support");
1473 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1474 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1475 		"optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1476 		"Optimal Number of Open Sequential Write Preferred Zones");
1477 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1478 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1479 		"optimal_nonseq_zones", CTLFLAG_RD,
1480 		&softc->optimal_nonseq_zones,
1481 		"Optimal Number of Non-Sequentially Written Sequential Write "
1482 		"Preferred Zones");
1483 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1484 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1485 		"max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1486 		"Maximum Number of Open Sequential Write Required Zones");
1487 
1488 #ifdef ADA_TEST_FAILURE
1489 	/*
1490 	 * Add a 'door bell' sysctl which allows one to set it from userland
1491 	 * and cause something bad to happen.  For the moment, we only allow
1492 	 * whacking the next read or write.
1493 	 */
1494 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1495 		OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1496 		&softc->force_read_error, 0,
1497 		"Force a read error for the next N reads.");
1498 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1499 		OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1500 		&softc->force_write_error, 0,
1501 		"Force a write error for the next N writes.");
1502 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1503 		OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1504 		&softc->periodic_read_error, 0,
1505 		"Force a read error every N reads (don't set too low).");
1506 #endif
1507 
1508 #ifdef CAM_IO_STATS
1509 	softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1510 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1511 		CTLFLAG_RD, 0, "Statistics");
1512 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1513 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1514 		OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1515 		&softc->timeouts, 0,
1516 		"Device timeouts reported by the SIM");
1517 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1518 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1519 		OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1520 		&softc->errors, 0,
1521 		"Transport errors reported by the SIM.");
1522 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1523 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1524 		OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1525 		&softc->invalidations, 0,
1526 		"Device pack invalidations.");
1527 #endif
1528 
1529 	cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1530 	    softc->sysctl_tree);
1531 
1532 	cam_periph_release(periph);
1533 }
1534 
1535 static int
1536 adagetattr(struct bio *bp)
1537 {
1538 	int ret;
1539 	struct cam_periph *periph;
1540 
1541 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1542 	cam_periph_lock(periph);
1543 	ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1544 	    periph->path);
1545 	cam_periph_unlock(periph);
1546 	if (ret == 0)
1547 		bp->bio_completed = bp->bio_length;
1548 	return ret;
1549 }
1550 
1551 static int
1552 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1553 {
1554 	char buf[16];
1555 	const char *p;
1556 	struct ada_softc *softc;
1557 	int i, error, value, methods;
1558 
1559 	softc = (struct ada_softc *)arg1;
1560 
1561 	value = softc->delete_method;
1562 	if (value < 0 || value > ADA_DELETE_MAX)
1563 		p = "UNKNOWN";
1564 	else
1565 		p = ada_delete_method_names[value];
1566 	strncpy(buf, p, sizeof(buf));
1567 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1568 	if (error != 0 || req->newptr == NULL)
1569 		return (error);
1570 	methods = 1 << ADA_DELETE_DISABLE;
1571 	if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1572 	    !(softc->flags & ADA_FLAG_CAN_48BIT))
1573 		methods |= 1 << ADA_DELETE_CFA_ERASE;
1574 	if (softc->flags & ADA_FLAG_CAN_TRIM)
1575 		methods |= 1 << ADA_DELETE_DSM_TRIM;
1576 	if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1577 		methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1578 	for (i = 0; i <= ADA_DELETE_MAX; i++) {
1579 		if (!(methods & (1 << i)) ||
1580 		    strcmp(buf, ada_delete_method_names[i]) != 0)
1581 			continue;
1582 		softc->delete_method = i;
1583 		return (0);
1584 	}
1585 	return (EINVAL);
1586 }
1587 
1588 static void
1589 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1590 {
1591 	if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1592 	    (cgd->inq_flags & SID_DMA))
1593 		softc->flags |= ADA_FLAG_CAN_DMA;
1594 	else
1595 		softc->flags &= ~ADA_FLAG_CAN_DMA;
1596 
1597 	if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1598 		softc->flags |= ADA_FLAG_CAN_48BIT;
1599 		if (cgd->inq_flags & SID_DMA48)
1600 			softc->flags |= ADA_FLAG_CAN_DMA48;
1601 		else
1602 			softc->flags &= ~ADA_FLAG_CAN_DMA48;
1603 	} else
1604 		softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1605 
1606 	if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1607 		softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1608 	else
1609 		softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1610 
1611 	if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1612 		softc->flags |= ADA_FLAG_CAN_POWERMGT;
1613 	else
1614 		softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1615 
1616 	if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1617 	    (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1618 		softc->flags |= ADA_FLAG_CAN_NCQ;
1619 	else
1620 		softc->flags &= ~ADA_FLAG_CAN_NCQ;
1621 
1622 	if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1623 	    (cgd->inq_flags & SID_DMA)) {
1624 		softc->flags |= ADA_FLAG_CAN_TRIM;
1625 		softc->trim_max_ranges = TRIM_MAX_RANGES;
1626 		if (cgd->ident_data.max_dsm_blocks != 0) {
1627 			softc->trim_max_ranges =
1628 			    min(cgd->ident_data.max_dsm_blocks *
1629 				ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1630 		}
1631 		/*
1632 		 * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1633 		 * to do NCQ trims, if we support trims at all. We also need
1634 		 * support from the SIM to do things properly. Perhaps we
1635 		 * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1636 		 * set too...
1637 		 */
1638 		if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1639 		    (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1640 		    (cgd->ident_data.satacapabilities2 &
1641 		     ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1642 		    (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1643 			softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1644 		else
1645 			softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1646 	} else
1647 		softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1648 
1649 	if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1650 		softc->flags |= ADA_FLAG_CAN_CFA;
1651 	else
1652 		softc->flags &= ~ADA_FLAG_CAN_CFA;
1653 
1654 	/*
1655 	 * Now that we've set the appropriate flags, setup the delete
1656 	 * method.
1657 	 */
1658 	adasetdeletemethod(softc);
1659 
1660 	if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1661 	 && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1662 		softc->flags |= ADA_FLAG_CAN_LOG;
1663 	else
1664 		softc->flags &= ~ADA_FLAG_CAN_LOG;
1665 
1666 	if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1667 	     ATA_SUPPORT_ZONE_HOST_AWARE)
1668 		softc->zone_mode = ADA_ZONE_HOST_AWARE;
1669 	else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1670 		   ATA_SUPPORT_ZONE_DEV_MANAGED)
1671 	      || (softc->quirks & ADA_Q_SMR_DM))
1672 		softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1673 	else
1674 		softc->zone_mode = ADA_ZONE_NONE;
1675 
1676 	if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1677 		softc->flags |= ADA_FLAG_CAN_RAHEAD;
1678 	else
1679 		softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1680 
1681 	if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1682 		softc->flags |= ADA_FLAG_CAN_WCACHE;
1683 	else
1684 		softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1685 }
1686 
1687 static cam_status
1688 adaregister(struct cam_periph *periph, void *arg)
1689 {
1690 	struct ada_softc *softc;
1691 	struct ccb_pathinq cpi;
1692 	struct ccb_getdev *cgd;
1693 	struct disk_params *dp;
1694 	struct sbuf sb;
1695 	char   *announce_buf;
1696 	caddr_t match;
1697 	u_int maxio;
1698 	int quirks;
1699 
1700 	cgd = (struct ccb_getdev *)arg;
1701 	if (cgd == NULL) {
1702 		printf("adaregister: no getdev CCB, can't register device\n");
1703 		return(CAM_REQ_CMP_ERR);
1704 	}
1705 
1706 	softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1707 	    M_NOWAIT|M_ZERO);
1708 
1709 	if (softc == NULL) {
1710 		printf("adaregister: Unable to probe new device. "
1711 		    "Unable to allocate softc\n");
1712 		return(CAM_REQ_CMP_ERR);
1713 	}
1714 
1715 	announce_buf = softc->announce_temp;
1716 	bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1717 
1718 	if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1719 		printf("adaregister: Unable to probe new device. "
1720 		       "Unable to allocate iosched memory\n");
1721 		free(softc, M_DEVBUF);
1722 		return(CAM_REQ_CMP_ERR);
1723 	}
1724 
1725 	periph->softc = softc;
1726 
1727 	/*
1728 	 * See if this device has any quirks.
1729 	 */
1730 	match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1731 			       (caddr_t)ada_quirk_table,
1732 			       nitems(ada_quirk_table),
1733 			       sizeof(*ada_quirk_table), ata_identify_match);
1734 	if (match != NULL)
1735 		softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1736 	else
1737 		softc->quirks = ADA_Q_NONE;
1738 
1739 	bzero(&cpi, sizeof(cpi));
1740 	xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
1741 	cpi.ccb_h.func_code = XPT_PATH_INQ;
1742 	xpt_action((union ccb *)&cpi);
1743 
1744 	TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1745 
1746 	/*
1747 	 * Register this media as a disk
1748 	 */
1749 	(void)cam_periph_hold(periph, PRIBIO);
1750 	cam_periph_unlock(periph);
1751 	snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1752 	    "kern.cam.ada.%d.quirks", periph->unit_number);
1753 	quirks = softc->quirks;
1754 	TUNABLE_INT_FETCH(announce_buf, &quirks);
1755 	softc->quirks = quirks;
1756 	softc->read_ahead = -1;
1757 	snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1758 	    "kern.cam.ada.%d.read_ahead", periph->unit_number);
1759 	TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1760 	softc->write_cache = -1;
1761 	snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1762 	    "kern.cam.ada.%d.write_cache", periph->unit_number);
1763 	TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1764 
1765 	/*
1766 	 * Set support flags based on the Identify data and quirks.
1767 	 */
1768 	adasetflags(softc, cgd);
1769 
1770 	/* Disable queue sorting for non-rotational media by default. */
1771 	if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1772 		softc->rotating = 0;
1773 	} else {
1774 		softc->rotating = 1;
1775 	}
1776 	cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1777 	adagetparams(periph, cgd);
1778 	softc->disk = disk_alloc();
1779 	softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1780 	softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1781 			  periph->unit_number, softc->params.secsize,
1782 			  DEVSTAT_ALL_SUPPORTED,
1783 			  DEVSTAT_TYPE_DIRECT |
1784 			  XPORT_DEVSTAT_TYPE(cpi.transport),
1785 			  DEVSTAT_PRIORITY_DISK);
1786 	softc->disk->d_open = adaopen;
1787 	softc->disk->d_close = adaclose;
1788 	softc->disk->d_strategy = adastrategy;
1789 	softc->disk->d_getattr = adagetattr;
1790 	softc->disk->d_dump = adadump;
1791 	softc->disk->d_gone = adadiskgonecb;
1792 	softc->disk->d_name = "ada";
1793 	softc->disk->d_drv1 = periph;
1794 	maxio = cpi.maxio;		/* Honor max I/O size of SIM */
1795 	if (maxio == 0)
1796 		maxio = DFLTPHYS;	/* traditional default */
1797 	else if (maxio > MAXPHYS)
1798 		maxio = MAXPHYS;	/* for safety */
1799 	if (softc->flags & ADA_FLAG_CAN_48BIT)
1800 		maxio = min(maxio, 65536 * softc->params.secsize);
1801 	else					/* 28bit ATA command limit */
1802 		maxio = min(maxio, 256 * softc->params.secsize);
1803 	softc->disk->d_maxsize = maxio;
1804 	softc->disk->d_unit = periph->unit_number;
1805 	softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1806 	if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1807 		softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1808 	if (softc->flags & ADA_FLAG_CAN_TRIM) {
1809 		softc->disk->d_flags |= DISKFLAG_CANDELETE;
1810 		softc->disk->d_delmaxsize = softc->params.secsize *
1811 					    ATA_DSM_RANGE_MAX *
1812 					    softc->trim_max_ranges;
1813 	} else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1814 	    !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1815 		softc->disk->d_flags |= DISKFLAG_CANDELETE;
1816 		softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1817 	} else
1818 		softc->disk->d_delmaxsize = maxio;
1819 	if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1820 		softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1821 		softc->unmappedio = 1;
1822 	}
1823 	if (cpi.hba_misc & PIM_ATA_EXT)
1824 		softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1825 	strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1826 	    MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1827 	strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1828 	    MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1829 	softc->disk->d_hba_vendor = cpi.hba_vendor;
1830 	softc->disk->d_hba_device = cpi.hba_device;
1831 	softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1832 	softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1833 
1834 	softc->disk->d_sectorsize = softc->params.secsize;
1835 	softc->disk->d_mediasize = (off_t)softc->params.sectors *
1836 	    softc->params.secsize;
1837 	if (ata_physical_sector_size(&cgd->ident_data) !=
1838 	    softc->params.secsize) {
1839 		softc->disk->d_stripesize =
1840 		    ata_physical_sector_size(&cgd->ident_data);
1841 		softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1842 		    ata_logical_sector_offset(&cgd->ident_data)) %
1843 		    softc->disk->d_stripesize;
1844 	} else if (softc->quirks & ADA_Q_4K) {
1845 		softc->disk->d_stripesize = 4096;
1846 		softc->disk->d_stripeoffset = 0;
1847 	}
1848 	softc->disk->d_fwsectors = softc->params.secs_per_track;
1849 	softc->disk->d_fwheads = softc->params.heads;
1850 	ata_disk_firmware_geom_adjust(softc->disk);
1851 
1852 	/*
1853 	 * Acquire a reference to the periph before we register with GEOM.
1854 	 * We'll release this reference once GEOM calls us back (via
1855 	 * adadiskgonecb()) telling us that our provider has been freed.
1856 	 */
1857 	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
1858 		xpt_print(periph->path, "%s: lost periph during "
1859 			  "registration!\n", __func__);
1860 		cam_periph_lock(periph);
1861 		return (CAM_REQ_CMP_ERR);
1862 	}
1863 	disk_create(softc->disk, DISK_VERSION);
1864 	cam_periph_lock(periph);
1865 
1866 	dp = &softc->params;
1867 	snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1868 	    "%juMB (%ju %u byte sectors)",
1869 	    ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1870 	    (uintmax_t)dp->sectors, dp->secsize);
1871 
1872 	sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1873 	xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1874 	xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1875 	sbuf_finish(&sb);
1876 	sbuf_putbuf(&sb);
1877 
1878 	/*
1879 	 * Create our sysctl variables, now that we know
1880 	 * we have successfully attached.
1881 	 */
1882 	if (cam_periph_acquire(periph) == CAM_REQ_CMP)
1883 		taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1884 
1885 	/*
1886 	 * Add async callbacks for bus reset and
1887 	 * bus device reset calls.  I don't bother
1888 	 * checking if this fails as, in most cases,
1889 	 * the system will function just fine without
1890 	 * them and the only alternative would be to
1891 	 * not attach the device on failure.
1892 	 */
1893 	xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1894 	    AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1895 	    adaasync, periph, periph->path);
1896 
1897 	/*
1898 	 * Schedule a periodic event to occasionally send an
1899 	 * ordered tag to a device.
1900 	 */
1901 	callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1902 	callout_reset(&softc->sendordered_c,
1903 	    (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1904 	    adasendorderedtag, softc);
1905 
1906 	if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1907 		softc->state = ADA_STATE_RAHEAD;
1908 	} else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1909 		softc->state = ADA_STATE_WCACHE;
1910 	} else if ((softc->flags & ADA_FLAG_CAN_LOG)
1911 		&& (softc->zone_mode != ADA_ZONE_NONE)) {
1912 		softc->state = ADA_STATE_LOGDIR;
1913 	} else {
1914 		/*
1915 		 * Nothing to probe, so we can just transition to the
1916 		 * normal state.
1917 		 */
1918 		adaprobedone(periph, NULL);
1919 		return(CAM_REQ_CMP);
1920 	}
1921 
1922 	xpt_schedule(periph, CAM_PRIORITY_DEV);
1923 
1924 	return(CAM_REQ_CMP);
1925 }
1926 
1927 static int
1928 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1929 {
1930 	uint64_t lastlba = (uint64_t)-1;
1931 	int c, lastcount = 0, off, ranges = 0;
1932 
1933 	bzero(req, sizeof(*req));
1934 	TAILQ_INIT(&req->bps);
1935 	do {
1936 		uint64_t lba = bp->bio_pblkno;
1937 		int count = bp->bio_bcount / softc->params.secsize;
1938 
1939 		/* Try to extend the previous range. */
1940 		if (lba == lastlba) {
1941 			c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1942 			lastcount += c;
1943 			off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1944 			req->data[off + 6] = lastcount & 0xff;
1945 			req->data[off + 7] =
1946 				(lastcount >> 8) & 0xff;
1947 			count -= c;
1948 			lba += c;
1949 		}
1950 
1951 		while (count > 0) {
1952 			c = min(count, ATA_DSM_RANGE_MAX);
1953 			off = ranges * ATA_DSM_RANGE_SIZE;
1954 			req->data[off + 0] = lba & 0xff;
1955 			req->data[off + 1] = (lba >> 8) & 0xff;
1956 			req->data[off + 2] = (lba >> 16) & 0xff;
1957 			req->data[off + 3] = (lba >> 24) & 0xff;
1958 			req->data[off + 4] = (lba >> 32) & 0xff;
1959 			req->data[off + 5] = (lba >> 40) & 0xff;
1960 			req->data[off + 6] = c & 0xff;
1961 			req->data[off + 7] = (c >> 8) & 0xff;
1962 			lba += c;
1963 			count -= c;
1964 			lastcount = c;
1965 			ranges++;
1966 			/*
1967 			 * Its the caller's responsibility to ensure the
1968 			 * request will fit so we don't need to check for
1969 			 * overrun here
1970 			 */
1971 		}
1972 		lastlba = lba;
1973 		TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1974 
1975 		bp = cam_iosched_next_trim(softc->cam_iosched);
1976 		if (bp == NULL)
1977 			break;
1978 		if (bp->bio_bcount / softc->params.secsize >
1979 		    (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1980 			cam_iosched_put_back_trim(softc->cam_iosched, bp);
1981 			break;
1982 		}
1983 	} while (1);
1984 
1985 	return (ranges);
1986 }
1987 
1988 static void
1989 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1990 {
1991 	struct trim_request *req = &softc->trim_req;
1992 	int ranges;
1993 
1994 	ranges = ada_dsmtrim_req_create(softc, bp, req);
1995 	cam_fill_ataio(ataio,
1996 	    ada_retry_count,
1997 	    adadone,
1998 	    CAM_DIR_OUT,
1999 	    0,
2000 	    req->data,
2001 	    howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2002 	    ada_default_timeout * 1000);
2003 	ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
2004 	    ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
2005 }
2006 
2007 static void
2008 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2009 {
2010 	struct trim_request *req = &softc->trim_req;
2011 	int ranges;
2012 
2013 	ranges = ada_dsmtrim_req_create(softc, bp, req);
2014 	cam_fill_ataio(ataio,
2015 	    ada_retry_count,
2016 	    adadone,
2017 	    CAM_DIR_OUT,
2018 	    0,
2019 	    req->data,
2020 	    howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2021 	    ada_default_timeout * 1000);
2022 	ata_ncq_cmd(ataio,
2023 	    ATA_SEND_FPDMA_QUEUED,
2024 	    0,
2025 	    howmany(ranges, ATA_DSM_BLK_RANGES));
2026 	ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2027 	ataio->ata_flags |= ATA_FLAG_AUX;
2028 	ataio->aux = 1;
2029 }
2030 
2031 static void
2032 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2033 {
2034 	struct trim_request *req = &softc->trim_req;
2035 	uint64_t lba = bp->bio_pblkno;
2036 	uint16_t count = bp->bio_bcount / softc->params.secsize;
2037 
2038 	bzero(req, sizeof(*req));
2039 	TAILQ_INIT(&req->bps);
2040 	TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2041 
2042 	cam_fill_ataio(ataio,
2043 	    ada_retry_count,
2044 	    adadone,
2045 	    CAM_DIR_NONE,
2046 	    0,
2047 	    NULL,
2048 	    0,
2049 	    ada_default_timeout*1000);
2050 
2051 	if (count >= 256)
2052 		count = 0;
2053 	ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2054 }
2055 
2056 static int
2057 ada_zone_bio_to_ata(int disk_zone_cmd)
2058 {
2059 	switch (disk_zone_cmd) {
2060 	case DISK_ZONE_OPEN:
2061 		return ATA_ZM_OPEN_ZONE;
2062 	case DISK_ZONE_CLOSE:
2063 		return ATA_ZM_CLOSE_ZONE;
2064 	case DISK_ZONE_FINISH:
2065 		return ATA_ZM_FINISH_ZONE;
2066 	case DISK_ZONE_RWP:
2067 		return ATA_ZM_RWP;
2068 	}
2069 
2070 	return -1;
2071 }
2072 
2073 static int
2074 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2075 	     int *queue_ccb)
2076 {
2077 	struct ada_softc *softc;
2078 	int error;
2079 
2080 	error = 0;
2081 
2082 	if (bp->bio_cmd != BIO_ZONE) {
2083 		error = EINVAL;
2084 		goto bailout;
2085 	}
2086 
2087 	softc = periph->softc;
2088 
2089 	switch (bp->bio_zone.zone_cmd) {
2090 	case DISK_ZONE_OPEN:
2091 	case DISK_ZONE_CLOSE:
2092 	case DISK_ZONE_FINISH:
2093 	case DISK_ZONE_RWP: {
2094 		int zone_flags;
2095 		int zone_sa;
2096 		uint64_t lba;
2097 
2098 		zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2099 		if (zone_sa == -1) {
2100 			xpt_print(periph->path, "Cannot translate zone "
2101 			    "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2102 			error = EINVAL;
2103 			goto bailout;
2104 		}
2105 
2106 		zone_flags = 0;
2107 		lba = bp->bio_zone.zone_params.rwp.id;
2108 
2109 		if (bp->bio_zone.zone_params.rwp.flags &
2110 		    DISK_ZONE_RWP_FLAG_ALL)
2111 			zone_flags |= ZBC_OUT_ALL;
2112 
2113 		ata_zac_mgmt_out(&ccb->ataio,
2114 				 /*retries*/ ada_retry_count,
2115 				 /*cbfcnp*/ adadone,
2116 				 /*use_ncq*/ (softc->flags &
2117 					      ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2118 				 /*zm_action*/ zone_sa,
2119 				 /*zone_id*/ lba,
2120 				 /*zone_flags*/ zone_flags,
2121 				 /*sector_count*/ 0,
2122 				 /*data_ptr*/ NULL,
2123 				 /*dxfer_len*/ 0,
2124 				 /*timeout*/ ada_default_timeout * 1000);
2125 		*queue_ccb = 1;
2126 
2127 		break;
2128 	}
2129 	case DISK_ZONE_REPORT_ZONES: {
2130 		uint8_t *rz_ptr;
2131 		uint32_t num_entries, alloc_size;
2132 		struct disk_zone_report *rep;
2133 
2134 		rep = &bp->bio_zone.zone_params.report;
2135 
2136 		num_entries = rep->entries_allocated;
2137 		if (num_entries == 0) {
2138 			xpt_print(periph->path, "No entries allocated for "
2139 			    "Report Zones request\n");
2140 			error = EINVAL;
2141 			goto bailout;
2142 		}
2143 		alloc_size = sizeof(struct scsi_report_zones_hdr) +
2144 		    (sizeof(struct scsi_report_zones_desc) * num_entries);
2145 		alloc_size = min(alloc_size, softc->disk->d_maxsize);
2146 		rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2147 		if (rz_ptr == NULL) {
2148 			xpt_print(periph->path, "Unable to allocate memory "
2149 			   "for Report Zones request\n");
2150 			error = ENOMEM;
2151 			goto bailout;
2152 		}
2153 
2154 		ata_zac_mgmt_in(&ccb->ataio,
2155 				/*retries*/ ada_retry_count,
2156 				/*cbcfnp*/ adadone,
2157 				/*use_ncq*/ (softc->flags &
2158 					     ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2159 				/*zm_action*/ ATA_ZM_REPORT_ZONES,
2160 				/*zone_id*/ rep->starting_id,
2161 				/*zone_flags*/ rep->rep_options,
2162 				/*data_ptr*/ rz_ptr,
2163 				/*dxfer_len*/ alloc_size,
2164 				/*timeout*/ ada_default_timeout * 1000);
2165 
2166 		/*
2167 		 * For BIO_ZONE, this isn't normally needed.  However, it
2168 		 * is used by devstat_end_transaction_bio() to determine
2169 		 * how much data was transferred.
2170 		 */
2171 		/*
2172 		 * XXX KDM we have a problem.  But I'm not sure how to fix
2173 		 * it.  devstat uses bio_bcount - bio_resid to calculate
2174 		 * the amount of data transferred.   The GEOM disk code
2175 		 * uses bio_length - bio_resid to calculate the amount of
2176 		 * data in bio_completed.  We have different structure
2177 		 * sizes above and below the ada(4) driver.  So, if we
2178 		 * use the sizes above, the amount transferred won't be
2179 		 * quite accurate for devstat.  If we use different sizes
2180 		 * for bio_bcount and bio_length (above and below
2181 		 * respectively), then the residual needs to match one or
2182 		 * the other.  Everything is calculated after the bio
2183 		 * leaves the driver, so changing the values around isn't
2184 		 * really an option.  For now, just set the count to the
2185 		 * passed in length.  This means that the calculations
2186 		 * above (e.g. bio_completed) will be correct, but the
2187 		 * amount of data reported to devstat will be slightly
2188 		 * under or overstated.
2189 		 */
2190 		bp->bio_bcount = bp->bio_length;
2191 
2192 		*queue_ccb = 1;
2193 
2194 		break;
2195 	}
2196 	case DISK_ZONE_GET_PARAMS: {
2197 		struct disk_zone_disk_params *params;
2198 
2199 		params = &bp->bio_zone.zone_params.disk_params;
2200 		bzero(params, sizeof(*params));
2201 
2202 		switch (softc->zone_mode) {
2203 		case ADA_ZONE_DRIVE_MANAGED:
2204 			params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2205 			break;
2206 		case ADA_ZONE_HOST_AWARE:
2207 			params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2208 			break;
2209 		case ADA_ZONE_HOST_MANAGED:
2210 			params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2211 			break;
2212 		default:
2213 		case ADA_ZONE_NONE:
2214 			params->zone_mode = DISK_ZONE_MODE_NONE;
2215 			break;
2216 		}
2217 
2218 		if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2219 			params->flags |= DISK_ZONE_DISK_URSWRZ;
2220 
2221 		if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2222 			params->optimal_seq_zones = softc->optimal_seq_zones;
2223 			params->flags |= DISK_ZONE_OPT_SEQ_SET;
2224 		}
2225 
2226 		if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2227 			params->optimal_nonseq_zones =
2228 			    softc->optimal_nonseq_zones;
2229 			params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2230 		}
2231 
2232 		if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2233 			params->max_seq_zones = softc->max_seq_zones;
2234 			params->flags |= DISK_ZONE_MAX_SEQ_SET;
2235 		}
2236 		if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2237 			params->flags |= DISK_ZONE_RZ_SUP;
2238 
2239 		if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2240 			params->flags |= DISK_ZONE_OPEN_SUP;
2241 
2242 		if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2243 			params->flags |= DISK_ZONE_CLOSE_SUP;
2244 
2245 		if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2246 			params->flags |= DISK_ZONE_FINISH_SUP;
2247 
2248 		if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2249 			params->flags |= DISK_ZONE_RWP_SUP;
2250 		break;
2251 	}
2252 	default:
2253 		break;
2254 	}
2255 bailout:
2256 	return (error);
2257 }
2258 
2259 static void
2260 adastart(struct cam_periph *periph, union ccb *start_ccb)
2261 {
2262 	struct ada_softc *softc = (struct ada_softc *)periph->softc;
2263 	struct ccb_ataio *ataio = &start_ccb->ataio;
2264 
2265 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2266 
2267 	switch (softc->state) {
2268 	case ADA_STATE_NORMAL:
2269 	{
2270 		struct bio *bp;
2271 		u_int8_t tag_code;
2272 
2273 		bp = cam_iosched_next_bio(softc->cam_iosched);
2274 		if (bp == NULL) {
2275 			xpt_release_ccb(start_ccb);
2276 			break;
2277 		}
2278 
2279 		if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2280 		    (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2281 			softc->flags &= ~ADA_FLAG_NEED_OTAG;
2282 			softc->flags |= ADA_FLAG_WAS_OTAG;
2283 			tag_code = 0;
2284 		} else {
2285 			tag_code = 1;
2286 		}
2287 		switch (bp->bio_cmd) {
2288 		case BIO_WRITE:
2289 		case BIO_READ:
2290 		{
2291 			uint64_t lba = bp->bio_pblkno;
2292 			uint16_t count = bp->bio_bcount / softc->params.secsize;
2293 			void *data_ptr;
2294 			int rw_op;
2295 
2296 			if (bp->bio_cmd == BIO_WRITE) {
2297 				softc->flags |= ADA_FLAG_DIRTY;
2298 				rw_op = CAM_DIR_OUT;
2299 			} else {
2300 				rw_op = CAM_DIR_IN;
2301 			}
2302 
2303 			data_ptr = bp->bio_data;
2304 			if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2305 				rw_op |= CAM_DATA_BIO;
2306 				data_ptr = bp;
2307 			}
2308 
2309 #ifdef ADA_TEST_FAILURE
2310 			int fail = 0;
2311 
2312 			/*
2313 			 * Support the failure ioctls.  If the command is a
2314 			 * read, and there are pending forced read errors, or
2315 			 * if a write and pending write errors, then fail this
2316 			 * operation with EIO.  This is useful for testing
2317 			 * purposes.  Also, support having every Nth read fail.
2318 			 *
2319 			 * This is a rather blunt tool.
2320 			 */
2321 			if (bp->bio_cmd == BIO_READ) {
2322 				if (softc->force_read_error) {
2323 					softc->force_read_error--;
2324 					fail = 1;
2325 				}
2326 				if (softc->periodic_read_error > 0) {
2327 					if (++softc->periodic_read_count >=
2328 					    softc->periodic_read_error) {
2329 						softc->periodic_read_count = 0;
2330 						fail = 1;
2331 					}
2332 				}
2333 			} else {
2334 				if (softc->force_write_error) {
2335 					softc->force_write_error--;
2336 					fail = 1;
2337 				}
2338 			}
2339 			if (fail) {
2340 				biofinish(bp, NULL, EIO);
2341 				xpt_release_ccb(start_ccb);
2342 				adaschedule(periph);
2343 				return;
2344 			}
2345 #endif
2346 			KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2347 			    round_page(bp->bio_bcount + bp->bio_ma_offset) /
2348 			    PAGE_SIZE == bp->bio_ma_n,
2349 			    ("Short bio %p", bp));
2350 			cam_fill_ataio(ataio,
2351 			    ada_retry_count,
2352 			    adadone,
2353 			    rw_op,
2354 			    0,
2355 			    data_ptr,
2356 			    bp->bio_bcount,
2357 			    ada_default_timeout*1000);
2358 
2359 			if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2360 				if (bp->bio_cmd == BIO_READ) {
2361 					ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2362 					    lba, count);
2363 				} else {
2364 					ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2365 					    lba, count);
2366 				}
2367 			} else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2368 			    (lba + count >= ATA_MAX_28BIT_LBA ||
2369 			    count > 256)) {
2370 				if (softc->flags & ADA_FLAG_CAN_DMA48) {
2371 					if (bp->bio_cmd == BIO_READ) {
2372 						ata_48bit_cmd(ataio, ATA_READ_DMA48,
2373 						    0, lba, count);
2374 					} else {
2375 						ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2376 						    0, lba, count);
2377 					}
2378 				} else {
2379 					if (bp->bio_cmd == BIO_READ) {
2380 						ata_48bit_cmd(ataio, ATA_READ_MUL48,
2381 						    0, lba, count);
2382 					} else {
2383 						ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2384 						    0, lba, count);
2385 					}
2386 				}
2387 			} else {
2388 				if (count == 256)
2389 					count = 0;
2390 				if (softc->flags & ADA_FLAG_CAN_DMA) {
2391 					if (bp->bio_cmd == BIO_READ) {
2392 						ata_28bit_cmd(ataio, ATA_READ_DMA,
2393 						    0, lba, count);
2394 					} else {
2395 						ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2396 						    0, lba, count);
2397 					}
2398 				} else {
2399 					if (bp->bio_cmd == BIO_READ) {
2400 						ata_28bit_cmd(ataio, ATA_READ_MUL,
2401 						    0, lba, count);
2402 					} else {
2403 						ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2404 						    0, lba, count);
2405 					}
2406 				}
2407 			}
2408 			break;
2409 		}
2410 		case BIO_DELETE:
2411 			switch (softc->delete_method) {
2412 			case ADA_DELETE_NCQ_DSM_TRIM:
2413 				ada_ncq_dsmtrim(softc, bp, ataio);
2414 				break;
2415 			case ADA_DELETE_DSM_TRIM:
2416 				ada_dsmtrim(softc, bp, ataio);
2417 				break;
2418 			case ADA_DELETE_CFA_ERASE:
2419 				ada_cfaerase(softc, bp, ataio);
2420 				break;
2421 			default:
2422 				biofinish(bp, NULL, EOPNOTSUPP);
2423 				xpt_release_ccb(start_ccb);
2424 				adaschedule(periph);
2425 				return;
2426 			}
2427 			start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2428 			start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2429 			cam_iosched_submit_trim(softc->cam_iosched);
2430 			goto out;
2431 		case BIO_FLUSH:
2432 			cam_fill_ataio(ataio,
2433 			    1,
2434 			    adadone,
2435 			    CAM_DIR_NONE,
2436 			    0,
2437 			    NULL,
2438 			    0,
2439 			    ada_default_timeout*1000);
2440 
2441 			if (softc->flags & ADA_FLAG_CAN_48BIT)
2442 				ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2443 			else
2444 				ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2445 			break;
2446 		case BIO_ZONE: {
2447 			int error, queue_ccb;
2448 
2449 			queue_ccb = 0;
2450 
2451 			error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2452 			if ((error != 0)
2453 			 || (queue_ccb == 0)) {
2454 				biofinish(bp, NULL, error);
2455 				xpt_release_ccb(start_ccb);
2456 				return;
2457 			}
2458 			break;
2459 		}
2460 		}
2461 		start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2462 		start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2463 out:
2464 		start_ccb->ccb_h.ccb_bp = bp;
2465 		softc->outstanding_cmds++;
2466 		softc->refcount++;
2467 		cam_periph_unlock(periph);
2468 		xpt_action(start_ccb);
2469 		cam_periph_lock(periph);
2470 		softc->refcount--;
2471 
2472 		/* May have more work to do, so ensure we stay scheduled */
2473 		adaschedule(periph);
2474 		break;
2475 	}
2476 	case ADA_STATE_RAHEAD:
2477 	case ADA_STATE_WCACHE:
2478 	{
2479 		cam_fill_ataio(ataio,
2480 		    1,
2481 		    adadone,
2482 		    CAM_DIR_NONE,
2483 		    0,
2484 		    NULL,
2485 		    0,
2486 		    ada_default_timeout*1000);
2487 
2488 		if (softc->state == ADA_STATE_RAHEAD) {
2489 			ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2490 			    ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2491 			start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2492 		} else {
2493 			ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2494 			    ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2495 			start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2496 		}
2497 		start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2498 		xpt_action(start_ccb);
2499 		break;
2500 	}
2501 	case ADA_STATE_LOGDIR:
2502 	{
2503 		struct ata_gp_log_dir *log_dir;
2504 
2505 		if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2506 			adaprobedone(periph, start_ccb);
2507 			break;
2508 		}
2509 
2510 		log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2511 		if (log_dir == NULL) {
2512 			xpt_print(periph->path, "Couldn't malloc log_dir "
2513 			    "data\n");
2514 			softc->state = ADA_STATE_NORMAL;
2515 			xpt_release_ccb(start_ccb);
2516 			break;
2517 		}
2518 
2519 
2520 		ata_read_log(ataio,
2521 		    /*retries*/1,
2522 		    /*cbfcnp*/adadone,
2523 		    /*log_address*/ ATA_LOG_DIRECTORY,
2524 		    /*page_number*/ 0,
2525 		    /*block_count*/ 1,
2526 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2527 				 CAM_ATAIO_DMA : 0,
2528 		    /*data_ptr*/ (uint8_t *)log_dir,
2529 		    /*dxfer_len*/sizeof(*log_dir),
2530 		    /*timeout*/ada_default_timeout*1000);
2531 
2532 		start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2533 		xpt_action(start_ccb);
2534 		break;
2535 	}
2536 	case ADA_STATE_IDDIR:
2537 	{
2538 		struct ata_identify_log_pages *id_dir;
2539 
2540 		id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2541 		if (id_dir == NULL) {
2542 			xpt_print(periph->path, "Couldn't malloc id_dir "
2543 			    "data\n");
2544 			adaprobedone(periph, start_ccb);
2545 			break;
2546 		}
2547 
2548 		ata_read_log(ataio,
2549 		    /*retries*/1,
2550 		    /*cbfcnp*/adadone,
2551 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2552 		    /*page_number*/ ATA_IDL_PAGE_LIST,
2553 		    /*block_count*/ 1,
2554 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2555 				 CAM_ATAIO_DMA : 0,
2556 		    /*data_ptr*/ (uint8_t *)id_dir,
2557 		    /*dxfer_len*/ sizeof(*id_dir),
2558 		    /*timeout*/ada_default_timeout*1000);
2559 
2560 		start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2561 		xpt_action(start_ccb);
2562 		break;
2563 	}
2564 	case ADA_STATE_SUP_CAP:
2565 	{
2566 		struct ata_identify_log_sup_cap *sup_cap;
2567 
2568 		sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2569 		if (sup_cap == NULL) {
2570 			xpt_print(periph->path, "Couldn't malloc sup_cap "
2571 			    "data\n");
2572 			adaprobedone(periph, start_ccb);
2573 			break;
2574 		}
2575 
2576 		ata_read_log(ataio,
2577 		    /*retries*/1,
2578 		    /*cbfcnp*/adadone,
2579 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2580 		    /*page_number*/ ATA_IDL_SUP_CAP,
2581 		    /*block_count*/ 1,
2582 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2583 				 CAM_ATAIO_DMA : 0,
2584 		    /*data_ptr*/ (uint8_t *)sup_cap,
2585 		    /*dxfer_len*/ sizeof(*sup_cap),
2586 		    /*timeout*/ada_default_timeout*1000);
2587 
2588 		start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2589 		xpt_action(start_ccb);
2590 		break;
2591 	}
2592 	case ADA_STATE_ZONE:
2593 	{
2594 		struct ata_zoned_info_log *ata_zone;
2595 
2596 		ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2597 		if (ata_zone == NULL) {
2598 			xpt_print(periph->path, "Couldn't malloc ata_zone "
2599 			    "data\n");
2600 			adaprobedone(periph, start_ccb);
2601 			break;
2602 		}
2603 
2604 		ata_read_log(ataio,
2605 		    /*retries*/1,
2606 		    /*cbfcnp*/adadone,
2607 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2608 		    /*page_number*/ ATA_IDL_ZDI,
2609 		    /*block_count*/ 1,
2610 		    /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2611 				 CAM_ATAIO_DMA : 0,
2612 		    /*data_ptr*/ (uint8_t *)ata_zone,
2613 		    /*dxfer_len*/ sizeof(*ata_zone),
2614 		    /*timeout*/ada_default_timeout*1000);
2615 
2616 		start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2617 		xpt_action(start_ccb);
2618 		break;
2619 	}
2620 	}
2621 }
2622 
2623 static void
2624 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2625 {
2626 	struct ada_softc *softc;
2627 
2628 	softc = (struct ada_softc *)periph->softc;
2629 
2630 	if (ccb != NULL)
2631 		xpt_release_ccb(ccb);
2632 
2633 	softc->state = ADA_STATE_NORMAL;
2634 	softc->flags |= ADA_FLAG_PROBED;
2635 	adaschedule(periph);
2636 	if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2637 		softc->flags |= ADA_FLAG_ANNOUNCED;
2638 		cam_periph_unhold(periph);
2639 	} else {
2640 		cam_periph_release_locked(periph);
2641 	}
2642 }
2643 
2644 static void
2645 adazonedone(struct cam_periph *periph, union ccb *ccb)
2646 {
2647 	struct ada_softc *softc;
2648 	struct bio *bp;
2649 
2650 	softc = periph->softc;
2651 	bp = (struct bio *)ccb->ccb_h.ccb_bp;
2652 
2653 	switch (bp->bio_zone.zone_cmd) {
2654 	case DISK_ZONE_OPEN:
2655 	case DISK_ZONE_CLOSE:
2656 	case DISK_ZONE_FINISH:
2657 	case DISK_ZONE_RWP:
2658 		break;
2659 	case DISK_ZONE_REPORT_ZONES: {
2660 		uint32_t avail_len;
2661 		struct disk_zone_report *rep;
2662 		struct scsi_report_zones_hdr *hdr;
2663 		struct scsi_report_zones_desc *desc;
2664 		struct disk_zone_rep_entry *entry;
2665 		uint32_t num_alloced, hdr_len, num_avail;
2666 		uint32_t num_to_fill, i;
2667 
2668 		rep = &bp->bio_zone.zone_params.report;
2669 		avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2670 		/*
2671 		 * Note that bio_resid isn't normally used for zone
2672 		 * commands, but it is used by devstat_end_transaction_bio()
2673 		 * to determine how much data was transferred.  Because
2674 		 * the size of the SCSI/ATA data structures is different
2675 		 * than the size of the BIO interface structures, the
2676 		 * amount of data actually transferred from the drive will
2677 		 * be different than the amount of data transferred to
2678 		 * the user.
2679 		 */
2680 		num_alloced = rep->entries_allocated;
2681 		hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2682 		if (avail_len < sizeof(*hdr)) {
2683 			/*
2684 			 * Is there a better error than EIO here?  We asked
2685 			 * for at least the header, and we got less than
2686 			 * that.
2687 			 */
2688 			bp->bio_error = EIO;
2689 			bp->bio_flags |= BIO_ERROR;
2690 			bp->bio_resid = bp->bio_bcount;
2691 			break;
2692 		}
2693 
2694 		hdr_len = le32dec(hdr->length);
2695 		if (hdr_len > 0)
2696 			rep->entries_available = hdr_len / sizeof(*desc);
2697 		else
2698 			rep->entries_available = 0;
2699 		/*
2700 		 * NOTE: using the same values for the BIO version of the
2701 		 * same field as the SCSI/ATA values.  This means we could
2702 		 * get some additional values that aren't defined in bio.h
2703 		 * if more values of the same field are defined later.
2704 		 */
2705 		rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2706 		rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2707 		/*
2708 		 * If the drive reports no entries that match the query,
2709 		 * we're done.
2710 		 */
2711 		if (hdr_len == 0) {
2712 			rep->entries_filled = 0;
2713 			bp->bio_resid = bp->bio_bcount;
2714 			break;
2715 		}
2716 
2717 		num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2718 				hdr_len / sizeof(*desc));
2719 		/*
2720 		 * If the drive didn't return any data, then we're done.
2721 		 */
2722 		if (num_avail == 0) {
2723 			rep->entries_filled = 0;
2724 			bp->bio_resid = bp->bio_bcount;
2725 			break;
2726 		}
2727 
2728 		num_to_fill = min(num_avail, rep->entries_allocated);
2729 		/*
2730 		 * If the user didn't allocate any entries for us to fill,
2731 		 * we're done.
2732 		 */
2733 		if (num_to_fill == 0) {
2734 			rep->entries_filled = 0;
2735 			bp->bio_resid = bp->bio_bcount;
2736 			break;
2737 		}
2738 
2739 		for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2740 		     i < num_to_fill; i++, desc++, entry++) {
2741 			/*
2742 			 * NOTE: we're mapping the values here directly
2743 			 * from the SCSI/ATA bit definitions to the bio.h
2744 			 * definitions.  There is also a warning in
2745 			 * disk_zone.h, but the impact is that if
2746 			 * additional values are added in the SCSI/ATA
2747 			 * specs these will be visible to consumers of
2748 			 * this interface.
2749 			 */
2750 			entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2751 			entry->zone_condition =
2752 			    (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2753 			    SRZ_ZONE_COND_SHIFT;
2754 			entry->zone_flags |= desc->zone_flags &
2755 			    (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2756 			entry->zone_length = le64dec(desc->zone_length);
2757 			entry->zone_start_lba = le64dec(desc->zone_start_lba);
2758 			entry->write_pointer_lba =
2759 			    le64dec(desc->write_pointer_lba);
2760 		}
2761 		rep->entries_filled = num_to_fill;
2762 		/*
2763 		 * Note that this residual is accurate from the user's
2764 		 * standpoint, but the amount transferred isn't accurate
2765 		 * from the standpoint of what actually came back from the
2766 		 * drive.
2767 		 */
2768 		bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2769 		break;
2770 	}
2771 	case DISK_ZONE_GET_PARAMS:
2772 	default:
2773 		/*
2774 		 * In theory we should not get a GET_PARAMS bio, since it
2775 		 * should be handled without queueing the command to the
2776 		 * drive.
2777 		 */
2778 		panic("%s: Invalid zone command %d", __func__,
2779 		    bp->bio_zone.zone_cmd);
2780 		break;
2781 	}
2782 
2783 	if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2784 		free(ccb->ataio.data_ptr, M_ATADA);
2785 }
2786 
2787 
2788 static void
2789 adadone(struct cam_periph *periph, union ccb *done_ccb)
2790 {
2791 	struct ada_softc *softc;
2792 	struct ccb_ataio *ataio;
2793 	struct cam_path *path;
2794 	uint32_t priority;
2795 	int state;
2796 
2797 	softc = (struct ada_softc *)periph->softc;
2798 	ataio = &done_ccb->ataio;
2799 	path = done_ccb->ccb_h.path;
2800 	priority = done_ccb->ccb_h.pinfo.priority;
2801 
2802 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2803 
2804 	state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2805 	switch (state) {
2806 	case ADA_CCB_BUFFER_IO:
2807 	case ADA_CCB_TRIM:
2808 	{
2809 		struct bio *bp;
2810 		int error;
2811 
2812 		cam_periph_lock(periph);
2813 		bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2814 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2815 			error = adaerror(done_ccb, 0, 0);
2816 			if (error == ERESTART) {
2817 				/* A retry was scheduled, so just return. */
2818 				cam_periph_unlock(periph);
2819 				return;
2820 			}
2821 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2822 				cam_release_devq(path,
2823 						 /*relsim_flags*/0,
2824 						 /*reduction*/0,
2825 						 /*timeout*/0,
2826 						 /*getcount_only*/0);
2827 			/*
2828 			 * If we get an error on an NCQ DSM TRIM, fall back
2829 			 * to a non-NCQ DSM TRIM forever. Please note that if
2830 			 * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2831 			 * However, for this one trim, we treat it as advisory
2832 			 * and return success up the stack.
2833 			 */
2834 			if (state == ADA_CCB_TRIM &&
2835 			    error != 0 &&
2836 			    (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2837 				softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2838 				error = 0;
2839 				adasetdeletemethod(softc);
2840 			}
2841 		} else {
2842 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2843 				panic("REQ_CMP with QFRZN");
2844 
2845 			error = 0;
2846 		}
2847 		bp->bio_error = error;
2848 		if (error != 0) {
2849 			bp->bio_resid = bp->bio_bcount;
2850 			bp->bio_flags |= BIO_ERROR;
2851 		} else {
2852 			if (bp->bio_cmd == BIO_ZONE)
2853 				adazonedone(periph, done_ccb);
2854 			else if (state == ADA_CCB_TRIM)
2855 				bp->bio_resid = 0;
2856 			else
2857 				bp->bio_resid = ataio->resid;
2858 
2859 			if ((bp->bio_resid > 0)
2860 			 && (bp->bio_cmd != BIO_ZONE))
2861 				bp->bio_flags |= BIO_ERROR;
2862 		}
2863 		softc->outstanding_cmds--;
2864 		if (softc->outstanding_cmds == 0)
2865 			softc->flags |= ADA_FLAG_WAS_OTAG;
2866 
2867 		/*
2868 		 * We need to call cam_iosched before we call biodone so that we
2869 		 * don't measure any activity that happens in the completion
2870 		 * routine, which in the case of sendfile can be quite
2871 		 * extensive.
2872 		 */
2873 		cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2874 		xpt_release_ccb(done_ccb);
2875 		if (state == ADA_CCB_TRIM) {
2876 			TAILQ_HEAD(, bio) queue;
2877 			struct bio *bp1;
2878 
2879 			TAILQ_INIT(&queue);
2880 			TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2881 			/*
2882 			 * Normally, the xpt_release_ccb() above would make sure
2883 			 * that when we have more work to do, that work would
2884 			 * get kicked off. However, we specifically keep
2885 			 * trim_running set to 0 before the call above to allow
2886 			 * other I/O to progress when many BIO_DELETE requests
2887 			 * are pushed down. We set trim_running to 0 and call
2888 			 * daschedule again so that we don't stall if there are
2889 			 * no other I/Os pending apart from BIO_DELETEs.
2890 			 */
2891 			cam_iosched_trim_done(softc->cam_iosched);
2892 			adaschedule(periph);
2893 			cam_periph_unlock(periph);
2894 			while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2895 				TAILQ_REMOVE(&queue, bp1, bio_queue);
2896 				bp1->bio_error = error;
2897 				if (error != 0) {
2898 					bp1->bio_flags |= BIO_ERROR;
2899 					bp1->bio_resid = bp1->bio_bcount;
2900 				} else
2901 					bp1->bio_resid = 0;
2902 				biodone(bp1);
2903 			}
2904 		} else {
2905 			adaschedule(periph);
2906 			cam_periph_unlock(periph);
2907 			biodone(bp);
2908 		}
2909 		return;
2910 	}
2911 	case ADA_CCB_RAHEAD:
2912 	{
2913 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2914 			if (adaerror(done_ccb, 0, 0) == ERESTART) {
2915 				/* Drop freeze taken due to CAM_DEV_QFREEZE */
2916 				cam_release_devq(path, 0, 0, 0, FALSE);
2917 				return;
2918 			} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2919 				cam_release_devq(path,
2920 				    /*relsim_flags*/0,
2921 				    /*reduction*/0,
2922 				    /*timeout*/0,
2923 				    /*getcount_only*/0);
2924 			}
2925 		}
2926 
2927 		/*
2928 		 * Since our peripheral may be invalidated by an error
2929 		 * above or an external event, we must release our CCB
2930 		 * before releasing the reference on the peripheral.
2931 		 * The peripheral will only go away once the last reference
2932 		 * is removed, and we need it around for the CCB release
2933 		 * operation.
2934 		 */
2935 
2936 		xpt_release_ccb(done_ccb);
2937 		softc->state = ADA_STATE_WCACHE;
2938 		xpt_schedule(periph, priority);
2939 		/* Drop freeze taken due to CAM_DEV_QFREEZE */
2940 		cam_release_devq(path, 0, 0, 0, FALSE);
2941 		return;
2942 	}
2943 	case ADA_CCB_WCACHE:
2944 	{
2945 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2946 			if (adaerror(done_ccb, 0, 0) == ERESTART) {
2947 				/* Drop freeze taken due to CAM_DEV_QFREEZE */
2948 				cam_release_devq(path, 0, 0, 0, FALSE);
2949 				return;
2950 			} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2951 				cam_release_devq(path,
2952 				    /*relsim_flags*/0,
2953 				    /*reduction*/0,
2954 				    /*timeout*/0,
2955 				    /*getcount_only*/0);
2956 			}
2957 		}
2958 
2959 		/* Drop freeze taken due to CAM_DEV_QFREEZE */
2960 		cam_release_devq(path, 0, 0, 0, FALSE);
2961 
2962 		if ((softc->flags & ADA_FLAG_CAN_LOG)
2963 		 && (softc->zone_mode != ADA_ZONE_NONE)) {
2964 			xpt_release_ccb(done_ccb);
2965 			softc->state = ADA_STATE_LOGDIR;
2966 			xpt_schedule(periph, priority);
2967 		} else {
2968 			adaprobedone(periph, done_ccb);
2969 		}
2970 		return;
2971 	}
2972 	case ADA_CCB_LOGDIR:
2973 	{
2974 		int error;
2975 
2976 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2977 			error = 0;
2978 			softc->valid_logdir_len = 0;
2979 			bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2980 			softc->valid_logdir_len =
2981 				ataio->dxfer_len - ataio->resid;
2982 			if (softc->valid_logdir_len > 0)
2983 				bcopy(ataio->data_ptr, &softc->ata_logdir,
2984 				    min(softc->valid_logdir_len,
2985 					sizeof(softc->ata_logdir)));
2986 			/*
2987 			 * Figure out whether the Identify Device log is
2988 			 * supported.  The General Purpose log directory
2989 			 * has a header, and lists the number of pages
2990 			 * available for each GP log identified by the
2991 			 * offset into the list.
2992 			 */
2993 			if ((softc->valid_logdir_len >=
2994 			    ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
2995 			 && (le16dec(softc->ata_logdir.header) ==
2996 			     ATA_GP_LOG_DIR_VERSION)
2997 			 && (le16dec(&softc->ata_logdir.num_pages[
2998 			     (ATA_IDENTIFY_DATA_LOG *
2999 			     sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
3000 				softc->flags |= ADA_FLAG_CAN_IDLOG;
3001 			} else {
3002 				softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3003 			}
3004 		} else {
3005 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3006 					 SF_RETRY_UA|SF_NO_PRINT);
3007 			if (error == ERESTART)
3008 				return;
3009 			else if (error != 0) {
3010 				/*
3011 				 * If we can't get the ATA log directory,
3012 				 * then ATA logs are effectively not
3013 				 * supported even if the bit is set in the
3014 				 * identify data.
3015 				 */
3016 				softc->flags &= ~(ADA_FLAG_CAN_LOG |
3017 						  ADA_FLAG_CAN_IDLOG);
3018 				if ((done_ccb->ccb_h.status &
3019 				     CAM_DEV_QFRZN) != 0) {
3020 					/* Don't wedge this device's queue */
3021 					cam_release_devq(done_ccb->ccb_h.path,
3022 							 /*relsim_flags*/0,
3023 							 /*reduction*/0,
3024 							 /*timeout*/0,
3025 							 /*getcount_only*/0);
3026 				}
3027 			}
3028 
3029 
3030 		}
3031 
3032 		free(ataio->data_ptr, M_ATADA);
3033 
3034 		if ((error == 0)
3035 		 && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3036 			softc->state = ADA_STATE_IDDIR;
3037 			xpt_release_ccb(done_ccb);
3038 			xpt_schedule(periph, priority);
3039 		} else
3040 			adaprobedone(periph, done_ccb);
3041 
3042 		return;
3043 	}
3044 	case ADA_CCB_IDDIR: {
3045 		int error;
3046 
3047 		if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3048 			off_t entries_offset, max_entries;
3049 			error = 0;
3050 
3051 			softc->valid_iddir_len = 0;
3052 			bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3053 			softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3054 					  ADA_FLAG_CAN_ZONE);
3055 			softc->valid_iddir_len =
3056 				ataio->dxfer_len - ataio->resid;
3057 			if (softc->valid_iddir_len > 0)
3058 				bcopy(ataio->data_ptr, &softc->ata_iddir,
3059 				    min(softc->valid_iddir_len,
3060 					sizeof(softc->ata_iddir)));
3061 
3062 			entries_offset =
3063 			    __offsetof(struct ata_identify_log_pages,entries);
3064 			max_entries = softc->valid_iddir_len - entries_offset;
3065 			if ((softc->valid_iddir_len > (entries_offset + 1))
3066 			 && (le64dec(softc->ata_iddir.header) ==
3067 			     ATA_IDLOG_REVISION)
3068 			 && (softc->ata_iddir.entry_count > 0)) {
3069 				int num_entries, i;
3070 
3071 				num_entries = softc->ata_iddir.entry_count;
3072 				num_entries = min(num_entries,
3073 				   softc->valid_iddir_len - entries_offset);
3074 				for (i = 0; i < num_entries &&
3075 				     i < max_entries; i++) {
3076 					if (softc->ata_iddir.entries[i] ==
3077 					    ATA_IDL_SUP_CAP)
3078 						softc->flags |=
3079 						    ADA_FLAG_CAN_SUPCAP;
3080 					else if (softc->ata_iddir.entries[i]==
3081 						 ATA_IDL_ZDI)
3082 						softc->flags |=
3083 						    ADA_FLAG_CAN_ZONE;
3084 
3085 					if ((softc->flags &
3086 					     ADA_FLAG_CAN_SUPCAP)
3087 					 && (softc->flags &
3088 					     ADA_FLAG_CAN_ZONE))
3089 						break;
3090 				}
3091 			}
3092 		} else {
3093 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3094 					 SF_RETRY_UA|SF_NO_PRINT);
3095 			if (error == ERESTART)
3096 				return;
3097 			else if (error != 0) {
3098 				/*
3099 				 * If we can't get the ATA Identify Data log
3100 				 * directory, then it effectively isn't
3101 				 * supported even if the ATA Log directory
3102 				 * a non-zero number of pages present for
3103 				 * this log.
3104 				 */
3105 				softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3106 				if ((done_ccb->ccb_h.status &
3107 				     CAM_DEV_QFRZN) != 0) {
3108 					/* Don't wedge this device's queue */
3109 					cam_release_devq(done_ccb->ccb_h.path,
3110 							 /*relsim_flags*/0,
3111 							 /*reduction*/0,
3112 							 /*timeout*/0,
3113 							 /*getcount_only*/0);
3114 				}
3115 			}
3116 		}
3117 
3118 		free(ataio->data_ptr, M_ATADA);
3119 
3120 		if ((error == 0)
3121 		 && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3122 			softc->state = ADA_STATE_SUP_CAP;
3123 			xpt_release_ccb(done_ccb);
3124 			xpt_schedule(periph, priority);
3125 		} else
3126 			adaprobedone(periph, done_ccb);
3127 		return;
3128 	}
3129 	case ADA_CCB_SUP_CAP: {
3130 		int error;
3131 
3132 		if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3133 			uint32_t valid_len;
3134 			size_t needed_size;
3135 			struct ata_identify_log_sup_cap *sup_cap;
3136 			error = 0;
3137 
3138 			sup_cap = (struct ata_identify_log_sup_cap *)
3139 			    ataio->data_ptr;
3140 			valid_len = ataio->dxfer_len - ataio->resid;
3141 			needed_size =
3142 			    __offsetof(struct ata_identify_log_sup_cap,
3143 			    sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3144 			if (valid_len >= needed_size) {
3145 				uint64_t zoned, zac_cap;
3146 
3147 				zoned = le64dec(sup_cap->zoned_cap);
3148 				if (zoned & ATA_ZONED_VALID) {
3149 					/*
3150 					 * This should have already been
3151 					 * set, because this is also in the
3152 					 * ATA identify data.
3153 					 */
3154 					if ((zoned & ATA_ZONED_MASK) ==
3155 					    ATA_SUPPORT_ZONE_HOST_AWARE)
3156 						softc->zone_mode =
3157 						    ADA_ZONE_HOST_AWARE;
3158 					else if ((zoned & ATA_ZONED_MASK) ==
3159 					    ATA_SUPPORT_ZONE_DEV_MANAGED)
3160 						softc->zone_mode =
3161 						    ADA_ZONE_DRIVE_MANAGED;
3162 				}
3163 
3164 				zac_cap = le64dec(sup_cap->sup_zac_cap);
3165 				if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3166 					if (zac_cap & ATA_REPORT_ZONES_SUP)
3167 						softc->zone_flags |=
3168 						    ADA_ZONE_FLAG_RZ_SUP;
3169 					if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3170 						softc->zone_flags |=
3171 						    ADA_ZONE_FLAG_OPEN_SUP;
3172 					if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3173 						softc->zone_flags |=
3174 						    ADA_ZONE_FLAG_CLOSE_SUP;
3175 					if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3176 						softc->zone_flags |=
3177 						    ADA_ZONE_FLAG_FINISH_SUP;
3178 					if (zac_cap & ATA_ND_RWP_SUP)
3179 						softc->zone_flags |=
3180 						    ADA_ZONE_FLAG_RWP_SUP;
3181 				} else {
3182 					/*
3183 					 * This field was introduced in
3184 					 * ACS-4, r08 on April 28th, 2015.
3185 					 * If the drive firmware was written
3186 					 * to an earlier spec, it won't have
3187 					 * the field.  So, assume all
3188 					 * commands are supported.
3189 					 */
3190 					softc->zone_flags |=
3191 					    ADA_ZONE_FLAG_SUP_MASK;
3192 				}
3193 
3194 			}
3195 		} else {
3196 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3197 					 SF_RETRY_UA|SF_NO_PRINT);
3198 			if (error == ERESTART)
3199 				return;
3200 			else if (error != 0) {
3201 				/*
3202 				 * If we can't get the ATA Identify Data
3203 				 * Supported Capabilities page, clear the
3204 				 * flag...
3205 				 */
3206 				softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3207 				/*
3208 				 * And clear zone capabilities.
3209 				 */
3210 				softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3211 				if ((done_ccb->ccb_h.status &
3212 				     CAM_DEV_QFRZN) != 0) {
3213 					/* Don't wedge this device's queue */
3214 					cam_release_devq(done_ccb->ccb_h.path,
3215 							 /*relsim_flags*/0,
3216 							 /*reduction*/0,
3217 							 /*timeout*/0,
3218 							 /*getcount_only*/0);
3219 				}
3220 			}
3221 		}
3222 
3223 		free(ataio->data_ptr, M_ATADA);
3224 
3225 		if ((error == 0)
3226 		 && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3227 			softc->state = ADA_STATE_ZONE;
3228 			xpt_release_ccb(done_ccb);
3229 			xpt_schedule(periph, priority);
3230 		} else
3231 			adaprobedone(periph, done_ccb);
3232 		return;
3233 	}
3234 	case ADA_CCB_ZONE: {
3235 		int error;
3236 
3237 		if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3238 			struct ata_zoned_info_log *zi_log;
3239 			uint32_t valid_len;
3240 			size_t needed_size;
3241 
3242 			zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3243 
3244 			valid_len = ataio->dxfer_len - ataio->resid;
3245 			needed_size = __offsetof(struct ata_zoned_info_log,
3246 			    version_info) + 1 + sizeof(zi_log->version_info);
3247 			if (valid_len >= needed_size) {
3248 				uint64_t tmpvar;
3249 
3250 				tmpvar = le64dec(zi_log->zoned_cap);
3251 				if (tmpvar & ATA_ZDI_CAP_VALID) {
3252 					if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3253 						softc->zone_flags |=
3254 						    ADA_ZONE_FLAG_URSWRZ;
3255 					else
3256 						softc->zone_flags &=
3257 						    ~ADA_ZONE_FLAG_URSWRZ;
3258 				}
3259 				tmpvar = le64dec(zi_log->optimal_seq_zones);
3260 				if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3261 					softc->zone_flags |=
3262 					    ADA_ZONE_FLAG_OPT_SEQ_SET;
3263 					softc->optimal_seq_zones = (tmpvar &
3264 					    ATA_ZDI_OPT_SEQ_MASK);
3265 				} else {
3266 					softc->zone_flags &=
3267 					    ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3268 					softc->optimal_seq_zones = 0;
3269 				}
3270 
3271 				tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3272 				if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3273 					softc->zone_flags |=
3274 					    ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3275 					softc->optimal_nonseq_zones =
3276 					    (tmpvar & ATA_ZDI_OPT_NS_MASK);
3277 				} else {
3278 					softc->zone_flags &=
3279 					    ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3280 					softc->optimal_nonseq_zones = 0;
3281 				}
3282 
3283 				tmpvar = le64dec(zi_log->max_seq_req_zones);
3284 				if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3285 					softc->zone_flags |=
3286 					    ADA_ZONE_FLAG_MAX_SEQ_SET;
3287 					softc->max_seq_zones =
3288 					    (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3289 				} else {
3290 					softc->zone_flags &=
3291 					    ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3292 					softc->max_seq_zones = 0;
3293 				}
3294 			}
3295 		} else {
3296 			error = adaerror(done_ccb, CAM_RETRY_SELTO,
3297 					 SF_RETRY_UA|SF_NO_PRINT);
3298 			if (error == ERESTART)
3299 				return;
3300 			else if (error != 0) {
3301 				softc->flags &= ~ADA_FLAG_CAN_ZONE;
3302 				softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3303 
3304 				if ((done_ccb->ccb_h.status &
3305 				     CAM_DEV_QFRZN) != 0) {
3306 					/* Don't wedge this device's queue */
3307 					cam_release_devq(done_ccb->ccb_h.path,
3308 							 /*relsim_flags*/0,
3309 							 /*reduction*/0,
3310 							 /*timeout*/0,
3311 							 /*getcount_only*/0);
3312 				}
3313 			}
3314 
3315 		}
3316 		free(ataio->data_ptr, M_ATADA);
3317 
3318 		adaprobedone(periph, done_ccb);
3319 		return;
3320 	}
3321 	case ADA_CCB_DUMP:
3322 		/* No-op.  We're polling */
3323 		return;
3324 	default:
3325 		break;
3326 	}
3327 	xpt_release_ccb(done_ccb);
3328 }
3329 
3330 static int
3331 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3332 {
3333 #ifdef CAM_IO_STATS
3334 	struct ada_softc *softc;
3335 	struct cam_periph *periph;
3336 
3337 	periph = xpt_path_periph(ccb->ccb_h.path);
3338 	softc = (struct ada_softc *)periph->softc;
3339 
3340 	switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3341 	case CAM_CMD_TIMEOUT:
3342 		softc->timeouts++;
3343 		break;
3344 	case CAM_REQ_ABORTED:
3345 	case CAM_REQ_CMP_ERR:
3346 	case CAM_REQ_TERMIO:
3347 	case CAM_UNREC_HBA_ERROR:
3348 	case CAM_DATA_RUN_ERR:
3349 	case CAM_ATA_STATUS_ERROR:
3350 		softc->errors++;
3351 		break;
3352 	default:
3353 		break;
3354 	}
3355 #endif
3356 
3357 	return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
3358 }
3359 
3360 static void
3361 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3362 {
3363 	struct ada_softc *softc = (struct ada_softc *)periph->softc;
3364 	struct disk_params *dp = &softc->params;
3365 	u_int64_t lbasize48;
3366 	u_int32_t lbasize;
3367 
3368 	dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3369 	if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3370 		cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3371 		dp->heads = cgd->ident_data.current_heads;
3372 		dp->secs_per_track = cgd->ident_data.current_sectors;
3373 		dp->cylinders = cgd->ident_data.cylinders;
3374 		dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3375 			  ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3376 	} else {
3377 		dp->heads = cgd->ident_data.heads;
3378 		dp->secs_per_track = cgd->ident_data.sectors;
3379 		dp->cylinders = cgd->ident_data.cylinders;
3380 		dp->sectors = cgd->ident_data.cylinders * dp->heads * dp->secs_per_track;
3381 	}
3382 	lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3383 		  ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3384 
3385 	/* use the 28bit LBA size if valid or bigger than the CHS mapping */
3386 	if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3387 		dp->sectors = lbasize;
3388 
3389 	/* use the 48bit LBA size if valid */
3390 	lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3391 		    ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3392 		    ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3393 		    ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3394 	if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3395 	    lbasize48 > ATA_MAX_28BIT_LBA)
3396 		dp->sectors = lbasize48;
3397 }
3398 
3399 static void
3400 adasendorderedtag(void *arg)
3401 {
3402 	struct ada_softc *softc = arg;
3403 
3404 	if (ada_send_ordered) {
3405 		if (softc->outstanding_cmds > 0) {
3406 			if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3407 				softc->flags |= ADA_FLAG_NEED_OTAG;
3408 			softc->flags &= ~ADA_FLAG_WAS_OTAG;
3409 		}
3410 	}
3411 	/* Queue us up again */
3412 	callout_reset(&softc->sendordered_c,
3413 	    (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3414 	    adasendorderedtag, softc);
3415 }
3416 
3417 /*
3418  * Step through all ADA peripheral drivers, and if the device is still open,
3419  * sync the disk cache to physical media.
3420  */
3421 static void
3422 adaflush(void)
3423 {
3424 	struct cam_periph *periph;
3425 	struct ada_softc *softc;
3426 	union ccb *ccb;
3427 	int error;
3428 
3429 	CAM_PERIPH_FOREACH(periph, &adadriver) {
3430 		softc = (struct ada_softc *)periph->softc;
3431 		if (SCHEDULER_STOPPED()) {
3432 			/* If we paniced with the lock held, do not recurse. */
3433 			if (!cam_periph_owned(periph) &&
3434 			    (softc->flags & ADA_FLAG_OPEN)) {
3435 				adadump(softc->disk, NULL, 0, 0, 0);
3436 			}
3437 			continue;
3438 		}
3439 		cam_periph_lock(periph);
3440 		/*
3441 		 * We only sync the cache if the drive is still open, and
3442 		 * if the drive is capable of it..
3443 		 */
3444 		if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3445 		    (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3446 			cam_periph_unlock(periph);
3447 			continue;
3448 		}
3449 
3450 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3451 		cam_fill_ataio(&ccb->ataio,
3452 				    0,
3453 				    adadone,
3454 				    CAM_DIR_NONE,
3455 				    0,
3456 				    NULL,
3457 				    0,
3458 				    ada_default_timeout*1000);
3459 		if (softc->flags & ADA_FLAG_CAN_48BIT)
3460 			ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3461 		else
3462 			ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3463 
3464 		error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3465 		    /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3466 		    softc->disk->d_devstat);
3467 		if (error != 0)
3468 			xpt_print(periph->path, "Synchronize cache failed\n");
3469 		xpt_release_ccb(ccb);
3470 		cam_periph_unlock(periph);
3471 	}
3472 }
3473 
3474 static void
3475 adaspindown(uint8_t cmd, int flags)
3476 {
3477 	struct cam_periph *periph;
3478 	struct ada_softc *softc;
3479 	struct ccb_ataio local_ccb;
3480 	int error;
3481 
3482 	CAM_PERIPH_FOREACH(periph, &adadriver) {
3483 		/* If we paniced with lock held - not recurse here. */
3484 		if (cam_periph_owned(periph))
3485 			continue;
3486 		cam_periph_lock(periph);
3487 		softc = (struct ada_softc *)periph->softc;
3488 		/*
3489 		 * We only spin-down the drive if it is capable of it..
3490 		 */
3491 		if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3492 			cam_periph_unlock(periph);
3493 			continue;
3494 		}
3495 
3496 		if (bootverbose)
3497 			xpt_print(periph->path, "spin-down\n");
3498 
3499 		memset(&local_ccb, 0, sizeof(local_ccb));
3500 		xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3501 		local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3502 
3503 		cam_fill_ataio(&local_ccb,
3504 				    0,
3505 				    adadone,
3506 				    CAM_DIR_NONE | flags,
3507 				    0,
3508 				    NULL,
3509 				    0,
3510 				    ada_default_timeout*1000);
3511 		ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3512 
3513 		if (!SCHEDULER_STOPPED()) {
3514 			/*
3515 			 * Not panicing, can just do the normal runccb
3516 			 * XXX should make cam_periph_runccb work while
3517 			 * XXX panicing... later
3518 			 */
3519 			error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3520 			    /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3521 			    softc->disk->d_devstat);
3522 		} else {
3523 			/*
3524 			 * Panicing, so we have to do this by hand: do
3525 			 * xpt_polled_action to run the request through the SIM,
3526 			 * extract the error, and if the queue was frozen,
3527 			 * unfreeze it. cam_periph_runccb takes care of these
3528 			 * details, but xpt_polled_action doesn't.
3529 			 */
3530 			xpt_polled_action((union ccb *)&local_ccb);
3531 			error = adaerror((union ccb *)&local_ccb, 0,
3532 			    SF_NO_RECOVERY | SF_NO_RETRY);
3533 			if ((local_ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
3534 				cam_release_devq(local_ccb.ccb_h.path,
3535 				    /*relsim_flags*/0, /*reduction*/0,
3536 				    /*timeout*/0, /*getcount_only*/0);
3537 		}
3538 		if (error != 0)
3539 			xpt_print(periph->path, "Spin-down disk failed\n");
3540 		cam_periph_unlock(periph);
3541 	}
3542 }
3543 
3544 static void
3545 adashutdown(void *arg, int howto)
3546 {
3547 	int how;
3548 
3549 	adaflush();
3550 
3551 	/*
3552 	 * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3553 	 * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3554 	 * a spindown. We send the former when we expect to lose power soon. For
3555 	 * a warm boot, we send the latter to avoid a thundering herd of spinups
3556 	 * just after the kernel loads while probing. We have to do something to
3557 	 * flush the data because the BIOS in many systems resets the HBA
3558 	 * causing a COMINIT/COMRESET negotiation, which some drives interpret
3559 	 * as license to toss the volatile data, and others count as unclean
3560 	 * shutdown when in the Active PM state in SMART attributes.
3561 	 *
3562 	 * adaspindown will ensure that we don't send this to a drive that
3563 	 * doesn't support it.
3564 	 */
3565 	if (ada_spindown_shutdown != 0) {
3566 		how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3567 		    ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3568 		adaspindown(how, 0);
3569 	}
3570 }
3571 
3572 static void
3573 adasuspend(void *arg)
3574 {
3575 
3576 	adaflush();
3577 	/*
3578 	 * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3579 	 * so we don't need to send it as well.
3580 	 */
3581 	if (ada_spindown_suspend != 0)
3582 		adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3583 }
3584 
3585 static void
3586 adaresume(void *arg)
3587 {
3588 	struct cam_periph *periph;
3589 	struct ada_softc *softc;
3590 
3591 	if (ada_spindown_suspend == 0)
3592 		return;
3593 
3594 	CAM_PERIPH_FOREACH(periph, &adadriver) {
3595 		cam_periph_lock(periph);
3596 		softc = (struct ada_softc *)periph->softc;
3597 		/*
3598 		 * We only spin-down the drive if it is capable of it..
3599 		 */
3600 		if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3601 			cam_periph_unlock(periph);
3602 			continue;
3603 		}
3604 
3605 		if (bootverbose)
3606 			xpt_print(periph->path, "resume\n");
3607 
3608 		/*
3609 		 * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3610 		 * sleep request.
3611 		 */
3612 		cam_release_devq(periph->path,
3613 			 /*relsim_flags*/0,
3614 			 /*openings*/0,
3615 			 /*timeout*/0,
3616 			 /*getcount_only*/0);
3617 
3618 		cam_periph_unlock(periph);
3619 	}
3620 }
3621 
3622 #endif /* _KERNEL */
3623