xref: /freebsd/sys/cam/scsi/scsi_da.c (revision 4f52dfbb8d6c4d446500c5b097e3806ec219fbd4)
1 /*-
2  * Implementation of SCSI Direct Access Peripheral driver for CAM.
3  *
4  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
5  *
6  * Copyright (c) 1997 Justin T. Gibbs.
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions, and the following disclaimer,
14  *    without modification, immediately at the beginning of the file.
15  * 2. The name of the author may not be used to endorse or promote products
16  *    derived from this software without specific prior written permission.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
22  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 
36 #ifdef _KERNEL
37 #include "opt_da.h"
38 #include <sys/systm.h>
39 #include <sys/kernel.h>
40 #include <sys/bio.h>
41 #include <sys/sysctl.h>
42 #include <sys/taskqueue.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/conf.h>
46 #include <sys/devicestat.h>
47 #include <sys/eventhandler.h>
48 #include <sys/malloc.h>
49 #include <sys/cons.h>
50 #include <sys/endian.h>
51 #include <sys/proc.h>
52 #include <sys/sbuf.h>
53 #include <geom/geom.h>
54 #include <geom/geom_disk.h>
55 #include <machine/atomic.h>
56 #endif /* _KERNEL */
57 
58 #ifndef _KERNEL
59 #include <stdio.h>
60 #include <string.h>
61 #endif /* _KERNEL */
62 
63 #include <cam/cam.h>
64 #include <cam/cam_ccb.h>
65 #include <cam/cam_periph.h>
66 #include <cam/cam_xpt_periph.h>
67 #include <cam/cam_sim.h>
68 #include <cam/cam_iosched.h>
69 
70 #include <cam/scsi/scsi_message.h>
71 #include <cam/scsi/scsi_da.h>
72 
73 #ifdef _KERNEL
74 /*
75  * Note that there are probe ordering dependencies here.  The order isn't
76  * controlled by this enumeration, but by explicit state transitions in
77  * dastart() and dadone().  Here are some of the dependencies:
78  *
79  * 1. RC should come first, before RC16, unless there is evidence that RC16
80  *    is supported.
81  * 2. BDC needs to come before any of the ATA probes, or the ZONE probe.
82  * 3. The ATA probes should go in this order:
83  *    ATA -> LOGDIR -> IDDIR -> SUP -> ATA_ZONE
84  */
85 typedef enum {
86 	DA_STATE_PROBE_WP,
87 	DA_STATE_PROBE_RC,
88 	DA_STATE_PROBE_RC16,
89 	DA_STATE_PROBE_LBP,
90 	DA_STATE_PROBE_BLK_LIMITS,
91 	DA_STATE_PROBE_BDC,
92 	DA_STATE_PROBE_ATA,
93 	DA_STATE_PROBE_ATA_LOGDIR,
94 	DA_STATE_PROBE_ATA_IDDIR,
95 	DA_STATE_PROBE_ATA_SUP,
96 	DA_STATE_PROBE_ATA_ZONE,
97 	DA_STATE_PROBE_ZONE,
98 	DA_STATE_NORMAL
99 } da_state;
100 
101 typedef enum {
102 	DA_FLAG_PACK_INVALID	= 0x000001,
103 	DA_FLAG_NEW_PACK	= 0x000002,
104 	DA_FLAG_PACK_LOCKED	= 0x000004,
105 	DA_FLAG_PACK_REMOVABLE	= 0x000008,
106 	DA_FLAG_NEED_OTAG	= 0x000020,
107 	DA_FLAG_WAS_OTAG	= 0x000040,
108 	DA_FLAG_RETRY_UA	= 0x000080,
109 	DA_FLAG_OPEN		= 0x000100,
110 	DA_FLAG_SCTX_INIT	= 0x000200,
111 	DA_FLAG_CAN_RC16	= 0x000400,
112 	DA_FLAG_PROBED		= 0x000800,
113 	DA_FLAG_DIRTY		= 0x001000,
114 	DA_FLAG_ANNOUNCED	= 0x002000,
115 	DA_FLAG_CAN_ATA_DMA	= 0x004000,
116 	DA_FLAG_CAN_ATA_LOG	= 0x008000,
117 	DA_FLAG_CAN_ATA_IDLOG	= 0x010000,
118 	DA_FLAG_CAN_ATA_SUPCAP	= 0x020000,
119 	DA_FLAG_CAN_ATA_ZONE	= 0x040000,
120 	DA_FLAG_TUR_PENDING	= 0x080000
121 } da_flags;
122 
123 typedef enum {
124 	DA_Q_NONE		= 0x00,
125 	DA_Q_NO_SYNC_CACHE	= 0x01,
126 	DA_Q_NO_6_BYTE		= 0x02,
127 	DA_Q_NO_PREVENT		= 0x04,
128 	DA_Q_4K			= 0x08,
129 	DA_Q_NO_RC16		= 0x10,
130 	DA_Q_NO_UNMAP		= 0x20,
131 	DA_Q_RETRY_BUSY		= 0x40,
132 	DA_Q_SMR_DM		= 0x80,
133 	DA_Q_STRICT_UNMAP	= 0x100
134 } da_quirks;
135 
136 #define DA_Q_BIT_STRING		\
137 	"\020"			\
138 	"\001NO_SYNC_CACHE"	\
139 	"\002NO_6_BYTE"		\
140 	"\003NO_PREVENT"	\
141 	"\0044K"		\
142 	"\005NO_RC16"		\
143 	"\006NO_UNMAP"		\
144 	"\007RETRY_BUSY"	\
145 	"\010SMR_DM"		\
146 	"\011STRICT_UNMAP"
147 
148 typedef enum {
149 	DA_CCB_PROBE_RC		= 0x01,
150 	DA_CCB_PROBE_RC16	= 0x02,
151 	DA_CCB_PROBE_LBP	= 0x03,
152 	DA_CCB_PROBE_BLK_LIMITS	= 0x04,
153 	DA_CCB_PROBE_BDC	= 0x05,
154 	DA_CCB_PROBE_ATA	= 0x06,
155 	DA_CCB_BUFFER_IO	= 0x07,
156 	DA_CCB_DUMP		= 0x0A,
157 	DA_CCB_DELETE		= 0x0B,
158 	DA_CCB_TUR		= 0x0C,
159 	DA_CCB_PROBE_ZONE	= 0x0D,
160 	DA_CCB_PROBE_ATA_LOGDIR	= 0x0E,
161 	DA_CCB_PROBE_ATA_IDDIR	= 0x0F,
162 	DA_CCB_PROBE_ATA_SUP	= 0x10,
163 	DA_CCB_PROBE_ATA_ZONE	= 0x11,
164 	DA_CCB_PROBE_WP		= 0x12,
165 	DA_CCB_TYPE_MASK	= 0x1F,
166 	DA_CCB_RETRY_UA		= 0x20
167 } da_ccb_state;
168 
169 /*
170  * Order here is important for method choice
171  *
172  * We prefer ATA_TRIM as tests run against a Sandforce 2281 SSD attached to
173  * LSI 2008 (mps) controller (FW: v12, Drv: v14) resulted 20% quicker deletes
174  * using ATA_TRIM than the corresponding UNMAP results for a real world mysql
175  * import taking 5mins.
176  *
177  */
178 typedef enum {
179 	DA_DELETE_NONE,
180 	DA_DELETE_DISABLE,
181 	DA_DELETE_ATA_TRIM,
182 	DA_DELETE_UNMAP,
183 	DA_DELETE_WS16,
184 	DA_DELETE_WS10,
185 	DA_DELETE_ZERO,
186 	DA_DELETE_MIN = DA_DELETE_ATA_TRIM,
187 	DA_DELETE_MAX = DA_DELETE_ZERO
188 } da_delete_methods;
189 
190 /*
191  * For SCSI, host managed drives show up as a separate device type.  For
192  * ATA, host managed drives also have a different device signature.
193  * XXX KDM figure out the ATA host managed signature.
194  */
195 typedef enum {
196 	DA_ZONE_NONE		= 0x00,
197 	DA_ZONE_DRIVE_MANAGED	= 0x01,
198 	DA_ZONE_HOST_AWARE	= 0x02,
199 	DA_ZONE_HOST_MANAGED	= 0x03
200 } da_zone_mode;
201 
202 /*
203  * We distinguish between these interface cases in addition to the drive type:
204  * o ATA drive behind a SCSI translation layer that knows about ZBC/ZAC
205  * o ATA drive behind a SCSI translation layer that does not know about
206  *   ZBC/ZAC, and so needs to be managed via ATA passthrough.  In this
207  *   case, we would need to share the ATA code with the ada(4) driver.
208  * o SCSI drive.
209  */
210 typedef enum {
211 	DA_ZONE_IF_SCSI,
212 	DA_ZONE_IF_ATA_PASS,
213 	DA_ZONE_IF_ATA_SAT,
214 } da_zone_interface;
215 
216 typedef enum {
217 	DA_ZONE_FLAG_RZ_SUP		= 0x0001,
218 	DA_ZONE_FLAG_OPEN_SUP		= 0x0002,
219 	DA_ZONE_FLAG_CLOSE_SUP		= 0x0004,
220 	DA_ZONE_FLAG_FINISH_SUP		= 0x0008,
221 	DA_ZONE_FLAG_RWP_SUP		= 0x0010,
222 	DA_ZONE_FLAG_SUP_MASK		= (DA_ZONE_FLAG_RZ_SUP |
223 					   DA_ZONE_FLAG_OPEN_SUP |
224 					   DA_ZONE_FLAG_CLOSE_SUP |
225 					   DA_ZONE_FLAG_FINISH_SUP |
226 					   DA_ZONE_FLAG_RWP_SUP),
227 	DA_ZONE_FLAG_URSWRZ		= 0x0020,
228 	DA_ZONE_FLAG_OPT_SEQ_SET	= 0x0040,
229 	DA_ZONE_FLAG_OPT_NONSEQ_SET	= 0x0080,
230 	DA_ZONE_FLAG_MAX_SEQ_SET	= 0x0100,
231 	DA_ZONE_FLAG_SET_MASK		= (DA_ZONE_FLAG_OPT_SEQ_SET |
232 					   DA_ZONE_FLAG_OPT_NONSEQ_SET |
233 					   DA_ZONE_FLAG_MAX_SEQ_SET)
234 } da_zone_flags;
235 
236 static struct da_zone_desc {
237 	da_zone_flags value;
238 	const char *desc;
239 } da_zone_desc_table[] = {
240 	{DA_ZONE_FLAG_RZ_SUP, "Report Zones" },
241 	{DA_ZONE_FLAG_OPEN_SUP, "Open" },
242 	{DA_ZONE_FLAG_CLOSE_SUP, "Close" },
243 	{DA_ZONE_FLAG_FINISH_SUP, "Finish" },
244 	{DA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
245 };
246 
247 typedef void da_delete_func_t (struct cam_periph *periph, union ccb *ccb,
248 			      struct bio *bp);
249 static da_delete_func_t da_delete_trim;
250 static da_delete_func_t da_delete_unmap;
251 static da_delete_func_t da_delete_ws;
252 
253 static const void * da_delete_functions[] = {
254 	NULL,
255 	NULL,
256 	da_delete_trim,
257 	da_delete_unmap,
258 	da_delete_ws,
259 	da_delete_ws,
260 	da_delete_ws
261 };
262 
263 static const char *da_delete_method_names[] =
264     { "NONE", "DISABLE", "ATA_TRIM", "UNMAP", "WS16", "WS10", "ZERO" };
265 static const char *da_delete_method_desc[] =
266     { "NONE", "DISABLED", "ATA TRIM", "UNMAP", "WRITE SAME(16) with UNMAP",
267       "WRITE SAME(10) with UNMAP", "ZERO" };
268 
269 /* Offsets into our private area for storing information */
270 #define ccb_state	ppriv_field0
271 #define ccb_bp		ppriv_ptr1
272 
273 struct disk_params {
274 	u_int8_t  heads;
275 	u_int32_t cylinders;
276 	u_int8_t  secs_per_track;
277 	u_int32_t secsize;	/* Number of bytes/sector */
278 	u_int64_t sectors;	/* total number sectors */
279 	u_int     stripesize;
280 	u_int     stripeoffset;
281 };
282 
283 #define UNMAP_RANGE_MAX		0xffffffff
284 #define UNMAP_HEAD_SIZE		8
285 #define UNMAP_RANGE_SIZE	16
286 #define UNMAP_MAX_RANGES	2048 /* Protocol Max is 4095 */
287 #define UNMAP_BUF_SIZE		((UNMAP_MAX_RANGES * UNMAP_RANGE_SIZE) + \
288 				UNMAP_HEAD_SIZE)
289 
290 #define WS10_MAX_BLKS		0xffff
291 #define WS16_MAX_BLKS		0xffffffff
292 #define ATA_TRIM_MAX_RANGES	((UNMAP_BUF_SIZE / \
293 	(ATA_DSM_RANGE_SIZE * ATA_DSM_BLK_SIZE)) * ATA_DSM_BLK_SIZE)
294 
295 #define DA_WORK_TUR		(1 << 16)
296 
297 typedef enum {
298 	DA_REF_OPEN = 1,
299 	DA_REF_OPEN_HOLD,
300 	DA_REF_CLOSE_HOLD,
301 	DA_REF_PROBE_HOLD,
302 	DA_REF_TUR,
303 	DA_REF_GEOM,
304 	DA_REF_SYSCTL,
305 	DA_REF_REPROBE,
306 	DA_REF_MAX		/* KEEP LAST */
307 } da_ref_token;
308 
309 struct da_softc {
310 	struct   cam_iosched_softc *cam_iosched;
311 	struct	 bio_queue_head delete_run_queue;
312 	LIST_HEAD(, ccb_hdr) pending_ccbs;
313 	int	 refcount;		/* Active xpt_action() calls */
314 	da_state state;
315 	da_flags flags;
316 	da_quirks quirks;
317 	int	 minimum_cmd_size;
318 	int	 error_inject;
319 	int	 trim_max_ranges;
320 	int	 delete_available;	/* Delete methods possibly available */
321 	da_zone_mode			zone_mode;
322 	da_zone_interface		zone_interface;
323 	da_zone_flags			zone_flags;
324 	struct ata_gp_log_dir		ata_logdir;
325 	int				valid_logdir_len;
326 	struct ata_identify_log_pages	ata_iddir;
327 	int				valid_iddir_len;
328 	uint64_t			optimal_seq_zones;
329 	uint64_t			optimal_nonseq_zones;
330 	uint64_t			max_seq_zones;
331 	u_int			maxio;
332 	uint32_t		unmap_max_ranges;
333 	uint32_t		unmap_max_lba; /* Max LBAs in UNMAP req */
334 	uint32_t		unmap_gran;
335 	uint32_t		unmap_gran_align;
336 	uint64_t		ws_max_blks;
337 	da_delete_methods	delete_method_pref;
338 	da_delete_methods	delete_method;
339 	da_delete_func_t	*delete_func;
340 	int			unmappedio;
341 	int			rotating;
342 	struct	 disk_params params;
343 	struct	 disk *disk;
344 	union	 ccb saved_ccb;
345 	struct task		sysctl_task;
346 	struct sysctl_ctx_list	sysctl_ctx;
347 	struct sysctl_oid	*sysctl_tree;
348 	struct callout		sendordered_c;
349 	uint64_t wwpn;
350 	uint8_t	 unmap_buf[UNMAP_BUF_SIZE];
351 	struct scsi_read_capacity_data_long rcaplong;
352 	struct callout		mediapoll_c;
353 	int			ref_flags[DA_REF_MAX];
354 #ifdef CAM_IO_STATS
355 	struct sysctl_ctx_list	sysctl_stats_ctx;
356 	struct sysctl_oid	*sysctl_stats_tree;
357 	u_int	errors;
358 	u_int	timeouts;
359 	u_int	invalidations;
360 #endif
361 #define DA_ANNOUNCETMP_SZ 160
362 	char			announce_temp[DA_ANNOUNCETMP_SZ];
363 #define DA_ANNOUNCE_SZ 400
364 	char			announcebuf[DA_ANNOUNCE_SZ];
365 };
366 
367 #define dadeleteflag(softc, delete_method, enable)			\
368 	if (enable) {							\
369 		softc->delete_available |= (1 << delete_method);	\
370 	} else {							\
371 		softc->delete_available &= ~(1 << delete_method);	\
372 	}
373 
374 struct da_quirk_entry {
375 	struct scsi_inquiry_pattern inq_pat;
376 	da_quirks quirks;
377 };
378 
379 static const char quantum[] = "QUANTUM";
380 static const char microp[] = "MICROP";
381 
382 static struct da_quirk_entry da_quirk_table[] =
383 {
384 	/* SPI, FC devices */
385 	{
386 		/*
387 		 * Fujitsu M2513A MO drives.
388 		 * Tested devices: M2513A2 firmware versions 1200 & 1300.
389 		 * (dip switch selects whether T_DIRECT or T_OPTICAL device)
390 		 * Reported by: W.Scholten <whs@xs4all.nl>
391 		 */
392 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"},
393 		/*quirks*/ DA_Q_NO_SYNC_CACHE
394 	},
395 	{
396 		/* See above. */
397 		{T_OPTICAL, SIP_MEDIA_REMOVABLE, "FUJITSU", "M2513A", "*"},
398 		/*quirks*/ DA_Q_NO_SYNC_CACHE
399 	},
400 	{
401 		/*
402 		 * This particular Fujitsu drive doesn't like the
403 		 * synchronize cache command.
404 		 * Reported by: Tom Jackson <toj@gorilla.net>
405 		 */
406 		{T_DIRECT, SIP_MEDIA_FIXED, "FUJITSU", "M2954*", "*"},
407 		/*quirks*/ DA_Q_NO_SYNC_CACHE
408 	},
409 	{
410 		/*
411 		 * This drive doesn't like the synchronize cache command
412 		 * either.  Reported by: Matthew Jacob <mjacob@feral.com>
413 		 * in NetBSD PR kern/6027, August 24, 1998.
414 		 */
415 		{T_DIRECT, SIP_MEDIA_FIXED, microp, "2217*", "*"},
416 		/*quirks*/ DA_Q_NO_SYNC_CACHE
417 	},
418 	{
419 		/*
420 		 * This drive doesn't like the synchronize cache command
421 		 * either.  Reported by: Hellmuth Michaelis (hm@kts.org)
422 		 * (PR 8882).
423 		 */
424 		{T_DIRECT, SIP_MEDIA_FIXED, microp, "2112*", "*"},
425 		/*quirks*/ DA_Q_NO_SYNC_CACHE
426 	},
427 	{
428 		/*
429 		 * Doesn't like the synchronize cache command.
430 		 * Reported by: Blaz Zupan <blaz@gold.amis.net>
431 		 */
432 		{T_DIRECT, SIP_MEDIA_FIXED, "NEC", "D3847*", "*"},
433 		/*quirks*/ DA_Q_NO_SYNC_CACHE
434 	},
435 	{
436 		/*
437 		 * Doesn't like the synchronize cache command.
438 		 * Reported by: Blaz Zupan <blaz@gold.amis.net>
439 		 */
440 		{T_DIRECT, SIP_MEDIA_FIXED, quantum, "MAVERICK 540S", "*"},
441 		/*quirks*/ DA_Q_NO_SYNC_CACHE
442 	},
443 	{
444 		/*
445 		 * Doesn't like the synchronize cache command.
446 		 */
447 		{T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS525S", "*"},
448 		/*quirks*/ DA_Q_NO_SYNC_CACHE
449 	},
450 	{
451 		/*
452 		 * Doesn't like the synchronize cache command.
453 		 * Reported by: walter@pelissero.de
454 		 */
455 		{T_DIRECT, SIP_MEDIA_FIXED, quantum, "LPS540S", "*"},
456 		/*quirks*/ DA_Q_NO_SYNC_CACHE
457 	},
458 	{
459 		/*
460 		 * Doesn't work correctly with 6 byte reads/writes.
461 		 * Returns illegal request, and points to byte 9 of the
462 		 * 6-byte CDB.
463 		 * Reported by:  Adam McDougall <bsdx@spawnet.com>
464 		 */
465 		{T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 4*", "*"},
466 		/*quirks*/ DA_Q_NO_6_BYTE
467 	},
468 	{
469 		/* See above. */
470 		{T_DIRECT, SIP_MEDIA_FIXED, quantum, "VIKING 2*", "*"},
471 		/*quirks*/ DA_Q_NO_6_BYTE
472 	},
473 	{
474 		/*
475 		 * Doesn't like the synchronize cache command.
476 		 * Reported by: walter@pelissero.de
477 		 */
478 		{T_DIRECT, SIP_MEDIA_FIXED, "CONNER", "CP3500*", "*"},
479 		/*quirks*/ DA_Q_NO_SYNC_CACHE
480 	},
481 	{
482 		/*
483 		 * The CISS RAID controllers do not support SYNC_CACHE
484 		 */
485 		{T_DIRECT, SIP_MEDIA_FIXED, "COMPAQ", "RAID*", "*"},
486 		/*quirks*/ DA_Q_NO_SYNC_CACHE
487 	},
488 	{
489 		/*
490 		 * The STEC SSDs sometimes hang on UNMAP.
491 		 */
492 		{T_DIRECT, SIP_MEDIA_FIXED, "STEC", "*", "*"},
493 		/*quirks*/ DA_Q_NO_UNMAP
494 	},
495 	{
496 		/*
497 		 * VMware returns BUSY status when storage has transient
498 		 * connectivity problems, so better wait.
499 		 * Also VMware returns odd errors on misaligned UNMAPs.
500 		 */
501 		{T_DIRECT, SIP_MEDIA_FIXED, "VMware*", "*", "*"},
502 		/*quirks*/ DA_Q_RETRY_BUSY | DA_Q_STRICT_UNMAP
503 	},
504 	/* USB mass storage devices supported by umass(4) */
505 	{
506 		/*
507 		 * EXATELECOM (Sigmatel) i-Bead 100/105 USB Flash MP3 Player
508 		 * PR: kern/51675
509 		 */
510 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "EXATEL", "i-BEAD10*", "*"},
511 		/*quirks*/ DA_Q_NO_SYNC_CACHE
512 	},
513 	{
514 		/*
515 		 * Power Quotient Int. (PQI) USB flash key
516 		 * PR: kern/53067
517 		 */
518 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "USB Flash Disk*",
519 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
520 	},
521 	{
522 		/*
523 		 * Creative Nomad MUVO mp3 player (USB)
524 		 * PR: kern/53094
525 		 */
526 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "NOMAD_MUVO", "*"},
527 		/*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
528 	},
529 	{
530 		/*
531 		 * Jungsoft NEXDISK USB flash key
532 		 * PR: kern/54737
533 		 */
534 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "JUNGSOFT", "NEXDISK*", "*"},
535 		/*quirks*/ DA_Q_NO_SYNC_CACHE
536 	},
537 	{
538 		/*
539 		 * FreeDik USB Mini Data Drive
540 		 * PR: kern/54786
541 		 */
542 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "FreeDik*", "Mini Data Drive",
543 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
544 	},
545 	{
546 		/*
547 		 * Sigmatel USB Flash MP3 Player
548 		 * PR: kern/57046
549 		 */
550 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "SigmaTel", "MSCN", "*"},
551 		/*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
552 	},
553 	{
554 		/*
555 		 * Neuros USB Digital Audio Computer
556 		 * PR: kern/63645
557 		 */
558 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "NEUROS", "dig. audio comp.",
559 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
560 	},
561 	{
562 		/*
563 		 * SEAGRAND NP-900 MP3 Player
564 		 * PR: kern/64563
565 		 */
566 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "SEAGRAND", "NP-900*", "*"},
567 		/*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
568 	},
569 	{
570 		/*
571 		 * iRiver iFP MP3 player (with UMS Firmware)
572 		 * PR: kern/54881, i386/63941, kern/66124
573 		 */
574 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "iRiver", "iFP*", "*"},
575 		/*quirks*/ DA_Q_NO_SYNC_CACHE
576 	},
577 	{
578 		/*
579 		 * Frontier Labs NEX IA+ Digital Audio Player, rev 1.10/0.01
580 		 * PR: kern/70158
581 		 */
582 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "FL" , "Nex*", "*"},
583 		/*quirks*/ DA_Q_NO_SYNC_CACHE
584 	},
585 	{
586 		/*
587 		 * ZICPlay USB MP3 Player with FM
588 		 * PR: kern/75057
589 		 */
590 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "ACTIONS*" , "USB DISK*", "*"},
591 		/*quirks*/ DA_Q_NO_SYNC_CACHE
592 	},
593 	{
594 		/*
595 		 * TEAC USB floppy mechanisms
596 		 */
597 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "TEAC" , "FD-05*", "*"},
598 		/*quirks*/ DA_Q_NO_SYNC_CACHE
599 	},
600 	{
601 		/*
602 		 * Kingston DataTraveler II+ USB Pen-Drive.
603 		 * Reported by: Pawel Jakub Dawidek <pjd@FreeBSD.org>
604 		 */
605 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston" , "DataTraveler II+",
606 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
607 	},
608 	{
609 		/*
610 		 * USB DISK Pro PMAP
611 		 * Reported by: jhs
612 		 * PR: usb/96381
613 		 */
614 		{T_DIRECT, SIP_MEDIA_REMOVABLE, " ", "USB DISK Pro", "PMAP"},
615 		/*quirks*/ DA_Q_NO_SYNC_CACHE
616 	},
617 	{
618 		/*
619 		 * Motorola E398 Mobile Phone (TransFlash memory card).
620 		 * Reported by: Wojciech A. Koszek <dunstan@FreeBSD.czest.pl>
621 		 * PR: usb/89889
622 		 */
623 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Motorola" , "Motorola Phone",
624 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
625 	},
626 	{
627 		/*
628 		 * Qware BeatZkey! Pro
629 		 * PR: usb/79164
630 		 */
631 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "GENERIC", "USB DISK DEVICE",
632 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
633 	},
634 	{
635 		/*
636 		 * Time DPA20B 1GB MP3 Player
637 		 * PR: usb/81846
638 		 */
639 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "USB2.0*", "(FS) FLASH DISK*",
640 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
641 	},
642 	{
643 		/*
644 		 * Samsung USB key 128Mb
645 		 * PR: usb/90081
646 		 */
647 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "USB-DISK", "FreeDik-FlashUsb",
648 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
649 	},
650 	{
651 		/*
652 		 * Kingston DataTraveler 2.0 USB Flash memory.
653 		 * PR: usb/89196
654 		 */
655 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler 2.0",
656 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
657 	},
658 	{
659 		/*
660 		 * Creative MUVO Slim mp3 player (USB)
661 		 * PR: usb/86131
662 		 */
663 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "CREATIVE", "MuVo Slim",
664 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE|DA_Q_NO_PREVENT
665 		},
666 	{
667 		/*
668 		 * United MP5512 Portable MP3 Player (2-in-1 USB DISK/MP3)
669 		 * PR: usb/80487
670 		 */
671 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "MUSIC DISK",
672 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
673 	},
674 	{
675 		/*
676 		 * SanDisk Micro Cruzer 128MB
677 		 * PR: usb/75970
678 		 */
679 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "SanDisk" , "Micro Cruzer",
680 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
681 	},
682 	{
683 		/*
684 		 * TOSHIBA TransMemory USB sticks
685 		 * PR: kern/94660
686 		 */
687 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "TOSHIBA", "TransMemory",
688 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
689 	},
690 	{
691 		/*
692 		 * PNY USB 3.0 Flash Drives
693 		*/
694 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "PNY", "USB 3.0 FD*",
695 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_RC16
696 	},
697 	{
698 		/*
699 		 * PNY USB Flash keys
700 		 * PR: usb/75578, usb/72344, usb/65436
701 		 */
702 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "*" , "USB DISK*",
703 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
704 	},
705 	{
706 		/*
707 		 * Genesys GL3224
708 		 */
709 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*",
710 		"120?"}, /*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_4K | DA_Q_NO_RC16
711 	},
712 	{
713 		/*
714 		 * Genesys 6-in-1 Card Reader
715 		 * PR: usb/94647
716 		 */
717 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Generic*", "STORAGE DEVICE*",
718 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
719 	},
720 	{
721 		/*
722 		 * Rekam Digital CAMERA
723 		 * PR: usb/98713
724 		 */
725 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "CAMERA*", "4MP-9J6*",
726 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
727 	},
728 	{
729 		/*
730 		 * iRiver H10 MP3 player
731 		 * PR: usb/102547
732 		 */
733 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "H10*",
734 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
735 	},
736 	{
737 		/*
738 		 * iRiver U10 MP3 player
739 		 * PR: usb/92306
740 		 */
741 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "iriver", "U10*",
742 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
743 	},
744 	{
745 		/*
746 		 * X-Micro Flash Disk
747 		 * PR: usb/96901
748 		 */
749 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "X-Micro", "Flash Disk",
750 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
751 	},
752 	{
753 		/*
754 		 * EasyMP3 EM732X USB 2.0 Flash MP3 Player
755 		 * PR: usb/96546
756 		 */
757 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "EM732X", "MP3 Player*",
758 		"1.00"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
759 	},
760 	{
761 		/*
762 		 * Denver MP3 player
763 		 * PR: usb/107101
764 		 */
765 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "DENVER", "MP3 PLAYER",
766 		 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
767 	},
768 	{
769 		/*
770 		 * Philips USB Key Audio KEY013
771 		 * PR: usb/68412
772 		 */
773 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "PHILIPS", "Key*", "*"},
774 		/*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT
775 	},
776 	{
777 		/*
778 		 * JNC MP3 Player
779 		 * PR: usb/94439
780 		 */
781 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "JNC*" , "MP3 Player*",
782 		 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
783 	},
784 	{
785 		/*
786 		 * SAMSUNG MP0402H
787 		 * PR: usb/108427
788 		 */
789 		{T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "MP0402H", "*"},
790 		/*quirks*/ DA_Q_NO_SYNC_CACHE
791 	},
792 	{
793 		/*
794 		 * I/O Magic USB flash - Giga Bank
795 		 * PR: usb/108810
796 		 */
797 		{T_DIRECT, SIP_MEDIA_FIXED, "GS-Magic", "stor*", "*"},
798 		/*quirks*/ DA_Q_NO_SYNC_CACHE
799 	},
800 	{
801 		/*
802 		 * JoyFly 128mb USB Flash Drive
803 		 * PR: 96133
804 		 */
805 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "Flash Disk*",
806 		 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
807 	},
808 	{
809 		/*
810 		 * ChipsBnk usb stick
811 		 * PR: 103702
812 		 */
813 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "ChipsBnk", "USB*",
814 		 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
815 	},
816 	{
817 		/*
818 		 * Storcase (Kingston) InfoStation IFS FC2/SATA-R 201A
819 		 * PR: 129858
820 		 */
821 		{T_DIRECT, SIP_MEDIA_FIXED, "IFS", "FC2/SATA-R*",
822 		 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
823 	},
824 	{
825 		/*
826 		 * Samsung YP-U3 mp3-player
827 		 * PR: 125398
828 		 */
829 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Samsung", "YP-U3",
830 		 "*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
831 	},
832 	{
833 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Netac", "OnlyDisk*",
834 		 "2000"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
835 	},
836 	{
837 		/*
838 		 * Sony Cyber-Shot DSC cameras
839 		 * PR: usb/137035
840 		 */
841 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Sony", "Sony DSC", "*"},
842 		/*quirks*/ DA_Q_NO_SYNC_CACHE | DA_Q_NO_PREVENT
843 	},
844 	{
845 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "Kingston", "DataTraveler G3",
846 		 "1.00"}, /*quirks*/ DA_Q_NO_PREVENT
847 	},
848 	{
849 		/* At least several Transcent USB sticks lie on RC16. */
850 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "JetFlash", "Transcend*",
851 		 "*"}, /*quirks*/ DA_Q_NO_RC16
852 	},
853 	{
854 		/*
855 		 * I-O Data USB Flash Disk
856 		 * PR: usb/211716
857 		 */
858 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "I-O DATA", "USB Flash Disk*",
859 		 "*"}, /*quirks*/ DA_Q_NO_RC16
860 	},
861 	/* ATA/SATA devices over SAS/USB/... */
862 	{
863 		/* Hitachi Advanced Format (4k) drives */
864 		{ T_DIRECT, SIP_MEDIA_FIXED, "Hitachi", "H??????????E3*", "*" },
865 		/*quirks*/DA_Q_4K
866 	},
867 	{
868 		/* Micron Advanced Format (4k) drives */
869 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Micron 5100 MTFDDAK*", "*" },
870 		/*quirks*/DA_Q_4K
871 	},
872 	{
873 		/* Samsung Advanced Format (4k) drives */
874 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD155UI*", "*" },
875 		/*quirks*/DA_Q_4K
876 	},
877 	{
878 		/* Samsung Advanced Format (4k) drives */
879 		{ T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD155UI*", "*" },
880 		/*quirks*/DA_Q_4K
881 	},
882 	{
883 		/* Samsung Advanced Format (4k) drives */
884 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG HD204UI*", "*" },
885 		/*quirks*/DA_Q_4K
886 	},
887 	{
888 		/* Samsung Advanced Format (4k) drives */
889 		{ T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HD204UI*", "*" },
890 		/*quirks*/DA_Q_4K
891 	},
892 	{
893 		/* Seagate Barracuda Green Advanced Format (4k) drives */
894 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DL*", "*" },
895 		/*quirks*/DA_Q_4K
896 	},
897 	{
898 		/* Seagate Barracuda Green Advanced Format (4k) drives */
899 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST????DL", "*", "*" },
900 		/*quirks*/DA_Q_4K
901 	},
902 	{
903 		/* Seagate Barracuda Green Advanced Format (4k) drives */
904 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???DM*", "*" },
905 		/*quirks*/DA_Q_4K
906 	},
907 	{
908 		/* Seagate Barracuda Green Advanced Format (4k) drives */
909 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST???DM*", "*", "*" },
910 		/*quirks*/DA_Q_4K
911 	},
912 	{
913 		/* Seagate Barracuda Green Advanced Format (4k) drives */
914 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST????DM*", "*" },
915 		/*quirks*/DA_Q_4K
916 	},
917 	{
918 		/* Seagate Barracuda Green Advanced Format (4k) drives */
919 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST????DM", "*", "*" },
920 		/*quirks*/DA_Q_4K
921 	},
922 	{
923 		/* Seagate Momentus Advanced Format (4k) drives */
924 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500423AS*", "*" },
925 		/*quirks*/DA_Q_4K
926 	},
927 	{
928 		/* Seagate Momentus Advanced Format (4k) drives */
929 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "3AS*", "*" },
930 		/*quirks*/DA_Q_4K
931 	},
932 	{
933 		/* Seagate Momentus Advanced Format (4k) drives */
934 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9500424AS*", "*" },
935 		/*quirks*/DA_Q_4K
936 	},
937 	{
938 		/* Seagate Momentus Advanced Format (4k) drives */
939 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST950042", "4AS*", "*" },
940 		/*quirks*/DA_Q_4K
941 	},
942 	{
943 		/* Seagate Momentus Advanced Format (4k) drives */
944 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640423AS*", "*" },
945 		/*quirks*/DA_Q_4K
946 	},
947 	{
948 		/* Seagate Momentus Advanced Format (4k) drives */
949 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "3AS*", "*" },
950 		/*quirks*/DA_Q_4K
951 	},
952 	{
953 		/* Seagate Momentus Advanced Format (4k) drives */
954 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9640424AS*", "*" },
955 		/*quirks*/DA_Q_4K
956 	},
957 	{
958 		/* Seagate Momentus Advanced Format (4k) drives */
959 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST964042", "4AS*", "*" },
960 		/*quirks*/DA_Q_4K
961 	},
962 	{
963 		/* Seagate Momentus Advanced Format (4k) drives */
964 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750420AS*", "*" },
965 		/*quirks*/DA_Q_4K
966 	},
967 	{
968 		/* Seagate Momentus Advanced Format (4k) drives */
969 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "0AS*", "*" },
970 		/*quirks*/DA_Q_4K
971 	},
972 	{
973 		/* Seagate Momentus Advanced Format (4k) drives */
974 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750422AS*", "*" },
975 		/*quirks*/DA_Q_4K
976 	},
977 	{
978 		/* Seagate Momentus Advanced Format (4k) drives */
979 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "2AS*", "*" },
980 		/*quirks*/DA_Q_4K
981 	},
982 	{
983 		/* Seagate Momentus Advanced Format (4k) drives */
984 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST9750423AS*", "*" },
985 		/*quirks*/DA_Q_4K
986 	},
987 	{
988 		/* Seagate Momentus Advanced Format (4k) drives */
989 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST975042", "3AS*", "*" },
990 		/*quirks*/DA_Q_4K
991 	},
992 	{
993 		/* Seagate Momentus Thin Advanced Format (4k) drives */
994 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST???LT*", "*" },
995 		/*quirks*/DA_Q_4K
996 	},
997 	{
998 		/* Seagate Momentus Thin Advanced Format (4k) drives */
999 		{ T_DIRECT, SIP_MEDIA_FIXED, "ST???LT*", "*", "*" },
1000 		/*quirks*/DA_Q_4K
1001 	},
1002 	{
1003 		/* WDC Caviar Green Advanced Format (4k) drives */
1004 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RS*", "*" },
1005 		/*quirks*/DA_Q_4K
1006 	},
1007 	{
1008 		/* WDC Caviar Green Advanced Format (4k) drives */
1009 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RS*", "*" },
1010 		/*quirks*/DA_Q_4K
1011 	},
1012 	{
1013 		/* WDC Caviar Green Advanced Format (4k) drives */
1014 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD????RX*", "*" },
1015 		/*quirks*/DA_Q_4K
1016 	},
1017 	{
1018 		/* WDC Caviar Green Advanced Format (4k) drives */
1019 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "??RX*", "*" },
1020 		/*quirks*/DA_Q_4K
1021 	},
1022 	{
1023 		/* WDC Caviar Green Advanced Format (4k) drives */
1024 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RS*", "*" },
1025 		/*quirks*/DA_Q_4K
1026 	},
1027 	{
1028 		/* WDC Caviar Green Advanced Format (4k) drives */
1029 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RS*", "*" },
1030 		/*quirks*/DA_Q_4K
1031 	},
1032 	{
1033 		/* WDC Caviar Green Advanced Format (4k) drives */
1034 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD??????RX*", "*" },
1035 		/*quirks*/DA_Q_4K
1036 	},
1037 	{
1038 		/* WDC Caviar Green Advanced Format (4k) drives */
1039 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "????RX*", "*" },
1040 		/*quirks*/DA_Q_4K
1041 	},
1042 	{
1043 		/* WDC Scorpio Black Advanced Format (4k) drives */
1044 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PKT*", "*" },
1045 		/*quirks*/DA_Q_4K
1046 	},
1047 	{
1048 		/* WDC Scorpio Black Advanced Format (4k) drives */
1049 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PKT*", "*" },
1050 		/*quirks*/DA_Q_4K
1051 	},
1052 	{
1053 		/* WDC Scorpio Black Advanced Format (4k) drives */
1054 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PKT*", "*" },
1055 		/*quirks*/DA_Q_4K
1056 	},
1057 	{
1058 		/* WDC Scorpio Black Advanced Format (4k) drives */
1059 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PKT*", "*" },
1060 		/*quirks*/DA_Q_4K
1061 	},
1062 	{
1063 		/* WDC Scorpio Blue Advanced Format (4k) drives */
1064 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD???PVT*", "*" },
1065 		/*quirks*/DA_Q_4K
1066 	},
1067 	{
1068 		/* WDC Scorpio Blue Advanced Format (4k) drives */
1069 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "?PVT*", "*" },
1070 		/*quirks*/DA_Q_4K
1071 	},
1072 	{
1073 		/* WDC Scorpio Blue Advanced Format (4k) drives */
1074 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "WDC WD?????PVT*", "*" },
1075 		/*quirks*/DA_Q_4K
1076 	},
1077 	{
1078 		/* WDC Scorpio Blue Advanced Format (4k) drives */
1079 		{ T_DIRECT, SIP_MEDIA_FIXED, "WDC WD??", "???PVT*", "*" },
1080 		/*quirks*/DA_Q_4K
1081 	},
1082 	{
1083 		/*
1084 		 * Olympus FE-210 camera
1085 		 */
1086 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "OLYMPUS", "FE210*",
1087 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1088 	},
1089 	{
1090 		/*
1091 		 * LG UP3S MP3 player
1092 		 */
1093 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "LG", "UP3S",
1094 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1095 	},
1096 	{
1097 		/*
1098 		 * Laser MP3-2GA13 MP3 player
1099 		 */
1100 		{T_DIRECT, SIP_MEDIA_REMOVABLE, "USB 2.0", "(HS) Flash Disk",
1101 		"*"}, /*quirks*/ DA_Q_NO_SYNC_CACHE
1102 	},
1103 	{
1104 		/*
1105 		 * LaCie external 250GB Hard drive des by Porsche
1106 		 * Submitted by: Ben Stuyts <ben@altesco.nl>
1107 		 * PR: 121474
1108 		 */
1109 		{T_DIRECT, SIP_MEDIA_FIXED, "SAMSUNG", "HM250JI", "*"},
1110 		/*quirks*/ DA_Q_NO_SYNC_CACHE
1111 	},
1112 	/* SATA SSDs */
1113 	{
1114 		/*
1115 		 * Corsair Force 2 SSDs
1116 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1117 		 */
1118 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair CSSD-F*", "*" },
1119 		/*quirks*/DA_Q_4K
1120 	},
1121 	{
1122 		/*
1123 		 * Corsair Force 3 SSDs
1124 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1125 		 */
1126 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force 3*", "*" },
1127 		/*quirks*/DA_Q_4K
1128 	},
1129         {
1130 		/*
1131 		 * Corsair Neutron GTX SSDs
1132 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1133 		 */
1134 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
1135 		/*quirks*/DA_Q_4K
1136 	},
1137 	{
1138 		/*
1139 		 * Corsair Force GT & GS SSDs
1140 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1141 		 */
1142 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Corsair Force G*", "*" },
1143 		/*quirks*/DA_Q_4K
1144 	},
1145 	{
1146 		/*
1147 		 * Crucial M4 SSDs
1148 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1149 		 */
1150 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "M4-CT???M4SSD2*", "*" },
1151 		/*quirks*/DA_Q_4K
1152 	},
1153 	{
1154 		/*
1155 		 * Crucial RealSSD C300 SSDs
1156 		 * 4k optimised
1157 		 */
1158 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "C300-CTFDDAC???MAG*",
1159 		"*" }, /*quirks*/DA_Q_4K
1160 	},
1161 	{
1162 		/*
1163 		 * Intel 320 Series SSDs
1164 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1165 		 */
1166 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2CW*", "*" },
1167 		/*quirks*/DA_Q_4K
1168 	},
1169 	{
1170 		/*
1171 		 * Intel 330 Series SSDs
1172 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1173 		 */
1174 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2CT*", "*" },
1175 		/*quirks*/DA_Q_4K
1176 	},
1177 	{
1178 		/*
1179 		 * Intel 510 Series SSDs
1180 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1181 		 */
1182 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2MH*", "*" },
1183 		/*quirks*/DA_Q_4K
1184 	},
1185 	{
1186 		/*
1187 		 * Intel 520 Series SSDs
1188 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1189 		 */
1190 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BW*", "*" },
1191 		/*quirks*/DA_Q_4K
1192 	},
1193 	{
1194 		/*
1195 		 * Intel S3610 Series SSDs
1196 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1197 		 */
1198 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSC2BX*", "*" },
1199 		/*quirks*/DA_Q_4K
1200 	},
1201 	{
1202 		/*
1203 		 * Intel X25-M Series SSDs
1204 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1205 		 */
1206 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "INTEL SSDSA2M*", "*" },
1207 		/*quirks*/DA_Q_4K
1208 	},
1209 	{
1210 		/*
1211 		 * Kingston E100 Series SSDs
1212 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1213 		 */
1214 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SE100S3*", "*" },
1215 		/*quirks*/DA_Q_4K
1216 	},
1217 	{
1218 		/*
1219 		 * Kingston HyperX 3k SSDs
1220 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1221 		 */
1222 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "KINGSTON SH103S3*", "*" },
1223 		/*quirks*/DA_Q_4K
1224 	},
1225 	{
1226 		/*
1227 		 * Marvell SSDs (entry taken from OpenSolaris)
1228 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1229 		 */
1230 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MARVELL SD88SA02*", "*" },
1231 		/*quirks*/DA_Q_4K
1232 	},
1233 	{
1234 		/*
1235 		 * OCZ Agility 2 SSDs
1236 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1237 		 */
1238 		{ T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
1239 		/*quirks*/DA_Q_4K
1240 	},
1241 	{
1242 		/*
1243 		 * OCZ Agility 3 SSDs
1244 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1245 		 */
1246 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-AGILITY3*", "*" },
1247 		/*quirks*/DA_Q_4K
1248 	},
1249 	{
1250 		/*
1251 		 * OCZ Deneva R Series SSDs
1252 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1253 		 */
1254 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "DENRSTE251M45*", "*" },
1255 		/*quirks*/DA_Q_4K
1256 	},
1257 	{
1258 		/*
1259 		 * OCZ Vertex 2 SSDs (inc pro series)
1260 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1261 		 */
1262 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ?VERTEX2*", "*" },
1263 		/*quirks*/DA_Q_4K
1264 	},
1265 	{
1266 		/*
1267 		 * OCZ Vertex 3 SSDs
1268 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1269 		 */
1270 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX3*", "*" },
1271 		/*quirks*/DA_Q_4K
1272 	},
1273 	{
1274 		/*
1275 		 * OCZ Vertex 4 SSDs
1276 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1277 		 */
1278 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "OCZ-VERTEX4*", "*" },
1279 		/*quirks*/DA_Q_4K
1280 	},
1281 	{
1282 		/*
1283 		 * Samsung 750 Series SSDs
1284 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1285 		 */
1286 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 750*", "*" },
1287 		/*quirks*/DA_Q_4K
1288 	},
1289 	{
1290 		/*
1291 		 * Samsung 830 Series SSDs
1292 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1293 		 */
1294 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG SSD 830 Series*", "*" },
1295 		/*quirks*/DA_Q_4K
1296 	},
1297 	{
1298 		/*
1299 		 * Samsung 840 SSDs
1300 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1301 		 */
1302 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 840*", "*" },
1303 		/*quirks*/DA_Q_4K
1304 	},
1305 	{
1306 		/*
1307 		 * Samsung 845 SSDs
1308 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1309 		 */
1310 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 845*", "*" },
1311 		/*quirks*/DA_Q_4K
1312 	},
1313 	{
1314 		/*
1315 		 * Samsung 850 SSDs
1316 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1317 		 */
1318 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "Samsung SSD 850*", "*" },
1319 		/*quirks*/DA_Q_4K
1320 	},
1321 	{
1322 		/*
1323 		 * Samsung 843T Series SSDs (MZ7WD*)
1324 		 * Samsung PM851 Series SSDs (MZ7TE*)
1325 		 * Samsung PM853T Series SSDs (MZ7GE*)
1326 		 * Samsung SM863 Series SSDs (MZ7KM*)
1327 		 * 4k optimised
1328 		 */
1329 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SAMSUNG MZ7*", "*" },
1330 		/*quirks*/DA_Q_4K
1331 	},
1332 	{
1333 		/*
1334 		 * Same as for SAMSUNG MZ7* but enable the quirks for SSD
1335 		 * starting with MZ7* too
1336 		 */
1337 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "MZ7*", "*" },
1338 		/*quirks*/DA_Q_4K
1339 	},
1340 	{
1341 		/*
1342 		 * SuperTalent TeraDrive CT SSDs
1343 		 * 4k optimised & trim only works in 4k requests + 4k aligned
1344 		 */
1345 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "FTM??CT25H*", "*" },
1346 		/*quirks*/DA_Q_4K
1347 	},
1348 	{
1349 		/*
1350 		 * XceedIOPS SATA SSDs
1351 		 * 4k optimised
1352 		 */
1353 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "SG9XCS2D*", "*" },
1354 		/*quirks*/DA_Q_4K
1355 	},
1356 	{
1357 		/*
1358 		 * Hama Innostor USB-Stick
1359 		 */
1360 		{ T_DIRECT, SIP_MEDIA_REMOVABLE, "Innostor", "Innostor*", "*" },
1361 		/*quirks*/DA_Q_NO_RC16
1362 	},
1363 	{
1364 		/*
1365 		 * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
1366 		 * Drive Managed SATA hard drive.  This drive doesn't report
1367 		 * in firmware that it is a drive managed SMR drive.
1368 		 */
1369 		{ T_DIRECT, SIP_MEDIA_FIXED, "ATA", "ST8000AS000[23]*", "*" },
1370 		/*quirks*/DA_Q_SMR_DM
1371 	},
1372 	{
1373 		/*
1374 		 * MX-ES USB Drive by Mach Xtreme
1375 		 */
1376 		{ T_DIRECT, SIP_MEDIA_REMOVABLE, "MX", "MXUB3*", "*"},
1377 		/*quirks*/DA_Q_NO_RC16
1378 	},
1379 };
1380 
1381 static	disk_strategy_t	dastrategy;
1382 static	dumper_t	dadump;
1383 static	periph_init_t	dainit;
1384 static	void		daasync(void *callback_arg, u_int32_t code,
1385 				struct cam_path *path, void *arg);
1386 static	void		dasysctlinit(void *context, int pending);
1387 static	int		dasysctlsofttimeout(SYSCTL_HANDLER_ARGS);
1388 static	int		dacmdsizesysctl(SYSCTL_HANDLER_ARGS);
1389 static	int		dadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
1390 static	int		dazonemodesysctl(SYSCTL_HANDLER_ARGS);
1391 static	int		dazonesupsysctl(SYSCTL_HANDLER_ARGS);
1392 static	int		dadeletemaxsysctl(SYSCTL_HANDLER_ARGS);
1393 static	void		dadeletemethodset(struct da_softc *softc,
1394 					  da_delete_methods delete_method);
1395 static	off_t		dadeletemaxsize(struct da_softc *softc,
1396 					da_delete_methods delete_method);
1397 static	void		dadeletemethodchoose(struct da_softc *softc,
1398 					     da_delete_methods default_method);
1399 static	void		daprobedone(struct cam_periph *periph, union ccb *ccb);
1400 
1401 static	periph_ctor_t	daregister;
1402 static	periph_dtor_t	dacleanup;
1403 static	periph_start_t	dastart;
1404 static	periph_oninv_t	daoninvalidate;
1405 static	void		dazonedone(struct cam_periph *periph, union ccb *ccb);
1406 static	void		dadone(struct cam_periph *periph,
1407 			       union ccb *done_ccb);
1408 static void		dadone_probewp(struct cam_periph *periph,
1409 				       union ccb *done_ccb);
1410 static void		dadone_proberc(struct cam_periph *periph,
1411 				       union ccb *done_ccb);
1412 static void		dadone_probelbp(struct cam_periph *periph,
1413 					union ccb *done_ccb);
1414 static void		dadone_probeblklimits(struct cam_periph *periph,
1415 					      union ccb *done_ccb);
1416 static void		dadone_probebdc(struct cam_periph *periph,
1417 					union ccb *done_ccb);
1418 static void		dadone_probeata(struct cam_periph *periph,
1419 					union ccb *done_ccb);
1420 static void		dadone_probeatalogdir(struct cam_periph *periph,
1421 					      union ccb *done_ccb);
1422 static void		dadone_probeataiddir(struct cam_periph *periph,
1423 					     union ccb *done_ccb);
1424 static void		dadone_probeatasup(struct cam_periph *periph,
1425 					   union ccb *done_ccb);
1426 static void		dadone_probeatazone(struct cam_periph *periph,
1427 					    union ccb *done_ccb);
1428 static void		dadone_probezone(struct cam_periph *periph,
1429 					 union ccb *done_ccb);
1430 static void		dadone_tur(struct cam_periph *periph,
1431 				   union ccb *done_ccb);
1432 static  int		daerror(union ccb *ccb, u_int32_t cam_flags,
1433 				u_int32_t sense_flags);
1434 static void		daprevent(struct cam_periph *periph, int action);
1435 static void		dareprobe(struct cam_periph *periph);
1436 static void		dasetgeom(struct cam_periph *periph, uint32_t block_len,
1437 				  uint64_t maxsector,
1438 				  struct scsi_read_capacity_data_long *rcaplong,
1439 				  size_t rcap_size);
1440 static timeout_t	dasendorderedtag;
1441 static void		dashutdown(void *arg, int howto);
1442 static timeout_t	damediapoll;
1443 
1444 #ifndef	DA_DEFAULT_POLL_PERIOD
1445 #define	DA_DEFAULT_POLL_PERIOD	3
1446 #endif
1447 
1448 #ifndef DA_DEFAULT_TIMEOUT
1449 #define DA_DEFAULT_TIMEOUT 60	/* Timeout in seconds */
1450 #endif
1451 
1452 #ifndef DA_DEFAULT_SOFTTIMEOUT
1453 #define DA_DEFAULT_SOFTTIMEOUT	0
1454 #endif
1455 
1456 #ifndef	DA_DEFAULT_RETRY
1457 #define	DA_DEFAULT_RETRY	4
1458 #endif
1459 
1460 #ifndef	DA_DEFAULT_SEND_ORDERED
1461 #define	DA_DEFAULT_SEND_ORDERED	1
1462 #endif
1463 
1464 static int da_poll_period = DA_DEFAULT_POLL_PERIOD;
1465 static int da_retry_count = DA_DEFAULT_RETRY;
1466 static int da_default_timeout = DA_DEFAULT_TIMEOUT;
1467 static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT;
1468 static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
1469 
1470 static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD, 0,
1471             "CAM Direct Access Disk driver");
1472 SYSCTL_INT(_kern_cam_da, OID_AUTO, poll_period, CTLFLAG_RWTUN,
1473            &da_poll_period, 0, "Media polling period in seconds");
1474 SYSCTL_INT(_kern_cam_da, OID_AUTO, retry_count, CTLFLAG_RWTUN,
1475            &da_retry_count, 0, "Normal I/O retry count");
1476 SYSCTL_INT(_kern_cam_da, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
1477            &da_default_timeout, 0, "Normal I/O timeout (in seconds)");
1478 SYSCTL_INT(_kern_cam_da, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
1479            &da_send_ordered, 0, "Send Ordered Tags");
1480 
1481 SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout,
1482     CTLTYPE_UINT | CTLFLAG_RW, NULL, 0, dasysctlsofttimeout, "I",
1483     "Soft I/O timeout (ms)");
1484 TUNABLE_INT64("kern.cam.da.default_softtimeout", &da_default_softtimeout);
1485 
1486 /*
1487  * DA_ORDEREDTAG_INTERVAL determines how often, relative
1488  * to the default timeout, we check to see whether an ordered
1489  * tagged transaction is appropriate to prevent simple tag
1490  * starvation.  Since we'd like to ensure that there is at least
1491  * 1/2 of the timeout length left for a starved transaction to
1492  * complete after we've sent an ordered tag, we must poll at least
1493  * four times in every timeout period.  This takes care of the worst
1494  * case where a starved transaction starts during an interval that
1495  * meets the requirement "don't send an ordered tag" test so it takes
1496  * us two intervals to determine that a tag must be sent.
1497  */
1498 #ifndef DA_ORDEREDTAG_INTERVAL
1499 #define DA_ORDEREDTAG_INTERVAL 4
1500 #endif
1501 
1502 static struct periph_driver dadriver =
1503 {
1504 	dainit, "da",
1505 	TAILQ_HEAD_INITIALIZER(dadriver.units), /* generation */ 0
1506 };
1507 
1508 PERIPHDRIVER_DECLARE(da, dadriver);
1509 
1510 static MALLOC_DEFINE(M_SCSIDA, "scsi_da", "scsi_da buffers");
1511 
1512 /*
1513  * This driver takes out references / holds in well defined pairs, never
1514  * recursively. These macros / inline functions enforce those rules. They
1515  * are only enabled with DA_TRACK_REFS or INVARIANTS. If DA_TRACK_REFS is
1516  * defined to be 2 or larger, the tracking also includes debug printfs.
1517  */
1518 #if defined(DA_TRACK_REFS) || defined(INVARIANTS)
1519 
1520 #ifndef DA_TRACK_REFS
1521 #define DA_TRACK_REFS 1
1522 #endif
1523 
1524 #if DA_TRACK_REFS > 1
1525 static const char *da_ref_text[] = {
1526 	"bogus",
1527 	"open",
1528 	"open hold",
1529 	"close hold",
1530 	"reprobe hold",
1531 	"Test Unit Ready",
1532 	"Geom",
1533 	"sysctl",
1534 	"reprobe",
1535 	"max -- also bogus"
1536 };
1537 
1538 #define DA_PERIPH_PRINT(periph, msg, args...)		\
1539 	CAM_PERIPH_PRINT(periph, msg, ##args)
1540 #else
1541 #define DA_PERIPH_PRINT(periph, msg, args...)
1542 #endif
1543 
1544 static inline void
1545 token_sanity(da_ref_token token)
1546 {
1547 	if ((unsigned)token >= DA_REF_MAX)
1548 		panic("Bad token value passed in %d\n", token);
1549 }
1550 
1551 static inline int
1552 da_periph_hold(struct cam_periph *periph, int priority, da_ref_token token)
1553 {
1554 	int err = cam_periph_hold(periph, priority);
1555 
1556 	token_sanity(token);
1557 	DA_PERIPH_PRINT(periph, "Holding device %s (%d): %d\n",
1558 	    da_ref_text[token], token, err);
1559 	if (err == 0) {
1560 		int cnt;
1561 		struct da_softc *softc = periph->softc;
1562 
1563 		cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1);
1564 		if (cnt != 0)
1565 			panic("Re-holding for reason %d, cnt = %d", token, cnt);
1566 	}
1567 	return (err);
1568 }
1569 
1570 static inline void
1571 da_periph_unhold(struct cam_periph *periph, da_ref_token token)
1572 {
1573 	int cnt;
1574 	struct da_softc *softc = periph->softc;
1575 
1576 	token_sanity(token);
1577 	DA_PERIPH_PRINT(periph, "Unholding device %s (%d)\n",
1578 	    da_ref_text[token], token);
1579 	cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1580 	if (cnt != 1)
1581 		panic("Unholding %d with cnt = %d", token, cnt);
1582 	cam_periph_unhold(periph);
1583 }
1584 
1585 static inline int
1586 da_periph_acquire(struct cam_periph *periph, da_ref_token token)
1587 {
1588 	int err = cam_periph_acquire(periph);
1589 
1590 	token_sanity(token);
1591 	DA_PERIPH_PRINT(periph, "acquiring device %s (%d): %d\n",
1592 	    da_ref_text[token], token, err);
1593 	if (err == 0) {
1594 		int cnt;
1595 		struct da_softc *softc = periph->softc;
1596 
1597 		cnt = atomic_fetchadd_int(&softc->ref_flags[token], 1);
1598 		if (cnt != 0)
1599 			panic("Re-refing for reason %d, cnt = %d", token, cnt);
1600 	}
1601 	return (err);
1602 }
1603 
1604 static inline void
1605 da_periph_release(struct cam_periph *periph, da_ref_token token)
1606 {
1607 	int cnt;
1608 	struct da_softc *softc = periph->softc;
1609 
1610 	token_sanity(token);
1611 	DA_PERIPH_PRINT(periph, "releasing device %s (%d)\n",
1612 	    da_ref_text[token], token);
1613 	cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1614 	if (cnt != 1)
1615 		panic("Releasing %d with cnt = %d", token, cnt);
1616 	cam_periph_release(periph);
1617 }
1618 
1619 static inline void
1620 da_periph_release_locked(struct cam_periph *periph, da_ref_token token)
1621 {
1622 	int cnt;
1623 	struct da_softc *softc = periph->softc;
1624 
1625 	token_sanity(token);
1626 	DA_PERIPH_PRINT(periph, "releasing device (locked) %s (%d)\n",
1627 	    da_ref_text[token], token);
1628 	cnt = atomic_fetchadd_int(&softc->ref_flags[token], -1);
1629 	if (cnt != 1)
1630 		panic("Unholding %d with cnt = %d", token, cnt);
1631 	cam_periph_release_locked(periph);
1632 }
1633 
1634 #define cam_periph_hold POISON
1635 #define cam_periph_unhold POISON
1636 #define cam_periph_acquire POISON
1637 #define cam_periph_release POISON
1638 #define cam_periph_release_locked POISON
1639 
1640 #else
1641 #define	da_periph_hold(periph, prio, token)	cam_periph_hold((periph), (prio))
1642 #define da_periph_unhold(periph, token)		cam_periph_unhold((periph))
1643 #define da_periph_acquire(periph, token)	cam_periph_acquire((periph))
1644 #define da_periph_release(periph, token)	cam_periph_release((periph))
1645 #define da_periph_release_locked(periph, token)	cam_periph_release_locked((periph))
1646 #endif
1647 
1648 static int
1649 daopen(struct disk *dp)
1650 {
1651 	struct cam_periph *periph;
1652 	struct da_softc *softc;
1653 	int error;
1654 
1655 	periph = (struct cam_periph *)dp->d_drv1;
1656 	if (da_periph_acquire(periph, DA_REF_OPEN) != 0) {
1657 		return (ENXIO);
1658 	}
1659 
1660 	cam_periph_lock(periph);
1661 	if ((error = da_periph_hold(periph, PRIBIO|PCATCH, DA_REF_OPEN_HOLD)) != 0) {
1662 		cam_periph_unlock(periph);
1663 		da_periph_release(periph, DA_REF_OPEN);
1664 		return (error);
1665 	}
1666 
1667 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1668 	    ("daopen\n"));
1669 
1670 	softc = (struct da_softc *)periph->softc;
1671 	dareprobe(periph);
1672 
1673 	/* Wait for the disk size update.  */
1674 	error = cam_periph_sleep(periph, &softc->disk->d_mediasize, PRIBIO,
1675 	    "dareprobe", 0);
1676 	if (error != 0)
1677 		xpt_print(periph->path, "unable to retrieve capacity data\n");
1678 
1679 	if (periph->flags & CAM_PERIPH_INVALID)
1680 		error = ENXIO;
1681 
1682 	if (error == 0 && (softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 &&
1683 	    (softc->quirks & DA_Q_NO_PREVENT) == 0)
1684 		daprevent(periph, PR_PREVENT);
1685 
1686 	if (error == 0) {
1687 		softc->flags &= ~DA_FLAG_PACK_INVALID;
1688 		softc->flags |= DA_FLAG_OPEN;
1689 	}
1690 
1691 	da_periph_unhold(periph, DA_REF_OPEN_HOLD);
1692 	cam_periph_unlock(periph);
1693 
1694 	if (error != 0)
1695 		da_periph_release(periph, DA_REF_OPEN);
1696 
1697 	return (error);
1698 }
1699 
1700 static int
1701 daclose(struct disk *dp)
1702 {
1703 	struct	cam_periph *periph;
1704 	struct	da_softc *softc;
1705 	union	ccb *ccb;
1706 
1707 	periph = (struct cam_periph *)dp->d_drv1;
1708 	softc = (struct da_softc *)periph->softc;
1709 	cam_periph_lock(periph);
1710 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
1711 	    ("daclose\n"));
1712 
1713 	if (da_periph_hold(periph, PRIBIO, DA_REF_CLOSE_HOLD) == 0) {
1714 
1715 		/* Flush disk cache. */
1716 		if ((softc->flags & DA_FLAG_DIRTY) != 0 &&
1717 		    (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0 &&
1718 		    (softc->flags & DA_FLAG_PACK_INVALID) == 0) {
1719 			ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1720 			scsi_synchronize_cache(&ccb->csio, /*retries*/1,
1721 			    /*cbfcnp*/NULL, MSG_SIMPLE_Q_TAG,
1722 			    /*begin_lba*/0, /*lb_count*/0, SSD_FULL_SIZE,
1723 			    5 * 60 * 1000);
1724 			cam_periph_runccb(ccb, daerror, /*cam_flags*/0,
1725 			    /*sense_flags*/SF_RETRY_UA | SF_QUIET_IR,
1726 			    softc->disk->d_devstat);
1727 			softc->flags &= ~DA_FLAG_DIRTY;
1728 			xpt_release_ccb(ccb);
1729 		}
1730 
1731 		/* Allow medium removal. */
1732 		if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0 &&
1733 		    (softc->quirks & DA_Q_NO_PREVENT) == 0)
1734 			daprevent(periph, PR_ALLOW);
1735 
1736 		da_periph_unhold(periph, DA_REF_CLOSE_HOLD);
1737 	}
1738 
1739 	/*
1740 	 * If we've got removeable media, mark the blocksize as
1741 	 * unavailable, since it could change when new media is
1742 	 * inserted.
1743 	 */
1744 	if ((softc->flags & DA_FLAG_PACK_REMOVABLE) != 0)
1745 		softc->disk->d_devstat->flags |= DEVSTAT_BS_UNAVAILABLE;
1746 
1747 	softc->flags &= ~DA_FLAG_OPEN;
1748 	while (softc->refcount != 0)
1749 		cam_periph_sleep(periph, &softc->refcount, PRIBIO, "daclose", 1);
1750 	cam_periph_unlock(periph);
1751 	da_periph_release(periph, DA_REF_OPEN);
1752 	return (0);
1753 }
1754 
1755 static void
1756 daschedule(struct cam_periph *periph)
1757 {
1758 	struct da_softc *softc = (struct da_softc *)periph->softc;
1759 
1760 	if (softc->state != DA_STATE_NORMAL)
1761 		return;
1762 
1763 	cam_iosched_schedule(softc->cam_iosched, periph);
1764 }
1765 
1766 /*
1767  * Actually translate the requested transfer into one the physical driver
1768  * can understand.  The transfer is described by a buf and will include
1769  * only one physical transfer.
1770  */
1771 static void
1772 dastrategy(struct bio *bp)
1773 {
1774 	struct cam_periph *periph;
1775 	struct da_softc *softc;
1776 
1777 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1778 	softc = (struct da_softc *)periph->softc;
1779 
1780 	cam_periph_lock(periph);
1781 
1782 	/*
1783 	 * If the device has been made invalid, error out
1784 	 */
1785 	if ((softc->flags & DA_FLAG_PACK_INVALID)) {
1786 		cam_periph_unlock(periph);
1787 		biofinish(bp, NULL, ENXIO);
1788 		return;
1789 	}
1790 
1791 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastrategy(%p)\n", bp));
1792 
1793 	/*
1794 	 * Zone commands must be ordered, because they can depend on the
1795 	 * effects of previously issued commands, and they may affect
1796 	 * commands after them.
1797 	 */
1798 	if (bp->bio_cmd == BIO_ZONE)
1799 		bp->bio_flags |= BIO_ORDERED;
1800 
1801 	/*
1802 	 * Place it in the queue of disk activities for this disk
1803 	 */
1804 	cam_iosched_queue_work(softc->cam_iosched, bp);
1805 
1806 	/*
1807 	 * Schedule ourselves for performing the work.
1808 	 */
1809 	daschedule(periph);
1810 	cam_periph_unlock(periph);
1811 
1812 	return;
1813 }
1814 
1815 static int
1816 dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1817 {
1818 	struct	    cam_periph *periph;
1819 	struct	    da_softc *softc;
1820 	u_int	    secsize;
1821 	struct	    ccb_scsiio csio;
1822 	struct	    disk *dp;
1823 	int	    error = 0;
1824 
1825 	dp = arg;
1826 	periph = dp->d_drv1;
1827 	softc = (struct da_softc *)periph->softc;
1828 	secsize = softc->params.secsize;
1829 
1830 	if ((softc->flags & DA_FLAG_PACK_INVALID) != 0)
1831 		return (ENXIO);
1832 
1833 	memset(&csio, 0, sizeof(csio));
1834 	if (length > 0) {
1835 		xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1836 		csio.ccb_h.ccb_state = DA_CCB_DUMP;
1837 		scsi_read_write(&csio,
1838 				/*retries*/0,
1839 				/*cbfcnp*/NULL,
1840 				MSG_ORDERED_Q_TAG,
1841 				/*read*/SCSI_RW_WRITE,
1842 				/*byte2*/0,
1843 				/*minimum_cmd_size*/ softc->minimum_cmd_size,
1844 				offset / secsize,
1845 				length / secsize,
1846 				/*data_ptr*/(u_int8_t *) virtual,
1847 				/*dxfer_len*/length,
1848 				/*sense_len*/SSD_FULL_SIZE,
1849 				da_default_timeout * 1000);
1850 		error = cam_periph_runccb((union ccb *)&csio, cam_periph_error,
1851 		    0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1852 		if (error != 0)
1853 			printf("Aborting dump due to I/O error.\n");
1854 		return (error);
1855 	}
1856 
1857 	/*
1858 	 * Sync the disk cache contents to the physical media.
1859 	 */
1860 	if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
1861 
1862 		xpt_setup_ccb(&csio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1863 		csio.ccb_h.ccb_state = DA_CCB_DUMP;
1864 		scsi_synchronize_cache(&csio,
1865 				       /*retries*/0,
1866 				       /*cbfcnp*/NULL,
1867 				       MSG_SIMPLE_Q_TAG,
1868 				       /*begin_lba*/0,/* Cover the whole disk */
1869 				       /*lb_count*/0,
1870 				       SSD_FULL_SIZE,
1871 				       5 * 1000);
1872 		error = cam_periph_runccb((union ccb *)&csio, cam_periph_error,
1873 		    0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1874 		if (error != 0)
1875 			xpt_print(periph->path, "Synchronize cache failed\n");
1876 	}
1877 	return (error);
1878 }
1879 
1880 static int
1881 dagetattr(struct bio *bp)
1882 {
1883 	int ret;
1884 	struct cam_periph *periph;
1885 
1886 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1887 	cam_periph_lock(periph);
1888 	ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1889 	    periph->path);
1890 	cam_periph_unlock(periph);
1891 	if (ret == 0)
1892 		bp->bio_completed = bp->bio_length;
1893 	return ret;
1894 }
1895 
1896 static void
1897 dainit(void)
1898 {
1899 	cam_status status;
1900 
1901 	/*
1902 	 * Install a global async callback.  This callback will
1903 	 * receive async callbacks like "new device found".
1904 	 */
1905 	status = xpt_register_async(AC_FOUND_DEVICE, daasync, NULL, NULL);
1906 
1907 	if (status != CAM_REQ_CMP) {
1908 		printf("da: Failed to attach master async callback "
1909 		       "due to status 0x%x!\n", status);
1910 	} else if (da_send_ordered) {
1911 
1912 		/* Register our shutdown event handler */
1913 		if ((EVENTHANDLER_REGISTER(shutdown_post_sync, dashutdown,
1914 					   NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1915 		    printf("dainit: shutdown event registration failed!\n");
1916 	}
1917 }
1918 
1919 /*
1920  * Callback from GEOM, called when it has finished cleaning up its
1921  * resources.
1922  */
1923 static void
1924 dadiskgonecb(struct disk *dp)
1925 {
1926 	struct cam_periph *periph;
1927 
1928 	periph = (struct cam_periph *)dp->d_drv1;
1929 	da_periph_release(periph, DA_REF_GEOM);
1930 }
1931 
1932 static void
1933 daoninvalidate(struct cam_periph *periph)
1934 {
1935 	struct da_softc *softc;
1936 
1937 	cam_periph_assert(periph, MA_OWNED);
1938 	softc = (struct da_softc *)periph->softc;
1939 
1940 	/*
1941 	 * De-register any async callbacks.
1942 	 */
1943 	xpt_register_async(0, daasync, periph, periph->path);
1944 
1945 	softc->flags |= DA_FLAG_PACK_INVALID;
1946 #ifdef CAM_IO_STATS
1947 	softc->invalidations++;
1948 #endif
1949 
1950 	/*
1951 	 * Return all queued I/O with ENXIO.
1952 	 * XXX Handle any transactions queued to the card
1953 	 *     with XPT_ABORT_CCB.
1954 	 */
1955 	cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1956 
1957 	/*
1958 	 * Tell GEOM that we've gone away, we'll get a callback when it is
1959 	 * done cleaning up its resources.
1960 	 */
1961 	disk_gone(softc->disk);
1962 }
1963 
1964 static void
1965 dacleanup(struct cam_periph *periph)
1966 {
1967 	struct da_softc *softc;
1968 
1969 	softc = (struct da_softc *)periph->softc;
1970 
1971 	cam_periph_unlock(periph);
1972 
1973 	cam_iosched_fini(softc->cam_iosched);
1974 
1975 	/*
1976 	 * If we can't free the sysctl tree, oh well...
1977 	 */
1978 	if ((softc->flags & DA_FLAG_SCTX_INIT) != 0) {
1979 #ifdef CAM_IO_STATS
1980 		if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1981 			xpt_print(periph->path,
1982 			    "can't remove sysctl stats context\n");
1983 #endif
1984 		if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1985 			xpt_print(periph->path,
1986 			    "can't remove sysctl context\n");
1987 	}
1988 
1989 	callout_drain(&softc->mediapoll_c);
1990 	disk_destroy(softc->disk);
1991 	callout_drain(&softc->sendordered_c);
1992 	free(softc, M_DEVBUF);
1993 	cam_periph_lock(periph);
1994 }
1995 
1996 static void
1997 daasync(void *callback_arg, u_int32_t code,
1998 	struct cam_path *path, void *arg)
1999 {
2000 	struct cam_periph *periph;
2001 	struct da_softc *softc;
2002 
2003 	periph = (struct cam_periph *)callback_arg;
2004 	switch (code) {
2005 	case AC_FOUND_DEVICE:
2006 	{
2007 		struct ccb_getdev *cgd;
2008 		cam_status status;
2009 
2010 		cgd = (struct ccb_getdev *)arg;
2011 		if (cgd == NULL)
2012 			break;
2013 
2014 		if (cgd->protocol != PROTO_SCSI)
2015 			break;
2016 		if (SID_QUAL(&cgd->inq_data) != SID_QUAL_LU_CONNECTED)
2017 			break;
2018 		if (SID_TYPE(&cgd->inq_data) != T_DIRECT
2019 		    && SID_TYPE(&cgd->inq_data) != T_RBC
2020 		    && SID_TYPE(&cgd->inq_data) != T_OPTICAL
2021 		    && SID_TYPE(&cgd->inq_data) != T_ZBC_HM)
2022 			break;
2023 
2024 		/*
2025 		 * Allocate a peripheral instance for
2026 		 * this device and start the probe
2027 		 * process.
2028 		 */
2029 		status = cam_periph_alloc(daregister, daoninvalidate,
2030 					  dacleanup, dastart,
2031 					  "da", CAM_PERIPH_BIO,
2032 					  path, daasync,
2033 					  AC_FOUND_DEVICE, cgd);
2034 
2035 		if (status != CAM_REQ_CMP
2036 		 && status != CAM_REQ_INPROG)
2037 			printf("daasync: Unable to attach to new device "
2038 				"due to status 0x%x\n", status);
2039 		return;
2040 	}
2041 	case AC_ADVINFO_CHANGED:
2042 	{
2043 		uintptr_t buftype;
2044 
2045 		buftype = (uintptr_t)arg;
2046 		if (buftype == CDAI_TYPE_PHYS_PATH) {
2047 			struct da_softc *softc;
2048 
2049 			softc = periph->softc;
2050 			disk_attr_changed(softc->disk, "GEOM::physpath",
2051 					  M_NOWAIT);
2052 		}
2053 		break;
2054 	}
2055 	case AC_UNIT_ATTENTION:
2056 	{
2057 		union ccb *ccb;
2058 		int error_code, sense_key, asc, ascq;
2059 
2060 		softc = (struct da_softc *)periph->softc;
2061 		ccb = (union ccb *)arg;
2062 
2063 		/*
2064 		 * Handle all UNIT ATTENTIONs except our own,
2065 		 * as they will be handled by daerror().
2066 		 */
2067 		if (xpt_path_periph(ccb->ccb_h.path) != periph &&
2068 		    scsi_extract_sense_ccb(ccb,
2069 		     &error_code, &sense_key, &asc, &ascq)) {
2070 			if (asc == 0x2A && ascq == 0x09) {
2071 				xpt_print(ccb->ccb_h.path,
2072 				    "Capacity data has changed\n");
2073 				cam_periph_lock(periph);
2074 				softc->flags &= ~DA_FLAG_PROBED;
2075 				cam_periph_unlock(periph);
2076 				dareprobe(periph);
2077 			} else if (asc == 0x28 && ascq == 0x00) {
2078 				cam_periph_lock(periph);
2079 				softc->flags &= ~DA_FLAG_PROBED;
2080 				cam_periph_unlock(periph);
2081 				disk_media_changed(softc->disk, M_NOWAIT);
2082 			} else if (asc == 0x3F && ascq == 0x03) {
2083 				xpt_print(ccb->ccb_h.path,
2084 				    "INQUIRY data has changed\n");
2085 				cam_periph_lock(periph);
2086 				softc->flags &= ~DA_FLAG_PROBED;
2087 				cam_periph_unlock(periph);
2088 				dareprobe(periph);
2089 			}
2090 		}
2091 		break;
2092 	}
2093 	case AC_SCSI_AEN:
2094 		softc = (struct da_softc *)periph->softc;
2095 		cam_periph_lock(periph);
2096 		if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) &&
2097 		    (softc->flags & DA_FLAG_TUR_PENDING) == 0) {
2098 			if (da_periph_acquire(periph, DA_REF_TUR) == 0) {
2099 				cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR);
2100 				daschedule(periph);
2101 			}
2102 		}
2103 		cam_periph_unlock(periph);
2104 		/* FALLTHROUGH */
2105 	case AC_SENT_BDR:
2106 	case AC_BUS_RESET:
2107 	{
2108 		struct ccb_hdr *ccbh;
2109 
2110 		softc = (struct da_softc *)periph->softc;
2111 		/*
2112 		 * Don't fail on the expected unit attention
2113 		 * that will occur.
2114 		 */
2115 		cam_periph_lock(periph);
2116 		softc->flags |= DA_FLAG_RETRY_UA;
2117 		LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le)
2118 			ccbh->ccb_state |= DA_CCB_RETRY_UA;
2119 		cam_periph_unlock(periph);
2120 		break;
2121 	}
2122 	case AC_INQ_CHANGED:
2123 		cam_periph_lock(periph);
2124 		softc = (struct da_softc *)periph->softc;
2125 		softc->flags &= ~DA_FLAG_PROBED;
2126 		dareprobe(periph);
2127 		cam_periph_unlock(periph);
2128 		break;
2129 	default:
2130 		break;
2131 	}
2132 	cam_periph_async(periph, code, path, arg);
2133 }
2134 
2135 static void
2136 dasysctlinit(void *context, int pending)
2137 {
2138 	struct cam_periph *periph;
2139 	struct da_softc *softc;
2140 	char tmpstr[32], tmpstr2[16];
2141 	struct ccb_trans_settings cts;
2142 
2143 	periph = (struct cam_periph *)context;
2144 	/*
2145 	 * periph was held for us when this task was enqueued
2146 	 */
2147 	if (periph->flags & CAM_PERIPH_INVALID) {
2148 		da_periph_release(periph, DA_REF_SYSCTL);
2149 		return;
2150 	}
2151 
2152 	softc = (struct da_softc *)periph->softc;
2153 	snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number);
2154 	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
2155 
2156 	sysctl_ctx_init(&softc->sysctl_ctx);
2157 	cam_periph_lock(periph);
2158 	softc->flags |= DA_FLAG_SCTX_INIT;
2159 	cam_periph_unlock(periph);
2160 	softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
2161 		SYSCTL_STATIC_CHILDREN(_kern_cam_da), OID_AUTO, tmpstr2,
2162 		CTLFLAG_RD, 0, tmpstr, "device_index");
2163 	if (softc->sysctl_tree == NULL) {
2164 		printf("dasysctlinit: unable to allocate sysctl tree\n");
2165 		da_periph_release(periph, DA_REF_SYSCTL);
2166 		return;
2167 	}
2168 
2169 	/*
2170 	 * Now register the sysctl handler, so the user can change the value on
2171 	 * the fly.
2172 	 */
2173 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2174 		OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RWTUN,
2175 		softc, 0, dadeletemethodsysctl, "A",
2176 		"BIO_DELETE execution method");
2177 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2178 		OID_AUTO, "delete_max", CTLTYPE_U64 | CTLFLAG_RW,
2179 		softc, 0, dadeletemaxsysctl, "Q",
2180 		"Maximum BIO_DELETE size");
2181 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2182 		OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
2183 		&softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
2184 		"Minimum CDB size");
2185 
2186 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2187 		OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
2188 		softc, 0, dazonemodesysctl, "A",
2189 		"Zone Mode");
2190 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2191 		OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
2192 		softc, 0, dazonesupsysctl, "A",
2193 		"Zone Support");
2194 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2195 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2196 		"optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
2197 		"Optimal Number of Open Sequential Write Preferred Zones");
2198 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2199 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2200 		"optimal_nonseq_zones", CTLFLAG_RD,
2201 		&softc->optimal_nonseq_zones,
2202 		"Optimal Number of Non-Sequentially Written Sequential Write "
2203 		"Preferred Zones");
2204 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2205 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
2206 		"max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
2207 		"Maximum Number of Open Sequential Write Required Zones");
2208 
2209 	SYSCTL_ADD_INT(&softc->sysctl_ctx,
2210 		       SYSCTL_CHILDREN(softc->sysctl_tree),
2211 		       OID_AUTO,
2212 		       "error_inject",
2213 		       CTLFLAG_RW,
2214 		       &softc->error_inject,
2215 		       0,
2216 		       "error_inject leaf");
2217 
2218 	SYSCTL_ADD_INT(&softc->sysctl_ctx,
2219 		       SYSCTL_CHILDREN(softc->sysctl_tree),
2220 		       OID_AUTO,
2221 		       "unmapped_io",
2222 		       CTLFLAG_RD,
2223 		       &softc->unmappedio,
2224 		       0,
2225 		       "Unmapped I/O leaf");
2226 
2227 	SYSCTL_ADD_INT(&softc->sysctl_ctx,
2228 		       SYSCTL_CHILDREN(softc->sysctl_tree),
2229 		       OID_AUTO,
2230 		       "rotating",
2231 		       CTLFLAG_RD,
2232 		       &softc->rotating,
2233 		       0,
2234 		       "Rotating media");
2235 
2236 #ifdef CAM_TEST_FAILURE
2237 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
2238 		OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
2239 		periph, 0, cam_periph_invalidate_sysctl, "I",
2240 		"Write 1 to invalidate the drive immediately");
2241 #endif
2242 
2243 	/*
2244 	 * Add some addressing info.
2245 	 */
2246 	memset(&cts, 0, sizeof (cts));
2247 	xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE);
2248 	cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
2249 	cts.type = CTS_TYPE_CURRENT_SETTINGS;
2250 	cam_periph_lock(periph);
2251 	xpt_action((union ccb *)&cts);
2252 	cam_periph_unlock(periph);
2253 	if (cts.ccb_h.status != CAM_REQ_CMP) {
2254 		da_periph_release(periph, DA_REF_SYSCTL);
2255 		return;
2256 	}
2257 	if (cts.protocol == PROTO_SCSI && cts.transport == XPORT_FC) {
2258 		struct ccb_trans_settings_fc *fc = &cts.xport_specific.fc;
2259 		if (fc->valid & CTS_FC_VALID_WWPN) {
2260 			softc->wwpn = fc->wwpn;
2261 			SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
2262 			    SYSCTL_CHILDREN(softc->sysctl_tree),
2263 			    OID_AUTO, "wwpn", CTLFLAG_RD,
2264 			    &softc->wwpn, "World Wide Port Name");
2265 		}
2266 	}
2267 
2268 #ifdef CAM_IO_STATS
2269 	/*
2270 	 * Now add some useful stats.
2271 	 * XXX These should live in cam_periph and be common to all periphs
2272 	 */
2273 	softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
2274 	    SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
2275 	    CTLFLAG_RD, 0, "Statistics");
2276 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2277 		       SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2278 		       OID_AUTO,
2279 		       "errors",
2280 		       CTLFLAG_RD,
2281 		       &softc->errors,
2282 		       0,
2283 		       "Transport errors reported by the SIM");
2284 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2285 		       SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2286 		       OID_AUTO,
2287 		       "timeouts",
2288 		       CTLFLAG_RD,
2289 		       &softc->timeouts,
2290 		       0,
2291 		       "Device timeouts reported by the SIM");
2292 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
2293 		       SYSCTL_CHILDREN(softc->sysctl_stats_tree),
2294 		       OID_AUTO,
2295 		       "pack_invalidations",
2296 		       CTLFLAG_RD,
2297 		       &softc->invalidations,
2298 		       0,
2299 		       "Device pack invalidations");
2300 #endif
2301 
2302 	cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
2303 	    softc->sysctl_tree);
2304 
2305 	da_periph_release(periph, DA_REF_SYSCTL);
2306 }
2307 
2308 static int
2309 dadeletemaxsysctl(SYSCTL_HANDLER_ARGS)
2310 {
2311 	int error;
2312 	uint64_t value;
2313 	struct da_softc *softc;
2314 
2315 	softc = (struct da_softc *)arg1;
2316 
2317 	value = softc->disk->d_delmaxsize;
2318 	error = sysctl_handle_64(oidp, &value, 0, req);
2319 	if ((error != 0) || (req->newptr == NULL))
2320 		return (error);
2321 
2322 	/* only accept values smaller than the calculated value */
2323 	if (value > dadeletemaxsize(softc, softc->delete_method)) {
2324 		return (EINVAL);
2325 	}
2326 	softc->disk->d_delmaxsize = value;
2327 
2328 	return (0);
2329 }
2330 
2331 static int
2332 dacmdsizesysctl(SYSCTL_HANDLER_ARGS)
2333 {
2334 	int error, value;
2335 
2336 	value = *(int *)arg1;
2337 
2338 	error = sysctl_handle_int(oidp, &value, 0, req);
2339 
2340 	if ((error != 0)
2341 	 || (req->newptr == NULL))
2342 		return (error);
2343 
2344 	/*
2345 	 * Acceptable values here are 6, 10, 12 or 16.
2346 	 */
2347 	if (value < 6)
2348 		value = 6;
2349 	else if ((value > 6)
2350 	      && (value <= 10))
2351 		value = 10;
2352 	else if ((value > 10)
2353 	      && (value <= 12))
2354 		value = 12;
2355 	else if (value > 12)
2356 		value = 16;
2357 
2358 	*(int *)arg1 = value;
2359 
2360 	return (0);
2361 }
2362 
2363 static int
2364 dasysctlsofttimeout(SYSCTL_HANDLER_ARGS)
2365 {
2366 	sbintime_t value;
2367 	int error;
2368 
2369 	value = da_default_softtimeout / SBT_1MS;
2370 
2371 	error = sysctl_handle_int(oidp, (int *)&value, 0, req);
2372 	if ((error != 0) || (req->newptr == NULL))
2373 		return (error);
2374 
2375 	/* XXX Should clip this to a reasonable level */
2376 	if (value > da_default_timeout * 1000)
2377 		return (EINVAL);
2378 
2379 	da_default_softtimeout = value * SBT_1MS;
2380 	return (0);
2381 }
2382 
2383 static void
2384 dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method)
2385 {
2386 
2387 	softc->delete_method = delete_method;
2388 	softc->disk->d_delmaxsize = dadeletemaxsize(softc, delete_method);
2389 	softc->delete_func = da_delete_functions[delete_method];
2390 
2391 	if (softc->delete_method > DA_DELETE_DISABLE)
2392 		softc->disk->d_flags |= DISKFLAG_CANDELETE;
2393 	else
2394 		softc->disk->d_flags &= ~DISKFLAG_CANDELETE;
2395 }
2396 
2397 static off_t
2398 dadeletemaxsize(struct da_softc *softc, da_delete_methods delete_method)
2399 {
2400 	off_t sectors;
2401 
2402 	switch(delete_method) {
2403 	case DA_DELETE_UNMAP:
2404 		sectors = (off_t)softc->unmap_max_lba;
2405 		break;
2406 	case DA_DELETE_ATA_TRIM:
2407 		sectors = (off_t)ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
2408 		break;
2409 	case DA_DELETE_WS16:
2410 		sectors = omin(softc->ws_max_blks, WS16_MAX_BLKS);
2411 		break;
2412 	case DA_DELETE_ZERO:
2413 	case DA_DELETE_WS10:
2414 		sectors = omin(softc->ws_max_blks, WS10_MAX_BLKS);
2415 		break;
2416 	default:
2417 		return 0;
2418 	}
2419 
2420 	return (off_t)softc->params.secsize *
2421 	    omin(sectors, softc->params.sectors);
2422 }
2423 
2424 static void
2425 daprobedone(struct cam_periph *periph, union ccb *ccb)
2426 {
2427 	struct da_softc *softc;
2428 
2429 	softc = (struct da_softc *)periph->softc;
2430 
2431 	dadeletemethodchoose(softc, DA_DELETE_NONE);
2432 
2433 	if (bootverbose && (softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2434 		char buf[80];
2435 		int i, sep;
2436 
2437 		snprintf(buf, sizeof(buf), "Delete methods: <");
2438 		sep = 0;
2439 		for (i = 0; i <= DA_DELETE_MAX; i++) {
2440 			if ((softc->delete_available & (1 << i)) == 0 &&
2441 			    i != softc->delete_method)
2442 				continue;
2443 			if (sep)
2444 				strlcat(buf, ",", sizeof(buf));
2445 			strlcat(buf, da_delete_method_names[i],
2446 			    sizeof(buf));
2447 			if (i == softc->delete_method)
2448 				strlcat(buf, "(*)", sizeof(buf));
2449 			sep = 1;
2450 		}
2451 		strlcat(buf, ">", sizeof(buf));
2452 		printf("%s%d: %s\n", periph->periph_name,
2453 		    periph->unit_number, buf);
2454 	}
2455 
2456 	/*
2457 	 * Since our peripheral may be invalidated by an error
2458 	 * above or an external event, we must release our CCB
2459 	 * before releasing the probe lock on the peripheral.
2460 	 * The peripheral will only go away once the last lock
2461 	 * is removed, and we need it around for the CCB release
2462 	 * operation.
2463 	 */
2464 	xpt_release_ccb(ccb);
2465 	softc->state = DA_STATE_NORMAL;
2466 	softc->flags |= DA_FLAG_PROBED;
2467 	daschedule(periph);
2468 	wakeup(&softc->disk->d_mediasize);
2469 	if ((softc->flags & DA_FLAG_ANNOUNCED) == 0) {
2470 		softc->flags |= DA_FLAG_ANNOUNCED;
2471 		da_periph_unhold(periph, DA_REF_PROBE_HOLD);
2472 	} else
2473 		da_periph_release_locked(periph, DA_REF_REPROBE);
2474 }
2475 
2476 static void
2477 dadeletemethodchoose(struct da_softc *softc, da_delete_methods default_method)
2478 {
2479 	int i, methods;
2480 
2481 	/* If available, prefer the method requested by user. */
2482 	i = softc->delete_method_pref;
2483 	methods = softc->delete_available | (1 << DA_DELETE_DISABLE);
2484 	if (methods & (1 << i)) {
2485 		dadeletemethodset(softc, i);
2486 		return;
2487 	}
2488 
2489 	/* Use the pre-defined order to choose the best performing delete. */
2490 	for (i = DA_DELETE_MIN; i <= DA_DELETE_MAX; i++) {
2491 		if (i == DA_DELETE_ZERO)
2492 			continue;
2493 		if (softc->delete_available & (1 << i)) {
2494 			dadeletemethodset(softc, i);
2495 			return;
2496 		}
2497 	}
2498 
2499 	/* Fallback to default. */
2500 	dadeletemethodset(softc, default_method);
2501 }
2502 
2503 static int
2504 dadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
2505 {
2506 	char buf[16];
2507 	const char *p;
2508 	struct da_softc *softc;
2509 	int i, error, value;
2510 
2511 	softc = (struct da_softc *)arg1;
2512 
2513 	value = softc->delete_method;
2514 	if (value < 0 || value > DA_DELETE_MAX)
2515 		p = "UNKNOWN";
2516 	else
2517 		p = da_delete_method_names[value];
2518 	strncpy(buf, p, sizeof(buf));
2519 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
2520 	if (error != 0 || req->newptr == NULL)
2521 		return (error);
2522 	for (i = 0; i <= DA_DELETE_MAX; i++) {
2523 		if (strcmp(buf, da_delete_method_names[i]) == 0)
2524 			break;
2525 	}
2526 	if (i > DA_DELETE_MAX)
2527 		return (EINVAL);
2528 	softc->delete_method_pref = i;
2529 	dadeletemethodchoose(softc, DA_DELETE_NONE);
2530 	return (0);
2531 }
2532 
2533 static int
2534 dazonemodesysctl(SYSCTL_HANDLER_ARGS)
2535 {
2536 	char tmpbuf[40];
2537 	struct da_softc *softc;
2538 	int error;
2539 
2540 	softc = (struct da_softc *)arg1;
2541 
2542 	switch (softc->zone_mode) {
2543 	case DA_ZONE_DRIVE_MANAGED:
2544 		snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
2545 		break;
2546 	case DA_ZONE_HOST_AWARE:
2547 		snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
2548 		break;
2549 	case DA_ZONE_HOST_MANAGED:
2550 		snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
2551 		break;
2552 	case DA_ZONE_NONE:
2553 	default:
2554 		snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
2555 		break;
2556 	}
2557 
2558 	error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
2559 
2560 	return (error);
2561 }
2562 
2563 static int
2564 dazonesupsysctl(SYSCTL_HANDLER_ARGS)
2565 {
2566 	char tmpbuf[180];
2567 	struct da_softc *softc;
2568 	struct sbuf sb;
2569 	int error, first;
2570 	unsigned int i;
2571 
2572 	softc = (struct da_softc *)arg1;
2573 
2574 	error = 0;
2575 	first = 1;
2576 	sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
2577 
2578 	for (i = 0; i < sizeof(da_zone_desc_table) /
2579 	     sizeof(da_zone_desc_table[0]); i++) {
2580 		if (softc->zone_flags & da_zone_desc_table[i].value) {
2581 			if (first == 0)
2582 				sbuf_printf(&sb, ", ");
2583 			else
2584 				first = 0;
2585 			sbuf_cat(&sb, da_zone_desc_table[i].desc);
2586 		}
2587 	}
2588 
2589 	if (first == 1)
2590 		sbuf_printf(&sb, "None");
2591 
2592 	sbuf_finish(&sb);
2593 
2594 	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
2595 
2596 	return (error);
2597 }
2598 
2599 static cam_status
2600 daregister(struct cam_periph *periph, void *arg)
2601 {
2602 	struct da_softc *softc;
2603 	struct ccb_pathinq cpi;
2604 	struct ccb_getdev *cgd;
2605 	char tmpstr[80];
2606 	caddr_t match;
2607 
2608 	cgd = (struct ccb_getdev *)arg;
2609 	if (cgd == NULL) {
2610 		printf("daregister: no getdev CCB, can't register device\n");
2611 		return(CAM_REQ_CMP_ERR);
2612 	}
2613 
2614 	softc = (struct da_softc *)malloc(sizeof(*softc), M_DEVBUF,
2615 	    M_NOWAIT|M_ZERO);
2616 
2617 	if (softc == NULL) {
2618 		printf("daregister: Unable to probe new device. "
2619 		       "Unable to allocate softc\n");
2620 		return(CAM_REQ_CMP_ERR);
2621 	}
2622 
2623 	if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
2624 		printf("daregister: Unable to probe new device. "
2625 		       "Unable to allocate iosched memory\n");
2626 		free(softc, M_DEVBUF);
2627 		return(CAM_REQ_CMP_ERR);
2628 	}
2629 
2630 	LIST_INIT(&softc->pending_ccbs);
2631 	softc->state = DA_STATE_PROBE_WP;
2632 	bioq_init(&softc->delete_run_queue);
2633 	if (SID_IS_REMOVABLE(&cgd->inq_data))
2634 		softc->flags |= DA_FLAG_PACK_REMOVABLE;
2635 	softc->unmap_max_ranges = UNMAP_MAX_RANGES;
2636 	softc->unmap_max_lba = UNMAP_RANGE_MAX;
2637 	softc->unmap_gran = 0;
2638 	softc->unmap_gran_align = 0;
2639 	softc->ws_max_blks = WS16_MAX_BLKS;
2640 	softc->trim_max_ranges = ATA_TRIM_MAX_RANGES;
2641 	softc->rotating = 1;
2642 
2643 	periph->softc = softc;
2644 
2645 	/*
2646 	 * See if this device has any quirks.
2647 	 */
2648 	match = cam_quirkmatch((caddr_t)&cgd->inq_data,
2649 			       (caddr_t)da_quirk_table,
2650 			       nitems(da_quirk_table),
2651 			       sizeof(*da_quirk_table), scsi_inquiry_match);
2652 
2653 	if (match != NULL)
2654 		softc->quirks = ((struct da_quirk_entry *)match)->quirks;
2655 	else
2656 		softc->quirks = DA_Q_NONE;
2657 
2658 	/* Check if the SIM does not want 6 byte commands */
2659 	xpt_path_inq(&cpi, periph->path);
2660 	if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_misc & PIM_NO_6_BYTE))
2661 		softc->quirks |= DA_Q_NO_6_BYTE;
2662 
2663 	if (SID_TYPE(&cgd->inq_data) == T_ZBC_HM)
2664 		softc->zone_mode = DA_ZONE_HOST_MANAGED;
2665 	else if (softc->quirks & DA_Q_SMR_DM)
2666 		softc->zone_mode = DA_ZONE_DRIVE_MANAGED;
2667 	else
2668 		softc->zone_mode = DA_ZONE_NONE;
2669 
2670 	if (softc->zone_mode != DA_ZONE_NONE) {
2671 		if (scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) {
2672 			if (scsi_vpd_supported_page(periph, SVPD_ZONED_BDC))
2673 				softc->zone_interface = DA_ZONE_IF_ATA_SAT;
2674 			else
2675 				softc->zone_interface = DA_ZONE_IF_ATA_PASS;
2676 		} else
2677 			softc->zone_interface = DA_ZONE_IF_SCSI;
2678 	}
2679 
2680 	TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
2681 
2682 	/*
2683 	 * Take an exclusive refcount on the periph while dastart is called
2684 	 * to finish the probe.  The reference will be dropped in dadone at
2685 	 * the end of probe.
2686 	 *
2687 	 * XXX if cam_periph_hold returns an error, we don't hold a refcount.
2688 	 */
2689 	(void)da_periph_hold(periph, PRIBIO, DA_REF_PROBE_HOLD);
2690 
2691 	/*
2692 	 * Schedule a periodic event to occasionally send an
2693 	 * ordered tag to a device.
2694 	 */
2695 	callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
2696 	callout_reset(&softc->sendordered_c,
2697 	    (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL,
2698 	    dasendorderedtag, periph);
2699 
2700 	cam_periph_unlock(periph);
2701 	/*
2702 	 * RBC devices don't have to support READ(6), only READ(10).
2703 	 */
2704 	if (softc->quirks & DA_Q_NO_6_BYTE || SID_TYPE(&cgd->inq_data) == T_RBC)
2705 		softc->minimum_cmd_size = 10;
2706 	else
2707 		softc->minimum_cmd_size = 6;
2708 
2709 	/*
2710 	 * Load the user's default, if any.
2711 	 */
2712 	snprintf(tmpstr, sizeof(tmpstr), "kern.cam.da.%d.minimum_cmd_size",
2713 		 periph->unit_number);
2714 	TUNABLE_INT_FETCH(tmpstr, &softc->minimum_cmd_size);
2715 
2716 	/*
2717 	 * 6, 10, 12 and 16 are the currently permissible values.
2718 	 */
2719 	if (softc->minimum_cmd_size > 12)
2720 		softc->minimum_cmd_size = 16;
2721 	else if (softc->minimum_cmd_size > 10)
2722 		softc->minimum_cmd_size = 12;
2723 	else if (softc->minimum_cmd_size > 6)
2724 		softc->minimum_cmd_size = 10;
2725 	else
2726 		softc->minimum_cmd_size = 6;
2727 
2728 	/* Predict whether device may support READ CAPACITY(16). */
2729 	if (SID_ANSI_REV(&cgd->inq_data) >= SCSI_REV_SPC3 &&
2730 	    (softc->quirks & DA_Q_NO_RC16) == 0) {
2731 		softc->flags |= DA_FLAG_CAN_RC16;
2732 	}
2733 
2734 	/*
2735 	 * Register this media as a disk.
2736 	 */
2737 	softc->disk = disk_alloc();
2738 	softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
2739 			  periph->unit_number, 0,
2740 			  DEVSTAT_BS_UNAVAILABLE,
2741 			  SID_TYPE(&cgd->inq_data) |
2742 			  XPORT_DEVSTAT_TYPE(cpi.transport),
2743 			  DEVSTAT_PRIORITY_DISK);
2744 	softc->disk->d_open = daopen;
2745 	softc->disk->d_close = daclose;
2746 	softc->disk->d_strategy = dastrategy;
2747 	softc->disk->d_dump = dadump;
2748 	softc->disk->d_getattr = dagetattr;
2749 	softc->disk->d_gone = dadiskgonecb;
2750 	softc->disk->d_name = "da";
2751 	softc->disk->d_drv1 = periph;
2752 	if (cpi.maxio == 0)
2753 		softc->maxio = DFLTPHYS;	/* traditional default */
2754 	else if (cpi.maxio > MAXPHYS)
2755 		softc->maxio = MAXPHYS;		/* for safety */
2756 	else
2757 		softc->maxio = cpi.maxio;
2758 	softc->disk->d_maxsize = softc->maxio;
2759 	softc->disk->d_unit = periph->unit_number;
2760 	softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
2761 	if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0)
2762 		softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
2763 	if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
2764 		softc->unmappedio = 1;
2765 		softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
2766 	}
2767 	cam_strvis(softc->disk->d_descr, cgd->inq_data.vendor,
2768 	    sizeof(cgd->inq_data.vendor), sizeof(softc->disk->d_descr));
2769 	strlcat(softc->disk->d_descr, " ", sizeof(softc->disk->d_descr));
2770 	cam_strvis(&softc->disk->d_descr[strlen(softc->disk->d_descr)],
2771 	    cgd->inq_data.product, sizeof(cgd->inq_data.product),
2772 	    sizeof(softc->disk->d_descr) - strlen(softc->disk->d_descr));
2773 	softc->disk->d_hba_vendor = cpi.hba_vendor;
2774 	softc->disk->d_hba_device = cpi.hba_device;
2775 	softc->disk->d_hba_subvendor = cpi.hba_subvendor;
2776 	softc->disk->d_hba_subdevice = cpi.hba_subdevice;
2777 
2778 	/*
2779 	 * Acquire a reference to the periph before we register with GEOM.
2780 	 * We'll release this reference once GEOM calls us back (via
2781 	 * dadiskgonecb()) telling us that our provider has been freed.
2782 	 */
2783 	if (da_periph_acquire(periph, DA_REF_GEOM) != 0) {
2784 		xpt_print(periph->path, "%s: lost periph during "
2785 			  "registration!\n", __func__);
2786 		cam_periph_lock(periph);
2787 		return (CAM_REQ_CMP_ERR);
2788 	}
2789 
2790 	disk_create(softc->disk, DISK_VERSION);
2791 	cam_periph_lock(periph);
2792 
2793 	/*
2794 	 * Add async callbacks for events of interest.
2795 	 * I don't bother checking if this fails as,
2796 	 * in most cases, the system will function just
2797 	 * fine without them and the only alternative
2798 	 * would be to not attach the device on failure.
2799 	 */
2800 	xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
2801 	    AC_ADVINFO_CHANGED | AC_SCSI_AEN | AC_UNIT_ATTENTION |
2802 	    AC_INQ_CHANGED, daasync, periph, periph->path);
2803 
2804 	/*
2805 	 * Emit an attribute changed notification just in case
2806 	 * physical path information arrived before our async
2807 	 * event handler was registered, but after anyone attaching
2808 	 * to our disk device polled it.
2809 	 */
2810 	disk_attr_changed(softc->disk, "GEOM::physpath", M_NOWAIT);
2811 
2812 	/*
2813 	 * Schedule a periodic media polling events.
2814 	 */
2815 	callout_init_mtx(&softc->mediapoll_c, cam_periph_mtx(periph), 0);
2816 	if ((softc->flags & DA_FLAG_PACK_REMOVABLE) &&
2817 	    (cgd->inq_flags & SID_AEN) == 0 &&
2818 	    da_poll_period != 0)
2819 		callout_reset(&softc->mediapoll_c, da_poll_period * hz,
2820 		    damediapoll, periph);
2821 
2822 	xpt_schedule(periph, CAM_PRIORITY_DEV);
2823 
2824 	return(CAM_REQ_CMP);
2825 }
2826 
2827 static int
2828 da_zone_bio_to_scsi(int disk_zone_cmd)
2829 {
2830 	switch (disk_zone_cmd) {
2831 	case DISK_ZONE_OPEN:
2832 		return ZBC_OUT_SA_OPEN;
2833 	case DISK_ZONE_CLOSE:
2834 		return ZBC_OUT_SA_CLOSE;
2835 	case DISK_ZONE_FINISH:
2836 		return ZBC_OUT_SA_FINISH;
2837 	case DISK_ZONE_RWP:
2838 		return ZBC_OUT_SA_RWP;
2839 	}
2840 
2841 	return -1;
2842 }
2843 
2844 static int
2845 da_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2846 	    int *queue_ccb)
2847 {
2848 	struct da_softc *softc;
2849 	int error;
2850 
2851 	error = 0;
2852 
2853 	if (bp->bio_cmd != BIO_ZONE) {
2854 		error = EINVAL;
2855 		goto bailout;
2856 	}
2857 
2858 	softc = periph->softc;
2859 
2860 	switch (bp->bio_zone.zone_cmd) {
2861 	case DISK_ZONE_OPEN:
2862 	case DISK_ZONE_CLOSE:
2863 	case DISK_ZONE_FINISH:
2864 	case DISK_ZONE_RWP: {
2865 		int zone_flags;
2866 		int zone_sa;
2867 		uint64_t lba;
2868 
2869 		zone_sa = da_zone_bio_to_scsi(bp->bio_zone.zone_cmd);
2870 		if (zone_sa == -1) {
2871 			xpt_print(periph->path, "Cannot translate zone "
2872 			    "cmd %#x to SCSI\n", bp->bio_zone.zone_cmd);
2873 			error = EINVAL;
2874 			goto bailout;
2875 		}
2876 
2877 		zone_flags = 0;
2878 		lba = bp->bio_zone.zone_params.rwp.id;
2879 
2880 		if (bp->bio_zone.zone_params.rwp.flags &
2881 		    DISK_ZONE_RWP_FLAG_ALL)
2882 			zone_flags |= ZBC_OUT_ALL;
2883 
2884 		if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) {
2885 			scsi_zbc_out(&ccb->csio,
2886 				     /*retries*/ da_retry_count,
2887 				     /*cbfcnp*/ dadone,
2888 				     /*tag_action*/ MSG_SIMPLE_Q_TAG,
2889 				     /*service_action*/ zone_sa,
2890 				     /*zone_id*/ lba,
2891 				     /*zone_flags*/ zone_flags,
2892 				     /*data_ptr*/ NULL,
2893 				     /*dxfer_len*/ 0,
2894 				     /*sense_len*/ SSD_FULL_SIZE,
2895 				     /*timeout*/ da_default_timeout * 1000);
2896 		} else {
2897 			/*
2898 			 * Note that in this case, even though we can
2899 			 * technically use NCQ, we don't bother for several
2900 			 * reasons:
2901 			 * 1. It hasn't been tested on a SAT layer that
2902 			 *    supports it.  This is new as of SAT-4.
2903 			 * 2. Even when there is a SAT layer that supports
2904 			 *    it, that SAT layer will also probably support
2905 			 *    ZBC -> ZAC translation, since they are both
2906 			 *    in the SAT-4 spec.
2907 			 * 3. Translation will likely be preferable to ATA
2908 			 *    passthrough.  LSI / Avago at least single
2909 			 *    steps ATA passthrough commands in the HBA,
2910 			 *    regardless of protocol, so unless that
2911 			 *    changes, there is a performance penalty for
2912 			 *    doing ATA passthrough no matter whether
2913 			 *    you're using NCQ/FPDMA, DMA or PIO.
2914 			 * 4. It requires a 32-byte CDB, which at least at
2915 			 *    this point in CAM requires a CDB pointer, which
2916 			 *    would require us to allocate an additional bit
2917 			 *    of storage separate from the CCB.
2918 			 */
2919 			error = scsi_ata_zac_mgmt_out(&ccb->csio,
2920 			    /*retries*/ da_retry_count,
2921 			    /*cbfcnp*/ dadone,
2922 			    /*tag_action*/ MSG_SIMPLE_Q_TAG,
2923 			    /*use_ncq*/ 0,
2924 			    /*zm_action*/ zone_sa,
2925 			    /*zone_id*/ lba,
2926 			    /*zone_flags*/ zone_flags,
2927 			    /*data_ptr*/ NULL,
2928 			    /*dxfer_len*/ 0,
2929 			    /*cdb_storage*/ NULL,
2930 			    /*cdb_storage_len*/ 0,
2931 			    /*sense_len*/ SSD_FULL_SIZE,
2932 			    /*timeout*/ da_default_timeout * 1000);
2933 			if (error != 0) {
2934 				error = EINVAL;
2935 				xpt_print(periph->path,
2936 				    "scsi_ata_zac_mgmt_out() returned an "
2937 				    "error!");
2938 				goto bailout;
2939 			}
2940 		}
2941 		*queue_ccb = 1;
2942 
2943 		break;
2944 	}
2945 	case DISK_ZONE_REPORT_ZONES: {
2946 		uint8_t *rz_ptr;
2947 		uint32_t num_entries, alloc_size;
2948 		struct disk_zone_report *rep;
2949 
2950 		rep = &bp->bio_zone.zone_params.report;
2951 
2952 		num_entries = rep->entries_allocated;
2953 		if (num_entries == 0) {
2954 			xpt_print(periph->path, "No entries allocated for "
2955 			    "Report Zones request\n");
2956 			error = EINVAL;
2957 			goto bailout;
2958 		}
2959 		alloc_size = sizeof(struct scsi_report_zones_hdr) +
2960 		    (sizeof(struct scsi_report_zones_desc) * num_entries);
2961 		alloc_size = min(alloc_size, softc->disk->d_maxsize);
2962 		rz_ptr = malloc(alloc_size, M_SCSIDA, M_NOWAIT | M_ZERO);
2963 		if (rz_ptr == NULL) {
2964 			xpt_print(periph->path, "Unable to allocate memory "
2965 			   "for Report Zones request\n");
2966 			error = ENOMEM;
2967 			goto bailout;
2968 		}
2969 
2970 		if (softc->zone_interface != DA_ZONE_IF_ATA_PASS) {
2971 			scsi_zbc_in(&ccb->csio,
2972 				    /*retries*/ da_retry_count,
2973 				    /*cbcfnp*/ dadone,
2974 				    /*tag_action*/ MSG_SIMPLE_Q_TAG,
2975 				    /*service_action*/ ZBC_IN_SA_REPORT_ZONES,
2976 				    /*zone_start_lba*/ rep->starting_id,
2977 				    /*zone_options*/ rep->rep_options,
2978 				    /*data_ptr*/ rz_ptr,
2979 				    /*dxfer_len*/ alloc_size,
2980 				    /*sense_len*/ SSD_FULL_SIZE,
2981 				    /*timeout*/ da_default_timeout * 1000);
2982 		} else {
2983 			/*
2984 			 * Note that in this case, even though we can
2985 			 * technically use NCQ, we don't bother for several
2986 			 * reasons:
2987 			 * 1. It hasn't been tested on a SAT layer that
2988 			 *    supports it.  This is new as of SAT-4.
2989 			 * 2. Even when there is a SAT layer that supports
2990 			 *    it, that SAT layer will also probably support
2991 			 *    ZBC -> ZAC translation, since they are both
2992 			 *    in the SAT-4 spec.
2993 			 * 3. Translation will likely be preferable to ATA
2994 			 *    passthrough.  LSI / Avago at least single
2995 			 *    steps ATA passthrough commands in the HBA,
2996 			 *    regardless of protocol, so unless that
2997 			 *    changes, there is a performance penalty for
2998 			 *    doing ATA passthrough no matter whether
2999 			 *    you're using NCQ/FPDMA, DMA or PIO.
3000 			 * 4. It requires a 32-byte CDB, which at least at
3001 			 *    this point in CAM requires a CDB pointer, which
3002 			 *    would require us to allocate an additional bit
3003 			 *    of storage separate from the CCB.
3004 			 */
3005 			error = scsi_ata_zac_mgmt_in(&ccb->csio,
3006 			    /*retries*/ da_retry_count,
3007 			    /*cbcfnp*/ dadone,
3008 			    /*tag_action*/ MSG_SIMPLE_Q_TAG,
3009 			    /*use_ncq*/ 0,
3010 			    /*zm_action*/ ATA_ZM_REPORT_ZONES,
3011 			    /*zone_id*/ rep->starting_id,
3012 			    /*zone_flags*/ rep->rep_options,
3013 			    /*data_ptr*/ rz_ptr,
3014 			    /*dxfer_len*/ alloc_size,
3015 			    /*cdb_storage*/ NULL,
3016 			    /*cdb_storage_len*/ 0,
3017 			    /*sense_len*/ SSD_FULL_SIZE,
3018 			    /*timeout*/ da_default_timeout * 1000);
3019 			if (error != 0) {
3020 				error = EINVAL;
3021 				xpt_print(periph->path,
3022 				    "scsi_ata_zac_mgmt_in() returned an "
3023 				    "error!");
3024 				goto bailout;
3025 			}
3026 		}
3027 
3028 		/*
3029 		 * For BIO_ZONE, this isn't normally needed.  However, it
3030 		 * is used by devstat_end_transaction_bio() to determine
3031 		 * how much data was transferred.
3032 		 */
3033 		/*
3034 		 * XXX KDM we have a problem.  But I'm not sure how to fix
3035 		 * it.  devstat uses bio_bcount - bio_resid to calculate
3036 		 * the amount of data transferred.   The GEOM disk code
3037 		 * uses bio_length - bio_resid to calculate the amount of
3038 		 * data in bio_completed.  We have different structure
3039 		 * sizes above and below the ada(4) driver.  So, if we
3040 		 * use the sizes above, the amount transferred won't be
3041 		 * quite accurate for devstat.  If we use different sizes
3042 		 * for bio_bcount and bio_length (above and below
3043 		 * respectively), then the residual needs to match one or
3044 		 * the other.  Everything is calculated after the bio
3045 		 * leaves the driver, so changing the values around isn't
3046 		 * really an option.  For now, just set the count to the
3047 		 * passed in length.  This means that the calculations
3048 		 * above (e.g. bio_completed) will be correct, but the
3049 		 * amount of data reported to devstat will be slightly
3050 		 * under or overstated.
3051 		 */
3052 		bp->bio_bcount = bp->bio_length;
3053 
3054 		*queue_ccb = 1;
3055 
3056 		break;
3057 	}
3058 	case DISK_ZONE_GET_PARAMS: {
3059 		struct disk_zone_disk_params *params;
3060 
3061 		params = &bp->bio_zone.zone_params.disk_params;
3062 		bzero(params, sizeof(*params));
3063 
3064 		switch (softc->zone_mode) {
3065 		case DA_ZONE_DRIVE_MANAGED:
3066 			params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
3067 			break;
3068 		case DA_ZONE_HOST_AWARE:
3069 			params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
3070 			break;
3071 		case DA_ZONE_HOST_MANAGED:
3072 			params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
3073 			break;
3074 		default:
3075 		case DA_ZONE_NONE:
3076 			params->zone_mode = DISK_ZONE_MODE_NONE;
3077 			break;
3078 		}
3079 
3080 		if (softc->zone_flags & DA_ZONE_FLAG_URSWRZ)
3081 			params->flags |= DISK_ZONE_DISK_URSWRZ;
3082 
3083 		if (softc->zone_flags & DA_ZONE_FLAG_OPT_SEQ_SET) {
3084 			params->optimal_seq_zones = softc->optimal_seq_zones;
3085 			params->flags |= DISK_ZONE_OPT_SEQ_SET;
3086 		}
3087 
3088 		if (softc->zone_flags & DA_ZONE_FLAG_OPT_NONSEQ_SET) {
3089 			params->optimal_nonseq_zones =
3090 			    softc->optimal_nonseq_zones;
3091 			params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
3092 		}
3093 
3094 		if (softc->zone_flags & DA_ZONE_FLAG_MAX_SEQ_SET) {
3095 			params->max_seq_zones = softc->max_seq_zones;
3096 			params->flags |= DISK_ZONE_MAX_SEQ_SET;
3097 		}
3098 		if (softc->zone_flags & DA_ZONE_FLAG_RZ_SUP)
3099 			params->flags |= DISK_ZONE_RZ_SUP;
3100 
3101 		if (softc->zone_flags & DA_ZONE_FLAG_OPEN_SUP)
3102 			params->flags |= DISK_ZONE_OPEN_SUP;
3103 
3104 		if (softc->zone_flags & DA_ZONE_FLAG_CLOSE_SUP)
3105 			params->flags |= DISK_ZONE_CLOSE_SUP;
3106 
3107 		if (softc->zone_flags & DA_ZONE_FLAG_FINISH_SUP)
3108 			params->flags |= DISK_ZONE_FINISH_SUP;
3109 
3110 		if (softc->zone_flags & DA_ZONE_FLAG_RWP_SUP)
3111 			params->flags |= DISK_ZONE_RWP_SUP;
3112 		break;
3113 	}
3114 	default:
3115 		break;
3116 	}
3117 bailout:
3118 	return (error);
3119 }
3120 
3121 static void
3122 dastart(struct cam_periph *periph, union ccb *start_ccb)
3123 {
3124 	struct da_softc *softc;
3125 
3126 	cam_periph_assert(periph, MA_OWNED);
3127 	softc = (struct da_softc *)periph->softc;
3128 
3129 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dastart\n"));
3130 
3131 skipstate:
3132 	switch (softc->state) {
3133 	case DA_STATE_NORMAL:
3134 	{
3135 		struct bio *bp;
3136 		uint8_t tag_code;
3137 
3138 more:
3139 		bp = cam_iosched_next_bio(softc->cam_iosched);
3140 		if (bp == NULL) {
3141 			if (cam_iosched_has_work_flags(softc->cam_iosched,
3142 			    DA_WORK_TUR)) {
3143 				softc->flags |= DA_FLAG_TUR_PENDING;
3144 				cam_iosched_clr_work_flags(softc->cam_iosched,
3145 				    DA_WORK_TUR);
3146 				scsi_test_unit_ready(&start_ccb->csio,
3147 				     /*retries*/ da_retry_count,
3148 				     dadone_tur,
3149 				     MSG_SIMPLE_Q_TAG,
3150 				     SSD_FULL_SIZE,
3151 				     da_default_timeout * 1000);
3152 				start_ccb->ccb_h.ccb_bp = NULL;
3153 				start_ccb->ccb_h.ccb_state = DA_CCB_TUR;
3154 				xpt_action(start_ccb);
3155 			} else
3156 				xpt_release_ccb(start_ccb);
3157 			break;
3158 		}
3159 
3160 		if (bp->bio_cmd == BIO_DELETE) {
3161 			if (softc->delete_func != NULL) {
3162 				softc->delete_func(periph, start_ccb, bp);
3163 				goto out;
3164 			} else {
3165 				/*
3166 				 * Not sure this is possible, but failsafe by
3167 				 * lying and saying "sure, done."
3168 				 */
3169 				biofinish(bp, NULL, 0);
3170 				goto more;
3171 			}
3172 		}
3173 
3174 		if (cam_iosched_has_work_flags(softc->cam_iosched,
3175 		    DA_WORK_TUR)) {
3176 			cam_iosched_clr_work_flags(softc->cam_iosched,
3177 			    DA_WORK_TUR);
3178 			da_periph_release_locked(periph, DA_REF_TUR);
3179 		}
3180 
3181 		if ((bp->bio_flags & BIO_ORDERED) != 0 ||
3182 		    (softc->flags & DA_FLAG_NEED_OTAG) != 0) {
3183 			softc->flags &= ~DA_FLAG_NEED_OTAG;
3184 			softc->flags |= DA_FLAG_WAS_OTAG;
3185 			tag_code = MSG_ORDERED_Q_TAG;
3186 		} else {
3187 			tag_code = MSG_SIMPLE_Q_TAG;
3188 		}
3189 
3190 		switch (bp->bio_cmd) {
3191 		case BIO_WRITE:
3192 		case BIO_READ:
3193 		{
3194 			void *data_ptr;
3195 			int rw_op;
3196 
3197 			biotrack(bp, __func__);
3198 
3199 			if (bp->bio_cmd == BIO_WRITE) {
3200 				softc->flags |= DA_FLAG_DIRTY;
3201 				rw_op = SCSI_RW_WRITE;
3202 			} else {
3203 				rw_op = SCSI_RW_READ;
3204 			}
3205 
3206 			data_ptr = bp->bio_data;
3207 			if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
3208 				rw_op |= SCSI_RW_BIO;
3209 				data_ptr = bp;
3210 			}
3211 
3212 			scsi_read_write(&start_ccb->csio,
3213 					/*retries*/da_retry_count,
3214 					/*cbfcnp*/dadone,
3215 					/*tag_action*/tag_code,
3216 					rw_op,
3217 					/*byte2*/0,
3218 					softc->minimum_cmd_size,
3219 					/*lba*/bp->bio_pblkno,
3220 					/*block_count*/bp->bio_bcount /
3221 					softc->params.secsize,
3222 					data_ptr,
3223 					/*dxfer_len*/ bp->bio_bcount,
3224 					/*sense_len*/SSD_FULL_SIZE,
3225 					da_default_timeout * 1000);
3226 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
3227 			start_ccb->csio.bio = bp;
3228 #endif
3229 			break;
3230 		}
3231 		case BIO_FLUSH:
3232 			/*
3233 			 * If we don't support sync cache, or the disk
3234 			 * isn't dirty, FLUSH is a no-op.  Use the
3235 			 * allocated CCB for the next bio if one is
3236 			 * available.
3237 			 */
3238 			if ((softc->quirks & DA_Q_NO_SYNC_CACHE) != 0 ||
3239 			    (softc->flags & DA_FLAG_DIRTY) == 0) {
3240 				biodone(bp);
3241 				goto skipstate;
3242 			}
3243 
3244 			/*
3245 			 * BIO_FLUSH doesn't currently communicate
3246 			 * range data, so we synchronize the cache
3247 			 * over the whole disk.  We also force
3248 			 * ordered tag semantics the flush applies
3249 			 * to all previously queued I/O.
3250 			 */
3251 			scsi_synchronize_cache(&start_ccb->csio,
3252 					       /*retries*/1,
3253 					       /*cbfcnp*/dadone,
3254 					       MSG_ORDERED_Q_TAG,
3255 					       /*begin_lba*/0,
3256 					       /*lb_count*/0,
3257 					       SSD_FULL_SIZE,
3258 					       da_default_timeout*1000);
3259 			/*
3260 			 * Clear the dirty flag before sending the command.
3261 			 * Either this sync cache will be successful, or it
3262 			 * will fail after a retry.  If it fails, it is
3263 			 * unlikely to be successful if retried later, so
3264 			 * we'll save ourselves time by just marking the
3265 			 * device clean.
3266 			 */
3267 			softc->flags &= ~DA_FLAG_DIRTY;
3268 			break;
3269 		case BIO_ZONE: {
3270 			int error, queue_ccb;
3271 
3272 			queue_ccb = 0;
3273 
3274 			error = da_zone_cmd(periph, start_ccb, bp,&queue_ccb);
3275 			if ((error != 0)
3276 			 || (queue_ccb == 0)) {
3277 				biofinish(bp, NULL, error);
3278 				xpt_release_ccb(start_ccb);
3279 				return;
3280 			}
3281 			break;
3282 		}
3283 		}
3284 		start_ccb->ccb_h.ccb_state = DA_CCB_BUFFER_IO;
3285 		start_ccb->ccb_h.flags |= CAM_UNLOCKED;
3286 		start_ccb->ccb_h.softtimeout = sbttotv(da_default_softtimeout);
3287 
3288 out:
3289 		LIST_INSERT_HEAD(&softc->pending_ccbs,
3290 				 &start_ccb->ccb_h, periph_links.le);
3291 
3292 		/* We expect a unit attention from this device */
3293 		if ((softc->flags & DA_FLAG_RETRY_UA) != 0) {
3294 			start_ccb->ccb_h.ccb_state |= DA_CCB_RETRY_UA;
3295 			softc->flags &= ~DA_FLAG_RETRY_UA;
3296 		}
3297 
3298 		start_ccb->ccb_h.ccb_bp = bp;
3299 		softc->refcount++;
3300 		cam_periph_unlock(periph);
3301 		xpt_action(start_ccb);
3302 		cam_periph_lock(periph);
3303 		softc->refcount--;
3304 
3305 		/* May have more work to do, so ensure we stay scheduled */
3306 		daschedule(periph);
3307 		break;
3308 	}
3309 	case DA_STATE_PROBE_WP:
3310 	{
3311 		void  *mode_buf;
3312 		int    mode_buf_len;
3313 
3314 		mode_buf_len = 192;
3315 		mode_buf = malloc(mode_buf_len, M_SCSIDA, M_NOWAIT);
3316 		if (mode_buf == NULL) {
3317 			xpt_print(periph->path, "Unable to send mode sense - "
3318 			    "malloc failure\n");
3319 			softc->state = DA_STATE_PROBE_RC;
3320 			goto skipstate;
3321 		}
3322 		scsi_mode_sense_len(&start_ccb->csio,
3323 				    /*retries*/ da_retry_count,
3324 				    /*cbfcnp*/ dadone_probewp,
3325 				    /*tag_action*/ MSG_SIMPLE_Q_TAG,
3326 				    /*dbd*/ FALSE,
3327 				    /*pc*/ SMS_PAGE_CTRL_CURRENT,
3328 				    /*page*/ SMS_ALL_PAGES_PAGE,
3329 				    /*param_buf*/ mode_buf,
3330 				    /*param_len*/ mode_buf_len,
3331 				    /*minimum_cmd_size*/ softc->minimum_cmd_size,
3332 				    /*sense_len*/ SSD_FULL_SIZE,
3333 				    /*timeout*/ da_default_timeout * 1000);
3334 		start_ccb->ccb_h.ccb_bp = NULL;
3335 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_WP;
3336 		xpt_action(start_ccb);
3337 		break;
3338 	}
3339 	case DA_STATE_PROBE_RC:
3340 	{
3341 		struct scsi_read_capacity_data *rcap;
3342 
3343 		rcap = (struct scsi_read_capacity_data *)
3344 		    malloc(sizeof(*rcap), M_SCSIDA, M_NOWAIT|M_ZERO);
3345 		if (rcap == NULL) {
3346 			printf("dastart: Couldn't malloc read_capacity data\n");
3347 			/* da_free_periph??? */
3348 			break;
3349 		}
3350 		scsi_read_capacity(&start_ccb->csio,
3351 				   /*retries*/da_retry_count,
3352 				   dadone_proberc,
3353 				   MSG_SIMPLE_Q_TAG,
3354 				   rcap,
3355 				   SSD_FULL_SIZE,
3356 				   /*timeout*/5000);
3357 		start_ccb->ccb_h.ccb_bp = NULL;
3358 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC;
3359 		xpt_action(start_ccb);
3360 		break;
3361 	}
3362 	case DA_STATE_PROBE_RC16:
3363 	{
3364 		struct scsi_read_capacity_data_long *rcaplong;
3365 
3366 		rcaplong = (struct scsi_read_capacity_data_long *)
3367 			malloc(sizeof(*rcaplong), M_SCSIDA, M_NOWAIT|M_ZERO);
3368 		if (rcaplong == NULL) {
3369 			printf("dastart: Couldn't malloc read_capacity data\n");
3370 			/* da_free_periph??? */
3371 			break;
3372 		}
3373 		scsi_read_capacity_16(&start_ccb->csio,
3374 				      /*retries*/ da_retry_count,
3375 				      /*cbfcnp*/ dadone_proberc,
3376 				      /*tag_action*/ MSG_SIMPLE_Q_TAG,
3377 				      /*lba*/ 0,
3378 				      /*reladr*/ 0,
3379 				      /*pmi*/ 0,
3380 				      /*rcap_buf*/ (uint8_t *)rcaplong,
3381 				      /*rcap_buf_len*/ sizeof(*rcaplong),
3382 				      /*sense_len*/ SSD_FULL_SIZE,
3383 				      /*timeout*/ da_default_timeout * 1000);
3384 		start_ccb->ccb_h.ccb_bp = NULL;
3385 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_RC16;
3386 		xpt_action(start_ccb);
3387 		break;
3388 	}
3389 	case DA_STATE_PROBE_LBP:
3390 	{
3391 		struct scsi_vpd_logical_block_prov *lbp;
3392 
3393 		if (!scsi_vpd_supported_page(periph, SVPD_LBP)) {
3394 			/*
3395 			 * If we get here we don't support any SBC-3 delete
3396 			 * methods with UNMAP as the Logical Block Provisioning
3397 			 * VPD page support is required for devices which
3398 			 * support it according to T10/1799-D Revision 31
3399 			 * however older revisions of the spec don't mandate
3400 			 * this so we currently don't remove these methods
3401 			 * from the available set.
3402 			 */
3403 			softc->state = DA_STATE_PROBE_BLK_LIMITS;
3404 			goto skipstate;
3405 		}
3406 
3407 		lbp = (struct scsi_vpd_logical_block_prov *)
3408 			malloc(sizeof(*lbp), M_SCSIDA, M_NOWAIT|M_ZERO);
3409 
3410 		if (lbp == NULL) {
3411 			printf("dastart: Couldn't malloc lbp data\n");
3412 			/* da_free_periph??? */
3413 			break;
3414 		}
3415 
3416 		scsi_inquiry(&start_ccb->csio,
3417 			     /*retries*/da_retry_count,
3418 			     /*cbfcnp*/dadone_probelbp,
3419 			     /*tag_action*/MSG_SIMPLE_Q_TAG,
3420 			     /*inq_buf*/(u_int8_t *)lbp,
3421 			     /*inq_len*/sizeof(*lbp),
3422 			     /*evpd*/TRUE,
3423 			     /*page_code*/SVPD_LBP,
3424 			     /*sense_len*/SSD_MIN_SIZE,
3425 			     /*timeout*/da_default_timeout * 1000);
3426 		start_ccb->ccb_h.ccb_bp = NULL;
3427 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_LBP;
3428 		xpt_action(start_ccb);
3429 		break;
3430 	}
3431 	case DA_STATE_PROBE_BLK_LIMITS:
3432 	{
3433 		struct scsi_vpd_block_limits *block_limits;
3434 
3435 		if (!scsi_vpd_supported_page(periph, SVPD_BLOCK_LIMITS)) {
3436 			/* Not supported skip to next probe */
3437 			softc->state = DA_STATE_PROBE_BDC;
3438 			goto skipstate;
3439 		}
3440 
3441 		block_limits = (struct scsi_vpd_block_limits *)
3442 			malloc(sizeof(*block_limits), M_SCSIDA, M_NOWAIT|M_ZERO);
3443 
3444 		if (block_limits == NULL) {
3445 			printf("dastart: Couldn't malloc block_limits data\n");
3446 			/* da_free_periph??? */
3447 			break;
3448 		}
3449 
3450 		scsi_inquiry(&start_ccb->csio,
3451 			     /*retries*/da_retry_count,
3452 			     /*cbfcnp*/dadone_probeblklimits,
3453 			     /*tag_action*/MSG_SIMPLE_Q_TAG,
3454 			     /*inq_buf*/(u_int8_t *)block_limits,
3455 			     /*inq_len*/sizeof(*block_limits),
3456 			     /*evpd*/TRUE,
3457 			     /*page_code*/SVPD_BLOCK_LIMITS,
3458 			     /*sense_len*/SSD_MIN_SIZE,
3459 			     /*timeout*/da_default_timeout * 1000);
3460 		start_ccb->ccb_h.ccb_bp = NULL;
3461 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BLK_LIMITS;
3462 		xpt_action(start_ccb);
3463 		break;
3464 	}
3465 	case DA_STATE_PROBE_BDC:
3466 	{
3467 		struct scsi_vpd_block_characteristics *bdc;
3468 
3469 		if (!scsi_vpd_supported_page(periph, SVPD_BDC)) {
3470 			softc->state = DA_STATE_PROBE_ATA;
3471 			goto skipstate;
3472 		}
3473 
3474 		bdc = (struct scsi_vpd_block_characteristics *)
3475 			malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO);
3476 
3477 		if (bdc == NULL) {
3478 			printf("dastart: Couldn't malloc bdc data\n");
3479 			/* da_free_periph??? */
3480 			break;
3481 		}
3482 
3483 		scsi_inquiry(&start_ccb->csio,
3484 			     /*retries*/da_retry_count,
3485 			     /*cbfcnp*/dadone_probebdc,
3486 			     /*tag_action*/MSG_SIMPLE_Q_TAG,
3487 			     /*inq_buf*/(u_int8_t *)bdc,
3488 			     /*inq_len*/sizeof(*bdc),
3489 			     /*evpd*/TRUE,
3490 			     /*page_code*/SVPD_BDC,
3491 			     /*sense_len*/SSD_MIN_SIZE,
3492 			     /*timeout*/da_default_timeout * 1000);
3493 		start_ccb->ccb_h.ccb_bp = NULL;
3494 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_BDC;
3495 		xpt_action(start_ccb);
3496 		break;
3497 	}
3498 	case DA_STATE_PROBE_ATA:
3499 	{
3500 		struct ata_params *ata_params;
3501 
3502 		if (!scsi_vpd_supported_page(periph, SVPD_ATA_INFORMATION)) {
3503 			if ((softc->zone_mode == DA_ZONE_HOST_AWARE)
3504 			 || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) {
3505 				/*
3506 				 * Note that if the ATA VPD page isn't
3507 				 * supported, we aren't talking to an ATA
3508 				 * device anyway.  Support for that VPD
3509 				 * page is mandatory for SCSI to ATA (SAT)
3510 				 * translation layers.
3511 				 */
3512 				softc->state = DA_STATE_PROBE_ZONE;
3513 				goto skipstate;
3514 			}
3515 			daprobedone(periph, start_ccb);
3516 			break;
3517 		}
3518 
3519 		ata_params = (struct ata_params*)
3520 			malloc(sizeof(*ata_params), M_SCSIDA,M_NOWAIT|M_ZERO);
3521 
3522 		if (ata_params == NULL) {
3523 			xpt_print(periph->path, "Couldn't malloc ata_params "
3524 			    "data\n");
3525 			/* da_free_periph??? */
3526 			break;
3527 		}
3528 
3529 		scsi_ata_identify(&start_ccb->csio,
3530 				  /*retries*/da_retry_count,
3531 				  /*cbfcnp*/dadone_probeata,
3532                                   /*tag_action*/MSG_SIMPLE_Q_TAG,
3533 				  /*data_ptr*/(u_int8_t *)ata_params,
3534 				  /*dxfer_len*/sizeof(*ata_params),
3535 				  /*sense_len*/SSD_FULL_SIZE,
3536 				  /*timeout*/da_default_timeout * 1000);
3537 		start_ccb->ccb_h.ccb_bp = NULL;
3538 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA;
3539 		xpt_action(start_ccb);
3540 		break;
3541 	}
3542 	case DA_STATE_PROBE_ATA_LOGDIR:
3543 	{
3544 		struct ata_gp_log_dir *log_dir;
3545 		int retval;
3546 
3547 		retval = 0;
3548 
3549 		if ((softc->flags & DA_FLAG_CAN_ATA_LOG) == 0) {
3550 			/*
3551 			 * If we don't have log support, not much point in
3552 			 * trying to probe zone support.
3553 			 */
3554 			daprobedone(periph, start_ccb);
3555 			break;
3556 		}
3557 
3558 		/*
3559 		 * If we have an ATA device (the SCSI ATA Information VPD
3560 		 * page should be present and the ATA identify should have
3561 		 * succeeded) and it supports logs, ask for the log directory.
3562 		 */
3563 
3564 		log_dir = malloc(sizeof(*log_dir), M_SCSIDA, M_NOWAIT|M_ZERO);
3565 		if (log_dir == NULL) {
3566 			xpt_print(periph->path, "Couldn't malloc log_dir "
3567 			    "data\n");
3568 			daprobedone(periph, start_ccb);
3569 			break;
3570 		}
3571 
3572 		retval = scsi_ata_read_log(&start_ccb->csio,
3573 		    /*retries*/ da_retry_count,
3574 		    /*cbfcnp*/ dadone_probeatalogdir,
3575 		    /*tag_action*/ MSG_SIMPLE_Q_TAG,
3576 		    /*log_address*/ ATA_LOG_DIRECTORY,
3577 		    /*page_number*/ 0,
3578 		    /*block_count*/ 1,
3579 		    /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3580 				 AP_PROTO_DMA : AP_PROTO_PIO_IN,
3581 		    /*data_ptr*/ (uint8_t *)log_dir,
3582 		    /*dxfer_len*/ sizeof(*log_dir),
3583 		    /*sense_len*/ SSD_FULL_SIZE,
3584 		    /*timeout*/ da_default_timeout * 1000);
3585 
3586 		if (retval != 0) {
3587 			xpt_print(periph->path, "scsi_ata_read_log() failed!");
3588 			free(log_dir, M_SCSIDA);
3589 			daprobedone(periph, start_ccb);
3590 			break;
3591 		}
3592 		start_ccb->ccb_h.ccb_bp = NULL;
3593 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_LOGDIR;
3594 		xpt_action(start_ccb);
3595 		break;
3596 	}
3597 	case DA_STATE_PROBE_ATA_IDDIR:
3598 	{
3599 		struct ata_identify_log_pages *id_dir;
3600 		int retval;
3601 
3602 		retval = 0;
3603 
3604 		/*
3605 		 * Check here to see whether the Identify Device log is
3606 		 * supported in the directory of logs.  If so, continue
3607 		 * with requesting the log of identify device pages.
3608 		 */
3609 		if ((softc->flags & DA_FLAG_CAN_ATA_IDLOG) == 0) {
3610 			daprobedone(periph, start_ccb);
3611 			break;
3612 		}
3613 
3614 		id_dir = malloc(sizeof(*id_dir), M_SCSIDA, M_NOWAIT | M_ZERO);
3615 		if (id_dir == NULL) {
3616 			xpt_print(periph->path, "Couldn't malloc id_dir "
3617 			    "data\n");
3618 			daprobedone(periph, start_ccb);
3619 			break;
3620 		}
3621 
3622 		retval = scsi_ata_read_log(&start_ccb->csio,
3623 		    /*retries*/ da_retry_count,
3624 		    /*cbfcnp*/ dadone_probeataiddir,
3625 		    /*tag_action*/ MSG_SIMPLE_Q_TAG,
3626 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3627 		    /*page_number*/ ATA_IDL_PAGE_LIST,
3628 		    /*block_count*/ 1,
3629 		    /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3630 				 AP_PROTO_DMA : AP_PROTO_PIO_IN,
3631 		    /*data_ptr*/ (uint8_t *)id_dir,
3632 		    /*dxfer_len*/ sizeof(*id_dir),
3633 		    /*sense_len*/ SSD_FULL_SIZE,
3634 		    /*timeout*/ da_default_timeout * 1000);
3635 
3636 		if (retval != 0) {
3637 			xpt_print(periph->path, "scsi_ata_read_log() failed!");
3638 			free(id_dir, M_SCSIDA);
3639 			daprobedone(periph, start_ccb);
3640 			break;
3641 		}
3642 		start_ccb->ccb_h.ccb_bp = NULL;
3643 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_IDDIR;
3644 		xpt_action(start_ccb);
3645 		break;
3646 	}
3647 	case DA_STATE_PROBE_ATA_SUP:
3648 	{
3649 		struct ata_identify_log_sup_cap *sup_cap;
3650 		int retval;
3651 
3652 		retval = 0;
3653 
3654 		/*
3655 		 * Check here to see whether the Supported Capabilities log
3656 		 * is in the list of Identify Device logs.
3657 		 */
3658 		if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP) == 0) {
3659 			daprobedone(periph, start_ccb);
3660 			break;
3661 		}
3662 
3663 		sup_cap = malloc(sizeof(*sup_cap), M_SCSIDA, M_NOWAIT|M_ZERO);
3664 		if (sup_cap == NULL) {
3665 			xpt_print(periph->path, "Couldn't malloc sup_cap "
3666 			    "data\n");
3667 			daprobedone(periph, start_ccb);
3668 			break;
3669 		}
3670 
3671 		retval = scsi_ata_read_log(&start_ccb->csio,
3672 		    /*retries*/ da_retry_count,
3673 		    /*cbfcnp*/ dadone_probeatasup,
3674 		    /*tag_action*/ MSG_SIMPLE_Q_TAG,
3675 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3676 		    /*page_number*/ ATA_IDL_SUP_CAP,
3677 		    /*block_count*/ 1,
3678 		    /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3679 				 AP_PROTO_DMA : AP_PROTO_PIO_IN,
3680 		    /*data_ptr*/ (uint8_t *)sup_cap,
3681 		    /*dxfer_len*/ sizeof(*sup_cap),
3682 		    /*sense_len*/ SSD_FULL_SIZE,
3683 		    /*timeout*/ da_default_timeout * 1000);
3684 
3685 		if (retval != 0) {
3686 			xpt_print(periph->path, "scsi_ata_read_log() failed!");
3687 			free(sup_cap, M_SCSIDA);
3688 			daprobedone(periph, start_ccb);
3689 			break;
3690 
3691 		}
3692 
3693 		start_ccb->ccb_h.ccb_bp = NULL;
3694 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_SUP;
3695 		xpt_action(start_ccb);
3696 		break;
3697 	}
3698 	case DA_STATE_PROBE_ATA_ZONE:
3699 	{
3700 		struct ata_zoned_info_log *ata_zone;
3701 		int retval;
3702 
3703 		retval = 0;
3704 
3705 		/*
3706 		 * Check here to see whether the zoned device information
3707 		 * page is supported.  If so, continue on to request it.
3708 		 * If not, skip to DA_STATE_PROBE_LOG or done.
3709 		 */
3710 		if ((softc->flags & DA_FLAG_CAN_ATA_ZONE) == 0) {
3711 			daprobedone(periph, start_ccb);
3712 			break;
3713 		}
3714 		ata_zone = malloc(sizeof(*ata_zone), M_SCSIDA,
3715 				  M_NOWAIT|M_ZERO);
3716 		if (ata_zone == NULL) {
3717 			xpt_print(periph->path, "Couldn't malloc ata_zone "
3718 			    "data\n");
3719 			daprobedone(periph, start_ccb);
3720 			break;
3721 		}
3722 
3723 		retval = scsi_ata_read_log(&start_ccb->csio,
3724 		    /*retries*/ da_retry_count,
3725 		    /*cbfcnp*/ dadone_probeatazone,
3726 		    /*tag_action*/ MSG_SIMPLE_Q_TAG,
3727 		    /*log_address*/ ATA_IDENTIFY_DATA_LOG,
3728 		    /*page_number*/ ATA_IDL_ZDI,
3729 		    /*block_count*/ 1,
3730 		    /*protocol*/ softc->flags & DA_FLAG_CAN_ATA_DMA ?
3731 				 AP_PROTO_DMA : AP_PROTO_PIO_IN,
3732 		    /*data_ptr*/ (uint8_t *)ata_zone,
3733 		    /*dxfer_len*/ sizeof(*ata_zone),
3734 		    /*sense_len*/ SSD_FULL_SIZE,
3735 		    /*timeout*/ da_default_timeout * 1000);
3736 
3737 		if (retval != 0) {
3738 			xpt_print(periph->path, "scsi_ata_read_log() failed!");
3739 			free(ata_zone, M_SCSIDA);
3740 			daprobedone(periph, start_ccb);
3741 			break;
3742 		}
3743 		start_ccb->ccb_h.ccb_bp = NULL;
3744 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ATA_ZONE;
3745 		xpt_action(start_ccb);
3746 
3747 		break;
3748 	}
3749 	case DA_STATE_PROBE_ZONE:
3750 	{
3751 		struct scsi_vpd_zoned_bdc *bdc;
3752 
3753 		/*
3754 		 * Note that this page will be supported for SCSI protocol
3755 		 * devices that support ZBC (SMR devices), as well as ATA
3756 		 * protocol devices that are behind a SAT (SCSI to ATA
3757 		 * Translation) layer that supports converting ZBC commands
3758 		 * to their ZAC equivalents.
3759 		 */
3760 		if (!scsi_vpd_supported_page(periph, SVPD_ZONED_BDC)) {
3761 			daprobedone(periph, start_ccb);
3762 			break;
3763 		}
3764 		bdc = (struct scsi_vpd_zoned_bdc *)
3765 			malloc(sizeof(*bdc), M_SCSIDA, M_NOWAIT|M_ZERO);
3766 
3767 		if (bdc == NULL) {
3768 			xpt_release_ccb(start_ccb);
3769 			xpt_print(periph->path, "Couldn't malloc zone VPD "
3770 			    "data\n");
3771 			break;
3772 		}
3773 		scsi_inquiry(&start_ccb->csio,
3774 			     /*retries*/da_retry_count,
3775 			     /*cbfcnp*/dadone_probezone,
3776 			     /*tag_action*/MSG_SIMPLE_Q_TAG,
3777 			     /*inq_buf*/(u_int8_t *)bdc,
3778 			     /*inq_len*/sizeof(*bdc),
3779 			     /*evpd*/TRUE,
3780 			     /*page_code*/SVPD_ZONED_BDC,
3781 			     /*sense_len*/SSD_FULL_SIZE,
3782 			     /*timeout*/da_default_timeout * 1000);
3783 		start_ccb->ccb_h.ccb_bp = NULL;
3784 		start_ccb->ccb_h.ccb_state = DA_CCB_PROBE_ZONE;
3785 		xpt_action(start_ccb);
3786 		break;
3787 	}
3788 	}
3789 }
3790 
3791 /*
3792  * In each of the methods below, while its the caller's
3793  * responsibility to ensure the request will fit into a
3794  * single device request, we might have changed the delete
3795  * method due to the device incorrectly advertising either
3796  * its supported methods or limits.
3797  *
3798  * To prevent this causing further issues we validate the
3799  * against the methods limits, and warn which would
3800  * otherwise be unnecessary.
3801  */
3802 static void
3803 da_delete_unmap(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
3804 {
3805 	struct da_softc *softc = (struct da_softc *)periph->softc;;
3806 	struct bio *bp1;
3807 	uint8_t *buf = softc->unmap_buf;
3808 	struct scsi_unmap_desc *d = (void *)&buf[UNMAP_HEAD_SIZE];
3809 	uint64_t lba, lastlba = (uint64_t)-1;
3810 	uint64_t totalcount = 0;
3811 	uint64_t count;
3812 	uint32_t c, lastcount = 0, ranges = 0;
3813 
3814 	/*
3815 	 * Currently this doesn't take the UNMAP
3816 	 * Granularity and Granularity Alignment
3817 	 * fields into account.
3818 	 *
3819 	 * This could result in both unoptimal unmap
3820 	 * requests as as well as UNMAP calls unmapping
3821 	 * fewer LBA's than requested.
3822 	 */
3823 
3824 	bzero(softc->unmap_buf, sizeof(softc->unmap_buf));
3825 	bp1 = bp;
3826 	do {
3827 		/*
3828 		 * Note: ada and da are different in how they store the
3829 		 * pending bp's in a trim. ada stores all of them in the
3830 		 * trim_req.bps. da stores all but the first one in the
3831 		 * delete_run_queue. ada then completes all the bps in
3832 		 * its adadone() loop. da completes all the bps in the
3833 		 * delete_run_queue in dadone, and relies on the biodone
3834 		 * after to complete. This should be reconciled since there's
3835 		 * no real reason to do it differently. XXX
3836 		 */
3837 		if (bp1 != bp)
3838 			bioq_insert_tail(&softc->delete_run_queue, bp1);
3839 		lba = bp1->bio_pblkno;
3840 		count = bp1->bio_bcount / softc->params.secsize;
3841 
3842 		/* Try to extend the previous range. */
3843 		if (lba == lastlba) {
3844 			c = omin(count, UNMAP_RANGE_MAX - lastcount);
3845 			lastlba += c;
3846 			lastcount += c;
3847 			scsi_ulto4b(lastcount, d[ranges - 1].length);
3848 			count -= c;
3849 			lba += c;
3850 			totalcount += c;
3851 		} else if ((softc->quirks & DA_Q_STRICT_UNMAP) &&
3852 		    softc->unmap_gran != 0) {
3853 			/* Align length of the previous range. */
3854 			if ((c = lastcount % softc->unmap_gran) != 0) {
3855 				if (lastcount <= c) {
3856 					totalcount -= lastcount;
3857 					lastlba = (uint64_t)-1;
3858 					lastcount = 0;
3859 					ranges--;
3860 				} else {
3861 					totalcount -= c;
3862 					lastlba -= c;
3863 					lastcount -= c;
3864 					scsi_ulto4b(lastcount,
3865 					    d[ranges - 1].length);
3866 				}
3867 			}
3868 			/* Align beginning of the new range. */
3869 			c = (lba - softc->unmap_gran_align) % softc->unmap_gran;
3870 			if (c != 0) {
3871 				c = softc->unmap_gran - c;
3872 				if (count <= c) {
3873 					count = 0;
3874 				} else {
3875 					lba += c;
3876 					count -= c;
3877 				}
3878 			}
3879 		}
3880 
3881 		while (count > 0) {
3882 			c = omin(count, UNMAP_RANGE_MAX);
3883 			if (totalcount + c > softc->unmap_max_lba ||
3884 			    ranges >= softc->unmap_max_ranges) {
3885 				xpt_print(periph->path,
3886 				    "%s issuing short delete %ld > %ld"
3887 				    "|| %d >= %d",
3888 				    da_delete_method_desc[softc->delete_method],
3889 				    totalcount + c, softc->unmap_max_lba,
3890 				    ranges, softc->unmap_max_ranges);
3891 				break;
3892 			}
3893 			scsi_u64to8b(lba, d[ranges].lba);
3894 			scsi_ulto4b(c, d[ranges].length);
3895 			lba += c;
3896 			totalcount += c;
3897 			ranges++;
3898 			count -= c;
3899 			lastlba = lba;
3900 			lastcount = c;
3901 		}
3902 		bp1 = cam_iosched_next_trim(softc->cam_iosched);
3903 		if (bp1 == NULL)
3904 			break;
3905 		if (ranges >= softc->unmap_max_ranges ||
3906 		    totalcount + bp1->bio_bcount /
3907 		    softc->params.secsize > softc->unmap_max_lba) {
3908 			cam_iosched_put_back_trim(softc->cam_iosched, bp1);
3909 			break;
3910 		}
3911 	} while (1);
3912 
3913 	/* Align length of the last range. */
3914 	if ((softc->quirks & DA_Q_STRICT_UNMAP) && softc->unmap_gran != 0 &&
3915 	    (c = lastcount % softc->unmap_gran) != 0) {
3916 		if (lastcount <= c)
3917 			ranges--;
3918 		else
3919 			scsi_ulto4b(lastcount - c, d[ranges - 1].length);
3920 	}
3921 
3922 	scsi_ulto2b(ranges * 16 + 6, &buf[0]);
3923 	scsi_ulto2b(ranges * 16, &buf[2]);
3924 
3925 	scsi_unmap(&ccb->csio,
3926 		   /*retries*/da_retry_count,
3927 		   /*cbfcnp*/dadone,
3928 		   /*tag_action*/MSG_SIMPLE_Q_TAG,
3929 		   /*byte2*/0,
3930 		   /*data_ptr*/ buf,
3931 		   /*dxfer_len*/ ranges * 16 + 8,
3932 		   /*sense_len*/SSD_FULL_SIZE,
3933 		   da_default_timeout * 1000);
3934 	ccb->ccb_h.ccb_state = DA_CCB_DELETE;
3935 	ccb->ccb_h.flags |= CAM_UNLOCKED;
3936 	cam_iosched_submit_trim(softc->cam_iosched);
3937 }
3938 
3939 static void
3940 da_delete_trim(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
3941 {
3942 	struct da_softc *softc = (struct da_softc *)periph->softc;
3943 	struct bio *bp1;
3944 	uint8_t *buf = softc->unmap_buf;
3945 	uint64_t lastlba = (uint64_t)-1;
3946 	uint64_t count;
3947 	uint64_t lba;
3948 	uint32_t lastcount = 0, c, requestcount;
3949 	int ranges = 0, off, block_count;
3950 
3951 	bzero(softc->unmap_buf, sizeof(softc->unmap_buf));
3952 	bp1 = bp;
3953 	do {
3954 		if (bp1 != bp)//XXX imp XXX
3955 			bioq_insert_tail(&softc->delete_run_queue, bp1);
3956 		lba = bp1->bio_pblkno;
3957 		count = bp1->bio_bcount / softc->params.secsize;
3958 		requestcount = count;
3959 
3960 		/* Try to extend the previous range. */
3961 		if (lba == lastlba) {
3962 			c = omin(count, ATA_DSM_RANGE_MAX - lastcount);
3963 			lastcount += c;
3964 			off = (ranges - 1) * 8;
3965 			buf[off + 6] = lastcount & 0xff;
3966 			buf[off + 7] = (lastcount >> 8) & 0xff;
3967 			count -= c;
3968 			lba += c;
3969 		}
3970 
3971 		while (count > 0) {
3972 			c = omin(count, ATA_DSM_RANGE_MAX);
3973 			off = ranges * 8;
3974 
3975 			buf[off + 0] = lba & 0xff;
3976 			buf[off + 1] = (lba >> 8) & 0xff;
3977 			buf[off + 2] = (lba >> 16) & 0xff;
3978 			buf[off + 3] = (lba >> 24) & 0xff;
3979 			buf[off + 4] = (lba >> 32) & 0xff;
3980 			buf[off + 5] = (lba >> 40) & 0xff;
3981 			buf[off + 6] = c & 0xff;
3982 			buf[off + 7] = (c >> 8) & 0xff;
3983 			lba += c;
3984 			ranges++;
3985 			count -= c;
3986 			lastcount = c;
3987 			if (count != 0 && ranges == softc->trim_max_ranges) {
3988 				xpt_print(periph->path,
3989 				    "%s issuing short delete %ld > %ld\n",
3990 				    da_delete_method_desc[softc->delete_method],
3991 				    requestcount,
3992 				    (softc->trim_max_ranges - ranges) *
3993 				    ATA_DSM_RANGE_MAX);
3994 				break;
3995 			}
3996 		}
3997 		lastlba = lba;
3998 		bp1 = cam_iosched_next_trim(softc->cam_iosched);
3999 		if (bp1 == NULL)
4000 			break;
4001 		if (bp1->bio_bcount / softc->params.secsize >
4002 		    (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
4003 			cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4004 			break;
4005 		}
4006 	} while (1);
4007 
4008 	block_count = howmany(ranges, ATA_DSM_BLK_RANGES);
4009 	scsi_ata_trim(&ccb->csio,
4010 		      /*retries*/da_retry_count,
4011 		      /*cbfcnp*/dadone,
4012 		      /*tag_action*/MSG_SIMPLE_Q_TAG,
4013 		      block_count,
4014 		      /*data_ptr*/buf,
4015 		      /*dxfer_len*/block_count * ATA_DSM_BLK_SIZE,
4016 		      /*sense_len*/SSD_FULL_SIZE,
4017 		      da_default_timeout * 1000);
4018 	ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4019 	ccb->ccb_h.flags |= CAM_UNLOCKED;
4020 	cam_iosched_submit_trim(softc->cam_iosched);
4021 }
4022 
4023 /*
4024  * We calculate ws_max_blks here based off d_delmaxsize instead
4025  * of using softc->ws_max_blks as it is absolute max for the
4026  * device not the protocol max which may well be lower.
4027  */
4028 static void
4029 da_delete_ws(struct cam_periph *periph, union ccb *ccb, struct bio *bp)
4030 {
4031 	struct da_softc *softc;
4032 	struct bio *bp1;
4033 	uint64_t ws_max_blks;
4034 	uint64_t lba;
4035 	uint64_t count; /* forward compat with WS32 */
4036 
4037 	softc = (struct da_softc *)periph->softc;
4038 	ws_max_blks = softc->disk->d_delmaxsize / softc->params.secsize;
4039 	lba = bp->bio_pblkno;
4040 	count = 0;
4041 	bp1 = bp;
4042 	do {
4043 		if (bp1 != bp)//XXX imp XXX
4044 			bioq_insert_tail(&softc->delete_run_queue, bp1);
4045 		count += bp1->bio_bcount / softc->params.secsize;
4046 		if (count > ws_max_blks) {
4047 			xpt_print(periph->path,
4048 			    "%s issuing short delete %ld > %ld\n",
4049 			    da_delete_method_desc[softc->delete_method],
4050 			    count, ws_max_blks);
4051 			count = omin(count, ws_max_blks);
4052 			break;
4053 		}
4054 		bp1 = cam_iosched_next_trim(softc->cam_iosched);
4055 		if (bp1 == NULL)
4056 			break;
4057 		if (lba + count != bp1->bio_pblkno ||
4058 		    count + bp1->bio_bcount /
4059 		    softc->params.secsize > ws_max_blks) {
4060 			cam_iosched_put_back_trim(softc->cam_iosched, bp1);
4061 			break;
4062 		}
4063 	} while (1);
4064 
4065 	scsi_write_same(&ccb->csio,
4066 			/*retries*/da_retry_count,
4067 			/*cbfcnp*/dadone,
4068 			/*tag_action*/MSG_SIMPLE_Q_TAG,
4069 			/*byte2*/softc->delete_method ==
4070 			    DA_DELETE_ZERO ? 0 : SWS_UNMAP,
4071 			softc->delete_method == DA_DELETE_WS16 ? 16 : 10,
4072 			/*lba*/lba,
4073 			/*block_count*/count,
4074 			/*data_ptr*/ __DECONST(void *, zero_region),
4075 			/*dxfer_len*/ softc->params.secsize,
4076 			/*sense_len*/SSD_FULL_SIZE,
4077 			da_default_timeout * 1000);
4078 	ccb->ccb_h.ccb_state = DA_CCB_DELETE;
4079 	ccb->ccb_h.flags |= CAM_UNLOCKED;
4080 	cam_iosched_submit_trim(softc->cam_iosched);
4081 }
4082 
4083 static int
4084 cmd6workaround(union ccb *ccb)
4085 {
4086 	struct scsi_rw_6 cmd6;
4087 	struct scsi_rw_10 *cmd10;
4088 	struct da_softc *softc;
4089 	u_int8_t *cdb;
4090 	struct bio *bp;
4091 	int frozen;
4092 
4093 	cdb = ccb->csio.cdb_io.cdb_bytes;
4094 	softc = (struct da_softc *)xpt_path_periph(ccb->ccb_h.path)->softc;
4095 
4096 	if (ccb->ccb_h.ccb_state == DA_CCB_DELETE) {
4097 		da_delete_methods old_method = softc->delete_method;
4098 
4099 		/*
4100 		 * Typically there are two reasons for failure here
4101 		 * 1. Delete method was detected as supported but isn't
4102 		 * 2. Delete failed due to invalid params e.g. too big
4103 		 *
4104 		 * While we will attempt to choose an alternative delete method
4105 		 * this may result in short deletes if the existing delete
4106 		 * requests from geom are big for the new method chosen.
4107 		 *
4108 		 * This method assumes that the error which triggered this
4109 		 * will not retry the io otherwise a panic will occur
4110 		 */
4111 		dadeleteflag(softc, old_method, 0);
4112 		dadeletemethodchoose(softc, DA_DELETE_DISABLE);
4113 		if (softc->delete_method == DA_DELETE_DISABLE)
4114 			xpt_print(ccb->ccb_h.path,
4115 				  "%s failed, disabling BIO_DELETE\n",
4116 				  da_delete_method_desc[old_method]);
4117 		else
4118 			xpt_print(ccb->ccb_h.path,
4119 				  "%s failed, switching to %s BIO_DELETE\n",
4120 				  da_delete_method_desc[old_method],
4121 				  da_delete_method_desc[softc->delete_method]);
4122 
4123 		while ((bp = bioq_takefirst(&softc->delete_run_queue)) != NULL)
4124 			cam_iosched_queue_work(softc->cam_iosched, bp);
4125 		cam_iosched_queue_work(softc->cam_iosched,
4126 		    (struct bio *)ccb->ccb_h.ccb_bp);
4127 		ccb->ccb_h.ccb_bp = NULL;
4128 		return (0);
4129 	}
4130 
4131 	/* Detect unsupported PREVENT ALLOW MEDIUM REMOVAL. */
4132 	if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 &&
4133 	    (*cdb == PREVENT_ALLOW) &&
4134 	    (softc->quirks & DA_Q_NO_PREVENT) == 0) {
4135 		if (bootverbose)
4136 			xpt_print(ccb->ccb_h.path,
4137 			    "PREVENT ALLOW MEDIUM REMOVAL not supported.\n");
4138 		softc->quirks |= DA_Q_NO_PREVENT;
4139 		return (0);
4140 	}
4141 
4142 	/* Detect unsupported SYNCHRONIZE CACHE(10). */
4143 	if ((ccb->ccb_h.flags & CAM_CDB_POINTER) == 0 &&
4144 	    (*cdb == SYNCHRONIZE_CACHE) &&
4145 	    (softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
4146 		if (bootverbose)
4147 			xpt_print(ccb->ccb_h.path,
4148 			    "SYNCHRONIZE CACHE(10) not supported.\n");
4149 		softc->quirks |= DA_Q_NO_SYNC_CACHE;
4150 		softc->disk->d_flags &= ~DISKFLAG_CANFLUSHCACHE;
4151 		return (0);
4152 	}
4153 
4154 	/* Translation only possible if CDB is an array and cmd is R/W6 */
4155 	if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0 ||
4156 	    (*cdb != READ_6 && *cdb != WRITE_6))
4157 		return 0;
4158 
4159 	xpt_print(ccb->ccb_h.path, "READ(6)/WRITE(6) not supported, "
4160 	    "increasing minimum_cmd_size to 10.\n");
4161 	softc->minimum_cmd_size = 10;
4162 
4163 	bcopy(cdb, &cmd6, sizeof(struct scsi_rw_6));
4164 	cmd10 = (struct scsi_rw_10 *)cdb;
4165 	cmd10->opcode = (cmd6.opcode == READ_6) ? READ_10 : WRITE_10;
4166 	cmd10->byte2 = 0;
4167 	scsi_ulto4b(scsi_3btoul(cmd6.addr), cmd10->addr);
4168 	cmd10->reserved = 0;
4169 	scsi_ulto2b(cmd6.length, cmd10->length);
4170 	cmd10->control = cmd6.control;
4171 	ccb->csio.cdb_len = sizeof(*cmd10);
4172 
4173 	/* Requeue request, unfreezing queue if necessary */
4174 	frozen = (ccb->ccb_h.status & CAM_DEV_QFRZN) != 0;
4175 	ccb->ccb_h.status = CAM_REQUEUE_REQ;
4176 	xpt_action(ccb);
4177 	if (frozen) {
4178 		cam_release_devq(ccb->ccb_h.path,
4179 				 /*relsim_flags*/0,
4180 				 /*reduction*/0,
4181 				 /*timeout*/0,
4182 				 /*getcount_only*/0);
4183 	}
4184 	return (ERESTART);
4185 }
4186 
4187 static void
4188 dazonedone(struct cam_periph *periph, union ccb *ccb)
4189 {
4190 	struct da_softc *softc;
4191 	struct bio *bp;
4192 
4193 	softc = periph->softc;
4194 	bp = (struct bio *)ccb->ccb_h.ccb_bp;
4195 
4196 	switch (bp->bio_zone.zone_cmd) {
4197 	case DISK_ZONE_OPEN:
4198 	case DISK_ZONE_CLOSE:
4199 	case DISK_ZONE_FINISH:
4200 	case DISK_ZONE_RWP:
4201 		break;
4202 	case DISK_ZONE_REPORT_ZONES: {
4203 		uint32_t avail_len;
4204 		struct disk_zone_report *rep;
4205 		struct scsi_report_zones_hdr *hdr;
4206 		struct scsi_report_zones_desc *desc;
4207 		struct disk_zone_rep_entry *entry;
4208 		uint32_t hdr_len, num_avail;
4209 		uint32_t num_to_fill, i;
4210 		int ata;
4211 
4212 		rep = &bp->bio_zone.zone_params.report;
4213 		avail_len = ccb->csio.dxfer_len - ccb->csio.resid;
4214 		/*
4215 		 * Note that bio_resid isn't normally used for zone
4216 		 * commands, but it is used by devstat_end_transaction_bio()
4217 		 * to determine how much data was transferred.  Because
4218 		 * the size of the SCSI/ATA data structures is different
4219 		 * than the size of the BIO interface structures, the
4220 		 * amount of data actually transferred from the drive will
4221 		 * be different than the amount of data transferred to
4222 		 * the user.
4223 		 */
4224 		bp->bio_resid = ccb->csio.resid;
4225 		hdr = (struct scsi_report_zones_hdr *)ccb->csio.data_ptr;
4226 		if (avail_len < sizeof(*hdr)) {
4227 			/*
4228 			 * Is there a better error than EIO here?  We asked
4229 			 * for at least the header, and we got less than
4230 			 * that.
4231 			 */
4232 			bp->bio_error = EIO;
4233 			bp->bio_flags |= BIO_ERROR;
4234 			bp->bio_resid = bp->bio_bcount;
4235 			break;
4236 		}
4237 
4238 		if (softc->zone_interface == DA_ZONE_IF_ATA_PASS)
4239 			ata = 1;
4240 		else
4241 			ata = 0;
4242 
4243 		hdr_len = ata ? le32dec(hdr->length) :
4244 				scsi_4btoul(hdr->length);
4245 		if (hdr_len > 0)
4246 			rep->entries_available = hdr_len / sizeof(*desc);
4247 		else
4248 			rep->entries_available = 0;
4249 		/*
4250 		 * NOTE: using the same values for the BIO version of the
4251 		 * same field as the SCSI/ATA values.  This means we could
4252 		 * get some additional values that aren't defined in bio.h
4253 		 * if more values of the same field are defined later.
4254 		 */
4255 		rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
4256 		rep->header.maximum_lba = ata ?  le64dec(hdr->maximum_lba) :
4257 					  scsi_8btou64(hdr->maximum_lba);
4258 		/*
4259 		 * If the drive reports no entries that match the query,
4260 		 * we're done.
4261 		 */
4262 		if (hdr_len == 0) {
4263 			rep->entries_filled = 0;
4264 			break;
4265 		}
4266 
4267 		num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
4268 				hdr_len / sizeof(*desc));
4269 		/*
4270 		 * If the drive didn't return any data, then we're done.
4271 		 */
4272 		if (num_avail == 0) {
4273 			rep->entries_filled = 0;
4274 			break;
4275 		}
4276 
4277 		num_to_fill = min(num_avail, rep->entries_allocated);
4278 		/*
4279 		 * If the user didn't allocate any entries for us to fill,
4280 		 * we're done.
4281 		 */
4282 		if (num_to_fill == 0) {
4283 			rep->entries_filled = 0;
4284 			break;
4285 		}
4286 
4287 		for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
4288 		     i < num_to_fill; i++, desc++, entry++) {
4289 			/*
4290 			 * NOTE: we're mapping the values here directly
4291 			 * from the SCSI/ATA bit definitions to the bio.h
4292 			 * definitons.  There is also a warning in
4293 			 * disk_zone.h, but the impact is that if
4294 			 * additional values are added in the SCSI/ATA
4295 			 * specs these will be visible to consumers of
4296 			 * this interface.
4297 			 */
4298 			entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
4299 			entry->zone_condition =
4300 			    (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
4301 			    SRZ_ZONE_COND_SHIFT;
4302 			entry->zone_flags |= desc->zone_flags &
4303 			    (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
4304 			entry->zone_length =
4305 			    ata ? le64dec(desc->zone_length) :
4306 				  scsi_8btou64(desc->zone_length);
4307 			entry->zone_start_lba =
4308 			    ata ? le64dec(desc->zone_start_lba) :
4309 				  scsi_8btou64(desc->zone_start_lba);
4310 			entry->write_pointer_lba =
4311 			    ata ? le64dec(desc->write_pointer_lba) :
4312 				  scsi_8btou64(desc->write_pointer_lba);
4313 		}
4314 		rep->entries_filled = num_to_fill;
4315 		break;
4316 	}
4317 	case DISK_ZONE_GET_PARAMS:
4318 	default:
4319 		/*
4320 		 * In theory we should not get a GET_PARAMS bio, since it
4321 		 * should be handled without queueing the command to the
4322 		 * drive.
4323 		 */
4324 		panic("%s: Invalid zone command %d", __func__,
4325 		    bp->bio_zone.zone_cmd);
4326 		break;
4327 	}
4328 
4329 	if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
4330 		free(ccb->csio.data_ptr, M_SCSIDA);
4331 }
4332 
4333 static void
4334 dadone(struct cam_periph *periph, union ccb *done_ccb)
4335 {
4336 	struct bio *bp, *bp1;
4337 	struct da_softc *softc;
4338 	struct ccb_scsiio *csio;
4339 	u_int32_t  priority;
4340 	da_ccb_state state;
4341 
4342 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone\n"));
4343 
4344 	softc = (struct da_softc *)periph->softc;
4345 	priority = done_ccb->ccb_h.pinfo.priority;
4346 	csio = &done_ccb->csio;
4347 
4348 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4349 	if (csio->bio != NULL)
4350 		biotrack(csio->bio, __func__);
4351 #endif
4352 	state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
4353 
4354 	cam_periph_lock(periph);
4355 	bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
4356 	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
4357 		int error;
4358 		int sf;
4359 
4360 		if ((csio->ccb_h.ccb_state & DA_CCB_RETRY_UA) != 0)
4361 			sf = SF_RETRY_UA;
4362 		else
4363 			sf = 0;
4364 
4365 		error = daerror(done_ccb, CAM_RETRY_SELTO, sf);
4366 		if (error == ERESTART) {
4367 			/* A retry was scheduled, so just return. */
4368 			cam_periph_unlock(periph);
4369 			return;
4370 		}
4371 		bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
4372 		if (error != 0) {
4373 			int queued_error;
4374 
4375 			/*
4376 			 * return all queued I/O with EIO, so that
4377 			 * the client can retry these I/Os in the
4378 			 * proper order should it attempt to recover.
4379 			 */
4380 			queued_error = EIO;
4381 
4382 			if (error == ENXIO
4383 			 && (softc->flags & DA_FLAG_PACK_INVALID)== 0) {
4384 				/*
4385 				 * Catastrophic error.  Mark our pack as
4386 				 * invalid.
4387 				 *
4388 				 * XXX See if this is really a media
4389 				 * XXX change first?
4390 				 */
4391 				xpt_print(periph->path, "Invalidating pack\n");
4392 				softc->flags |= DA_FLAG_PACK_INVALID;
4393 #ifdef CAM_IO_STATS
4394 				softc->invalidations++;
4395 #endif
4396 				queued_error = ENXIO;
4397 			}
4398 			cam_iosched_flush(softc->cam_iosched, NULL,
4399 			   queued_error);
4400 			if (bp != NULL) {
4401 				bp->bio_error = error;
4402 				bp->bio_resid = bp->bio_bcount;
4403 				bp->bio_flags |= BIO_ERROR;
4404 			}
4405 		} else if (bp != NULL) {
4406 			if (state == DA_CCB_DELETE)
4407 				bp->bio_resid = 0;
4408 			else
4409 				bp->bio_resid = csio->resid;
4410 			bp->bio_error = 0;
4411 			if (bp->bio_resid != 0)
4412 				bp->bio_flags |= BIO_ERROR;
4413 		}
4414 		if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
4415 			cam_release_devq(done_ccb->ccb_h.path,
4416 					 /*relsim_flags*/0,
4417 					 /*reduction*/0,
4418 					 /*timeout*/0,
4419 					 /*getcount_only*/0);
4420 	} else if (bp != NULL) {
4421 		if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
4422 			panic("REQ_CMP with QFRZN");
4423 		if (bp->bio_cmd == BIO_ZONE)
4424 			dazonedone(periph, done_ccb);
4425 		else if (state == DA_CCB_DELETE)
4426 			bp->bio_resid = 0;
4427 		else
4428 			bp->bio_resid = csio->resid;
4429 		if ((csio->resid > 0) && (bp->bio_cmd != BIO_ZONE))
4430 			bp->bio_flags |= BIO_ERROR;
4431 		if (softc->error_inject != 0) {
4432 			bp->bio_error = softc->error_inject;
4433 			bp->bio_resid = bp->bio_bcount;
4434 			bp->bio_flags |= BIO_ERROR;
4435 			softc->error_inject = 0;
4436 		}
4437 	}
4438 
4439 	if (bp != NULL)
4440 		biotrack(bp, __func__);
4441 	LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
4442 	if (LIST_EMPTY(&softc->pending_ccbs))
4443 		softc->flags |= DA_FLAG_WAS_OTAG;
4444 
4445 	/*
4446 	 * We need to call cam_iosched before we call biodone so that we
4447 	 * don't measure any activity that happens in the completion
4448 	 * routine, which in the case of sendfile can be quite
4449 	 * extensive.
4450 	 */
4451 	cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
4452 	xpt_release_ccb(done_ccb);
4453 	if (state == DA_CCB_DELETE) {
4454 		TAILQ_HEAD(, bio) queue;
4455 
4456 		TAILQ_INIT(&queue);
4457 		TAILQ_CONCAT(&queue, &softc->delete_run_queue.queue, bio_queue);
4458 		softc->delete_run_queue.insert_point = NULL;
4459 		/*
4460 		 * Normally, the xpt_release_ccb() above would make sure
4461 		 * that when we have more work to do, that work would
4462 		 * get kicked off. However, we specifically keep
4463 		 * delete_running set to 0 before the call above to
4464 		 * allow other I/O to progress when many BIO_DELETE
4465 		 * requests are pushed down. We set delete_running to 0
4466 		 * and call daschedule again so that we don't stall if
4467 		 * there are no other I/Os pending apart from BIO_DELETEs.
4468 		 */
4469 		cam_iosched_trim_done(softc->cam_iosched);
4470 		daschedule(periph);
4471 		cam_periph_unlock(periph);
4472 		while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
4473 			TAILQ_REMOVE(&queue, bp1, bio_queue);
4474 			bp1->bio_error = bp->bio_error;
4475 			if (bp->bio_flags & BIO_ERROR) {
4476 				bp1->bio_flags |= BIO_ERROR;
4477 				bp1->bio_resid = bp1->bio_bcount;
4478 			} else
4479 				bp1->bio_resid = 0;
4480 			biodone(bp1);
4481 		}
4482 	} else {
4483 		daschedule(periph);
4484 		cam_periph_unlock(periph);
4485 	}
4486 	if (bp != NULL)
4487 		biodone(bp);
4488 	return;
4489 }
4490 
4491 static void
4492 dadone_probewp(struct cam_periph *periph, union ccb *done_ccb)
4493 {
4494 	struct scsi_mode_header_6 *mode_hdr6;
4495 	struct scsi_mode_header_10 *mode_hdr10;
4496 	struct da_softc *softc;
4497 	struct ccb_scsiio *csio;
4498 	u_int32_t  priority;
4499 	uint8_t dev_spec;
4500 
4501 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probewp\n"));
4502 
4503 	softc = (struct da_softc *)periph->softc;
4504 	priority = done_ccb->ccb_h.pinfo.priority;
4505 	csio = &done_ccb->csio;
4506 
4507 	if (softc->minimum_cmd_size > 6) {
4508 		mode_hdr10 = (struct scsi_mode_header_10 *)csio->data_ptr;
4509 		dev_spec = mode_hdr10->dev_spec;
4510 	} else {
4511 		mode_hdr6 = (struct scsi_mode_header_6 *)csio->data_ptr;
4512 		dev_spec = mode_hdr6->dev_spec;
4513 	}
4514 	if (cam_ccb_status(done_ccb) == CAM_REQ_CMP) {
4515 		if ((dev_spec & 0x80) != 0)
4516 			softc->disk->d_flags |= DISKFLAG_WRITE_PROTECT;
4517 		else
4518 			softc->disk->d_flags &= ~DISKFLAG_WRITE_PROTECT;
4519 	} else {
4520 		int error;
4521 
4522 		error = daerror(done_ccb, CAM_RETRY_SELTO,
4523 				SF_RETRY_UA|SF_NO_PRINT);
4524 		if (error == ERESTART)
4525 			return;
4526 		else if (error != 0) {
4527 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4528 				/* Don't wedge this device's queue */
4529 				cam_release_devq(done_ccb->ccb_h.path,
4530 						 /*relsim_flags*/0,
4531 						 /*reduction*/0,
4532 						 /*timeout*/0,
4533 						 /*getcount_only*/0);
4534 			}
4535 		}
4536 	}
4537 
4538 	free(csio->data_ptr, M_SCSIDA);
4539 	xpt_release_ccb(done_ccb);
4540 	if ((softc->flags & DA_FLAG_CAN_RC16) != 0)
4541 		softc->state = DA_STATE_PROBE_RC16;
4542 	else
4543 		softc->state = DA_STATE_PROBE_RC;
4544 	xpt_schedule(periph, priority);
4545 	return;
4546 }
4547 
4548 static void
4549 dadone_proberc(struct cam_periph *periph, union ccb *done_ccb)
4550 {
4551 	struct scsi_read_capacity_data *rdcap;
4552 	struct scsi_read_capacity_data_long *rcaplong;
4553 	struct da_softc *softc;
4554 	struct ccb_scsiio *csio;
4555 	da_ccb_state state;
4556 	char *announce_buf;
4557 	u_int32_t  priority;
4558 	int lbp;
4559 
4560 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_proberc\n"));
4561 
4562 	softc = (struct da_softc *)periph->softc;
4563 	priority = done_ccb->ccb_h.pinfo.priority;
4564 	csio = &done_ccb->csio;
4565 	state = csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK;
4566 
4567 	lbp = 0;
4568 	rdcap = NULL;
4569 	rcaplong = NULL;
4570 	/* XXX TODO: can this be a malloc? */
4571 	announce_buf = softc->announce_temp;
4572 	bzero(announce_buf, DA_ANNOUNCETMP_SZ);
4573 
4574 	if (state == DA_CCB_PROBE_RC)
4575 		rdcap =(struct scsi_read_capacity_data *)csio->data_ptr;
4576 	else
4577 		rcaplong = (struct scsi_read_capacity_data_long *)
4578 			csio->data_ptr;
4579 
4580 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4581 		struct disk_params *dp;
4582 		uint32_t block_size;
4583 		uint64_t maxsector;
4584 		u_int lalba;	/* Lowest aligned LBA. */
4585 
4586 		if (state == DA_CCB_PROBE_RC) {
4587 			block_size = scsi_4btoul(rdcap->length);
4588 			maxsector = scsi_4btoul(rdcap->addr);
4589 			lalba = 0;
4590 
4591 			/*
4592 			 * According to SBC-2, if the standard 10
4593 			 * byte READ CAPACITY command returns 2^32,
4594 			 * we should issue the 16 byte version of
4595 			 * the command, since the device in question
4596 			 * has more sectors than can be represented
4597 			 * with the short version of the command.
4598 			 */
4599 			if (maxsector == 0xffffffff) {
4600 				free(rdcap, M_SCSIDA);
4601 				xpt_release_ccb(done_ccb);
4602 				softc->state = DA_STATE_PROBE_RC16;
4603 				xpt_schedule(periph, priority);
4604 				return;
4605 			}
4606 		} else {
4607 			block_size = scsi_4btoul(rcaplong->length);
4608 			maxsector = scsi_8btou64(rcaplong->addr);
4609 			lalba = scsi_2btoul(rcaplong->lalba_lbp);
4610 		}
4611 
4612 		/*
4613 		 * Because GEOM code just will panic us if we
4614 		 * give them an 'illegal' value we'll avoid that
4615 		 * here.
4616 		 */
4617 		if (block_size == 0) {
4618 			block_size = 512;
4619 			if (maxsector == 0)
4620 				maxsector = -1;
4621 		}
4622 		if (block_size >= MAXPHYS) {
4623 			xpt_print(periph->path,
4624 			    "unsupportable block size %ju\n",
4625 			    (uintmax_t) block_size);
4626 			announce_buf = NULL;
4627 			cam_periph_invalidate(periph);
4628 		} else {
4629 			/*
4630 			 * We pass rcaplong into dasetgeom(),
4631 			 * because it will only use it if it is
4632 			 * non-NULL.
4633 			 */
4634 			dasetgeom(periph, block_size, maxsector,
4635 				  rcaplong, sizeof(*rcaplong));
4636 			lbp = (lalba & SRC16_LBPME_A);
4637 			dp = &softc->params;
4638 			snprintf(announce_buf, DA_ANNOUNCETMP_SZ,
4639 			    "%juMB (%ju %u byte sectors)",
4640 			    ((uintmax_t)dp->secsize * dp->sectors) /
4641 			     (1024 * 1024),
4642 			    (uintmax_t)dp->sectors, dp->secsize);
4643 		}
4644 	} else {
4645 		int error;
4646 
4647 		/*
4648 		 * Retry any UNIT ATTENTION type errors.  They
4649 		 * are expected at boot.
4650 		 */
4651 		error = daerror(done_ccb, CAM_RETRY_SELTO,
4652 				SF_RETRY_UA|SF_NO_PRINT);
4653 		if (error == ERESTART) {
4654 			/*
4655 			 * A retry was scheuled, so
4656 			 * just return.
4657 			 */
4658 			return;
4659 		} else if (error != 0) {
4660 			int asc, ascq;
4661 			int sense_key, error_code;
4662 			int have_sense;
4663 			cam_status status;
4664 			struct ccb_getdev cgd;
4665 
4666 			/* Don't wedge this device's queue */
4667 			status = done_ccb->ccb_h.status;
4668 			if ((status & CAM_DEV_QFRZN) != 0)
4669 				cam_release_devq(done_ccb->ccb_h.path,
4670 						 /*relsim_flags*/0,
4671 						 /*reduction*/0,
4672 						 /*timeout*/0,
4673 						 /*getcount_only*/0);
4674 
4675 
4676 			xpt_setup_ccb(&cgd.ccb_h, done_ccb->ccb_h.path,
4677 				      CAM_PRIORITY_NORMAL);
4678 			cgd.ccb_h.func_code = XPT_GDEV_TYPE;
4679 			xpt_action((union ccb *)&cgd);
4680 
4681 			if (scsi_extract_sense_ccb(done_ccb,
4682 			    &error_code, &sense_key, &asc, &ascq))
4683 				have_sense = TRUE;
4684 			else
4685 				have_sense = FALSE;
4686 
4687 			/*
4688 			 * If we tried READ CAPACITY(16) and failed,
4689 			 * fallback to READ CAPACITY(10).
4690 			 */
4691 			if ((state == DA_CCB_PROBE_RC16) &&
4692 			    (softc->flags & DA_FLAG_CAN_RC16) &&
4693 			    (((csio->ccb_h.status & CAM_STATUS_MASK) ==
4694 				CAM_REQ_INVALID) ||
4695 			     ((have_sense) &&
4696 			      (error_code == SSD_CURRENT_ERROR ||
4697 			       error_code == SSD_DESC_CURRENT_ERROR) &&
4698 			      (sense_key == SSD_KEY_ILLEGAL_REQUEST)))) {
4699 				cam_periph_assert(periph, MA_OWNED);
4700 				softc->flags &= ~DA_FLAG_CAN_RC16;
4701 				free(rdcap, M_SCSIDA);
4702 				xpt_release_ccb(done_ccb);
4703 				softc->state = DA_STATE_PROBE_RC;
4704 				xpt_schedule(periph, priority);
4705 				return;
4706 			}
4707 
4708 			/*
4709 			 * Attach to anything that claims to be a
4710 			 * direct access or optical disk device,
4711 			 * as long as it doesn't return a "Logical
4712 			 * unit not supported" (0x25) error.
4713 			 * "Internal Target Failure" (0x44) is also
4714 			 * special and typically means that the
4715 			 * device is a SATA drive behind a SATL
4716 			 * translation that's fallen into a
4717 			 * terminally fatal state.
4718 			 */
4719 			if ((have_sense)
4720 			 && (asc != 0x25) && (asc != 0x44)
4721 			 && (error_code == SSD_CURRENT_ERROR
4722 			  || error_code == SSD_DESC_CURRENT_ERROR)) {
4723 				const char *sense_key_desc;
4724 				const char *asc_desc;
4725 
4726 				dasetgeom(periph, 512, -1, NULL, 0);
4727 				scsi_sense_desc(sense_key, asc, ascq,
4728 						&cgd.inq_data, &sense_key_desc,
4729 						&asc_desc);
4730 				snprintf(announce_buf, DA_ANNOUNCETMP_SZ,
4731 				    "Attempt to query device "
4732 				    "size failed: %s, %s",
4733 				    sense_key_desc, asc_desc);
4734 			} else {
4735 				if (have_sense)
4736 					scsi_sense_print(&done_ccb->csio);
4737 				else {
4738 					xpt_print(periph->path,
4739 					    "got CAM status %#x\n",
4740 					    done_ccb->ccb_h.status);
4741 				}
4742 
4743 				xpt_print(periph->path, "fatal error, "
4744 				    "failed to attach to device\n");
4745 
4746 				announce_buf = NULL;
4747 
4748 				/*
4749 				 * Free up resources.
4750 				 */
4751 				cam_periph_invalidate(periph);
4752 			}
4753 		}
4754 	}
4755 	free(csio->data_ptr, M_SCSIDA);
4756 	if (announce_buf != NULL &&
4757 	    ((softc->flags & DA_FLAG_ANNOUNCED) == 0)) {
4758 		struct sbuf sb;
4759 
4760 		sbuf_new(&sb, softc->announcebuf, DA_ANNOUNCE_SZ,
4761 		    SBUF_FIXEDLEN);
4762 		xpt_announce_periph_sbuf(periph, &sb, announce_buf);
4763 		xpt_announce_quirks_sbuf(periph, &sb, softc->quirks,
4764 		    DA_Q_BIT_STRING);
4765 		sbuf_finish(&sb);
4766 		sbuf_putbuf(&sb);
4767 
4768 		/*
4769 		 * Create our sysctl variables, now that we know
4770 		 * we have successfully attached.
4771 		 */
4772 		/* increase the refcount */
4773 		if (da_periph_acquire(periph, DA_REF_SYSCTL) == 0) {
4774 			taskqueue_enqueue(taskqueue_thread,
4775 					  &softc->sysctl_task);
4776 		} else {
4777 			/* XXX This message is useless! */
4778 			xpt_print(periph->path, "fatal error, "
4779 			    "could not acquire reference count\n");
4780 		}
4781 	}
4782 
4783 	/* We already probed the device. */
4784 	if (softc->flags & DA_FLAG_PROBED) {
4785 		daprobedone(periph, done_ccb);
4786 		return;
4787 	}
4788 
4789 	/* Ensure re-probe doesn't see old delete. */
4790 	softc->delete_available = 0;
4791 	dadeleteflag(softc, DA_DELETE_ZERO, 1);
4792 	if (lbp && (softc->quirks & DA_Q_NO_UNMAP) == 0) {
4793 		/*
4794 		 * Based on older SBC-3 spec revisions
4795 		 * any of the UNMAP methods "may" be
4796 		 * available via LBP given this flag so
4797 		 * we flag all of them as available and
4798 		 * then remove those which further
4799 		 * probes confirm aren't available
4800 		 * later.
4801 		 *
4802 		 * We could also check readcap(16) p_type
4803 		 * flag to exclude one or more invalid
4804 		 * write same (X) types here
4805 		 */
4806 		dadeleteflag(softc, DA_DELETE_WS16, 1);
4807 		dadeleteflag(softc, DA_DELETE_WS10, 1);
4808 		dadeleteflag(softc, DA_DELETE_UNMAP, 1);
4809 
4810 		xpt_release_ccb(done_ccb);
4811 		softc->state = DA_STATE_PROBE_LBP;
4812 		xpt_schedule(periph, priority);
4813 		return;
4814 	}
4815 
4816 	xpt_release_ccb(done_ccb);
4817 	softc->state = DA_STATE_PROBE_BDC;
4818 	xpt_schedule(periph, priority);
4819 	return;
4820 }
4821 
4822 static void
4823 dadone_probelbp(struct cam_periph *periph, union ccb *done_ccb)
4824 {
4825 	struct scsi_vpd_logical_block_prov *lbp;
4826 	struct da_softc *softc;
4827 	struct ccb_scsiio *csio;
4828 	u_int32_t  priority;
4829 
4830 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probelbp\n"));
4831 
4832 	softc = (struct da_softc *)periph->softc;
4833 	priority = done_ccb->ccb_h.pinfo.priority;
4834 	csio = &done_ccb->csio;
4835 	lbp = (struct scsi_vpd_logical_block_prov *)csio->data_ptr;
4836 
4837 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4838 		/*
4839 		 * T10/1799-D Revision 31 states at least one of these
4840 		 * must be supported but we don't currently enforce this.
4841 		 */
4842 		dadeleteflag(softc, DA_DELETE_WS16,
4843 		     (lbp->flags & SVPD_LBP_WS16));
4844 		dadeleteflag(softc, DA_DELETE_WS10,
4845 			     (lbp->flags & SVPD_LBP_WS10));
4846 		dadeleteflag(softc, DA_DELETE_UNMAP,
4847 			     (lbp->flags & SVPD_LBP_UNMAP));
4848 	} else {
4849 		int error;
4850 		error = daerror(done_ccb, CAM_RETRY_SELTO,
4851 				SF_RETRY_UA|SF_NO_PRINT);
4852 		if (error == ERESTART)
4853 			return;
4854 		else if (error != 0) {
4855 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4856 				/* Don't wedge this device's queue */
4857 				cam_release_devq(done_ccb->ccb_h.path,
4858 						 /*relsim_flags*/0,
4859 						 /*reduction*/0,
4860 						 /*timeout*/0,
4861 						 /*getcount_only*/0);
4862 			}
4863 
4864 			/*
4865 			 * Failure indicates we don't support any SBC-3
4866 			 * delete methods with UNMAP
4867 			 */
4868 		}
4869 	}
4870 
4871 	free(lbp, M_SCSIDA);
4872 	xpt_release_ccb(done_ccb);
4873 	softc->state = DA_STATE_PROBE_BLK_LIMITS;
4874 	xpt_schedule(periph, priority);
4875 	return;
4876 }
4877 
4878 static void
4879 dadone_probeblklimits(struct cam_periph *periph, union ccb *done_ccb)
4880 {
4881 	struct scsi_vpd_block_limits *block_limits;
4882 	struct da_softc *softc;
4883 	struct ccb_scsiio *csio;
4884 	u_int32_t  priority;
4885 
4886 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeblklimits\n"));
4887 
4888 	softc = (struct da_softc *)periph->softc;
4889 	priority = done_ccb->ccb_h.pinfo.priority;
4890 	csio = &done_ccb->csio;
4891 	block_limits = (struct scsi_vpd_block_limits *)csio->data_ptr;
4892 
4893 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4894 		uint32_t max_txfer_len = scsi_4btoul(
4895 			block_limits->max_txfer_len);
4896 		uint32_t max_unmap_lba_cnt = scsi_4btoul(
4897 			block_limits->max_unmap_lba_cnt);
4898 		uint32_t max_unmap_blk_cnt = scsi_4btoul(
4899 			block_limits->max_unmap_blk_cnt);
4900 		uint32_t unmap_gran = scsi_4btoul(
4901 			block_limits->opt_unmap_grain);
4902 		uint32_t unmap_gran_align = scsi_4btoul(
4903 			block_limits->unmap_grain_align);
4904 		uint64_t ws_max_blks = scsi_8btou64(
4905 			block_limits->max_write_same_length);
4906 
4907 		if (max_txfer_len != 0) {
4908 			softc->disk->d_maxsize = MIN(softc->maxio,
4909 			    (off_t)max_txfer_len * softc->params.secsize);
4910 		}
4911 
4912 		/*
4913 		 * We should already support UNMAP but we check lba
4914 		 * and block count to be sure
4915 		 */
4916 		if (max_unmap_lba_cnt != 0x00L &&
4917 		    max_unmap_blk_cnt != 0x00L) {
4918 			softc->unmap_max_lba = max_unmap_lba_cnt;
4919 			softc->unmap_max_ranges = min(max_unmap_blk_cnt,
4920 				UNMAP_MAX_RANGES);
4921 			if (unmap_gran > 1) {
4922 				softc->unmap_gran = unmap_gran;
4923 				if (unmap_gran_align & 0x80000000) {
4924 					softc->unmap_gran_align =
4925 					    unmap_gran_align & 0x7fffffff;
4926 				}
4927 			}
4928 		} else {
4929 			/*
4930 			 * Unexpected UNMAP limits which means the
4931 			 * device doesn't actually support UNMAP
4932 			 */
4933 			dadeleteflag(softc, DA_DELETE_UNMAP, 0);
4934 		}
4935 
4936 		if (ws_max_blks != 0x00L)
4937 			softc->ws_max_blks = ws_max_blks;
4938 	} else {
4939 		int error;
4940 		error = daerror(done_ccb, CAM_RETRY_SELTO,
4941 				SF_RETRY_UA|SF_NO_PRINT);
4942 		if (error == ERESTART)
4943 			return;
4944 		else if (error != 0) {
4945 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4946 				/* Don't wedge this device's queue */
4947 				cam_release_devq(done_ccb->ccb_h.path,
4948 						 /*relsim_flags*/0,
4949 						 /*reduction*/0,
4950 						 /*timeout*/0,
4951 						 /*getcount_only*/0);
4952 			}
4953 
4954 			/*
4955 			 * Failure here doesn't mean UNMAP is not
4956 			 * supported as this is an optional page.
4957 			 */
4958 			softc->unmap_max_lba = 1;
4959 			softc->unmap_max_ranges = 1;
4960 		}
4961 	}
4962 
4963 	free(block_limits, M_SCSIDA);
4964 	xpt_release_ccb(done_ccb);
4965 	softc->state = DA_STATE_PROBE_BDC;
4966 	xpt_schedule(periph, priority);
4967 	return;
4968 }
4969 
4970 static void
4971 dadone_probebdc(struct cam_periph *periph, union ccb *done_ccb)
4972 {
4973 	struct scsi_vpd_block_device_characteristics *bdc;
4974 	struct da_softc *softc;
4975 	struct ccb_scsiio *csio;
4976 	u_int32_t  priority;
4977 
4978 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probebdc\n"));
4979 
4980 	softc = (struct da_softc *)periph->softc;
4981 	priority = done_ccb->ccb_h.pinfo.priority;
4982 	csio = &done_ccb->csio;
4983 	bdc = (struct scsi_vpd_block_device_characteristics *)csio->data_ptr;
4984 
4985 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
4986 		uint32_t valid_len;
4987 
4988 		/*
4989 		 * Disable queue sorting for non-rotational media
4990 		 * by default.
4991 		 */
4992 		u_int16_t old_rate = softc->disk->d_rotation_rate;
4993 
4994 		valid_len = csio->dxfer_len - csio->resid;
4995 		if (SBDC_IS_PRESENT(bdc, valid_len,
4996 		    medium_rotation_rate)) {
4997 			softc->disk->d_rotation_rate =
4998 				scsi_2btoul(bdc->medium_rotation_rate);
4999 			if (softc->disk->d_rotation_rate ==
5000 			    SVPD_BDC_RATE_NON_ROTATING) {
5001 				cam_iosched_set_sort_queue(
5002 				    softc->cam_iosched, 0);
5003 				softc->rotating = 0;
5004 			}
5005 			if (softc->disk->d_rotation_rate != old_rate) {
5006 				disk_attr_changed(softc->disk,
5007 				    "GEOM::rotation_rate", M_NOWAIT);
5008 			}
5009 		}
5010 		if ((SBDC_IS_PRESENT(bdc, valid_len, flags))
5011 		 && (softc->zone_mode == DA_ZONE_NONE)) {
5012 			int ata_proto;
5013 
5014 			if (scsi_vpd_supported_page(periph,
5015 			    SVPD_ATA_INFORMATION))
5016 				ata_proto = 1;
5017 			else
5018 				ata_proto = 0;
5019 
5020 			/*
5021 			 * The Zoned field will only be set for
5022 			 * Drive Managed and Host Aware drives.  If
5023 			 * they are Host Managed, the device type
5024 			 * in the standard INQUIRY data should be
5025 			 * set to T_ZBC_HM (0x14).
5026 			 */
5027 			if ((bdc->flags & SVPD_ZBC_MASK) ==
5028 			     SVPD_HAW_ZBC) {
5029 				softc->zone_mode = DA_ZONE_HOST_AWARE;
5030 				softc->zone_interface = (ata_proto) ?
5031 				   DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI;
5032 			} else if ((bdc->flags & SVPD_ZBC_MASK) ==
5033 			     SVPD_DM_ZBC) {
5034 				softc->zone_mode =DA_ZONE_DRIVE_MANAGED;
5035 				softc->zone_interface = (ata_proto) ?
5036 				   DA_ZONE_IF_ATA_SAT : DA_ZONE_IF_SCSI;
5037 			} else if ((bdc->flags & SVPD_ZBC_MASK) !=
5038 				  SVPD_ZBC_NR) {
5039 				xpt_print(periph->path, "Unknown zoned "
5040 				    "type %#x",
5041 				    bdc->flags & SVPD_ZBC_MASK);
5042 			}
5043 		}
5044 	} else {
5045 		int error;
5046 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5047 				SF_RETRY_UA|SF_NO_PRINT);
5048 		if (error == ERESTART)
5049 			return;
5050 		else if (error != 0) {
5051 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5052 				/* Don't wedge this device's queue */
5053 				cam_release_devq(done_ccb->ccb_h.path,
5054 						 /*relsim_flags*/0,
5055 						 /*reduction*/0,
5056 						 /*timeout*/0,
5057 						 /*getcount_only*/0);
5058 			}
5059 		}
5060 	}
5061 
5062 	free(bdc, M_SCSIDA);
5063 	xpt_release_ccb(done_ccb);
5064 	softc->state = DA_STATE_PROBE_ATA;
5065 	xpt_schedule(periph, priority);
5066 	return;
5067 }
5068 
5069 static void
5070 dadone_probeata(struct cam_periph *periph, union ccb *done_ccb)
5071 {
5072 	struct ata_params *ata_params;
5073 	struct ccb_scsiio *csio;
5074 	struct da_softc *softc;
5075 	u_int32_t  priority;
5076 	int continue_probe;
5077 	int error, i;
5078 	int16_t *ptr;
5079 
5080 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeata\n"));
5081 
5082 	softc = (struct da_softc *)periph->softc;
5083 	priority = done_ccb->ccb_h.pinfo.priority;
5084 	csio = &done_ccb->csio;
5085 	ata_params = (struct ata_params *)csio->data_ptr;
5086 	ptr = (uint16_t *)ata_params;
5087 	continue_probe = 0;
5088 	error = 0;
5089 
5090 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5091 		uint16_t old_rate;
5092 
5093 		for (i = 0; i < sizeof(*ata_params) / 2; i++)
5094 			ptr[i] = le16toh(ptr[i]);
5095 		if (ata_params->support_dsm & ATA_SUPPORT_DSM_TRIM &&
5096 		    (softc->quirks & DA_Q_NO_UNMAP) == 0) {
5097 			dadeleteflag(softc, DA_DELETE_ATA_TRIM, 1);
5098 			if (ata_params->max_dsm_blocks != 0)
5099 				softc->trim_max_ranges = min(
5100 				  softc->trim_max_ranges,
5101 				  ata_params->max_dsm_blocks *
5102 				  ATA_DSM_BLK_RANGES);
5103 		}
5104 		/*
5105 		 * Disable queue sorting for non-rotational media
5106 		 * by default.
5107 		 */
5108 		old_rate = softc->disk->d_rotation_rate;
5109 		softc->disk->d_rotation_rate = ata_params->media_rotation_rate;
5110 		if (softc->disk->d_rotation_rate == ATA_RATE_NON_ROTATING) {
5111 			cam_iosched_set_sort_queue(softc->cam_iosched, 0);
5112 			softc->rotating = 0;
5113 		}
5114 		if (softc->disk->d_rotation_rate != old_rate) {
5115 			disk_attr_changed(softc->disk,
5116 			    "GEOM::rotation_rate", M_NOWAIT);
5117 		}
5118 
5119 		cam_periph_assert(periph, MA_OWNED);
5120 		if (ata_params->capabilities1 & ATA_SUPPORT_DMA)
5121 			softc->flags |= DA_FLAG_CAN_ATA_DMA;
5122 
5123 		if (ata_params->support.extension & ATA_SUPPORT_GENLOG)
5124 			softc->flags |= DA_FLAG_CAN_ATA_LOG;
5125 
5126 		/*
5127 		 * At this point, if we have a SATA host aware drive,
5128 		 * we communicate via ATA passthrough unless the
5129 		 * SAT layer supports ZBC -> ZAC translation.  In
5130 		 * that case,
5131 		 *
5132 		 * XXX KDM figure out how to detect a host managed
5133 		 * SATA drive.
5134 		 */
5135 		if (softc->zone_mode == DA_ZONE_NONE) {
5136 			/*
5137 			 * Note that we don't override the zone
5138 			 * mode or interface if it has already been
5139 			 * set.  This is because it has either been
5140 			 * set as a quirk, or when we probed the
5141 			 * SCSI Block Device Characteristics page,
5142 			 * the zoned field was set.  The latter
5143 			 * means that the SAT layer supports ZBC to
5144 			 * ZAC translation, and we would prefer to
5145 			 * use that if it is available.
5146 			 */
5147 			if ((ata_params->support3 &
5148 			    ATA_SUPPORT_ZONE_MASK) ==
5149 			    ATA_SUPPORT_ZONE_HOST_AWARE) {
5150 				softc->zone_mode = DA_ZONE_HOST_AWARE;
5151 				softc->zone_interface =
5152 				    DA_ZONE_IF_ATA_PASS;
5153 			} else if ((ata_params->support3 &
5154 				    ATA_SUPPORT_ZONE_MASK) ==
5155 				    ATA_SUPPORT_ZONE_DEV_MANAGED) {
5156 				softc->zone_mode =DA_ZONE_DRIVE_MANAGED;
5157 				softc->zone_interface = DA_ZONE_IF_ATA_PASS;
5158 			}
5159 		}
5160 
5161 	} else {
5162 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5163 				SF_RETRY_UA|SF_NO_PRINT);
5164 		if (error == ERESTART)
5165 			return;
5166 		else if (error != 0) {
5167 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5168 				/* Don't wedge this device's queue */
5169 				cam_release_devq(done_ccb->ccb_h.path,
5170 						 /*relsim_flags*/0,
5171 						 /*reduction*/0,
5172 						 /*timeout*/0,
5173 						 /*getcount_only*/0);
5174 			}
5175 		}
5176 	}
5177 
5178 	free(ata_params, M_SCSIDA);
5179 	if ((softc->zone_mode == DA_ZONE_HOST_AWARE)
5180 	 || (softc->zone_mode == DA_ZONE_HOST_MANAGED)) {
5181 		/*
5182 		 * If the ATA IDENTIFY failed, we could be talking
5183 		 * to a SCSI drive, although that seems unlikely,
5184 		 * since the drive did report that it supported the
5185 		 * ATA Information VPD page.  If the ATA IDENTIFY
5186 		 * succeeded, and the SAT layer doesn't support
5187 		 * ZBC -> ZAC translation, continue on to get the
5188 		 * directory of ATA logs, and complete the rest of
5189 		 * the ZAC probe.  If the SAT layer does support
5190 		 * ZBC -> ZAC translation, we want to use that,
5191 		 * and we'll probe the SCSI Zoned Block Device
5192 		 * Characteristics VPD page next.
5193 		 */
5194 		if ((error == 0)
5195 		 && (softc->flags & DA_FLAG_CAN_ATA_LOG)
5196 		 && (softc->zone_interface == DA_ZONE_IF_ATA_PASS))
5197 			softc->state = DA_STATE_PROBE_ATA_LOGDIR;
5198 		else
5199 			softc->state = DA_STATE_PROBE_ZONE;
5200 		continue_probe = 1;
5201 	}
5202 	if (continue_probe != 0) {
5203 		xpt_release_ccb(done_ccb);
5204 		xpt_schedule(periph, priority);
5205 		return;
5206 	} else
5207 		daprobedone(periph, done_ccb);
5208 	return;
5209 }
5210 
5211 static void
5212 dadone_probeatalogdir(struct cam_periph *periph, union ccb *done_ccb)
5213 {
5214 	struct da_softc *softc;
5215 	struct ccb_scsiio *csio;
5216 	u_int32_t  priority;
5217 	int error;
5218 
5219 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatalogdir\n"));
5220 
5221 	softc = (struct da_softc *)periph->softc;
5222 	priority = done_ccb->ccb_h.pinfo.priority;
5223 	csio = &done_ccb->csio;
5224 
5225 	cam_periph_lock(periph);
5226 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5227 		error = 0;
5228 		softc->valid_logdir_len = 0;
5229 		bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
5230 		softc->valid_logdir_len = csio->dxfer_len - csio->resid;
5231 		if (softc->valid_logdir_len > 0)
5232 			bcopy(csio->data_ptr, &softc->ata_logdir,
5233 			    min(softc->valid_logdir_len,
5234 				sizeof(softc->ata_logdir)));
5235 		/*
5236 		 * Figure out whether the Identify Device log is
5237 		 * supported.  The General Purpose log directory
5238 		 * has a header, and lists the number of pages
5239 		 * available for each GP log identified by the
5240 		 * offset into the list.
5241 		 */
5242 		if ((softc->valid_logdir_len >=
5243 		    ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
5244 		 && (le16dec(softc->ata_logdir.header) ==
5245 		     ATA_GP_LOG_DIR_VERSION)
5246 		 && (le16dec(&softc->ata_logdir.num_pages[
5247 		     (ATA_IDENTIFY_DATA_LOG *
5248 		     sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
5249 			softc->flags |= DA_FLAG_CAN_ATA_IDLOG;
5250 		} else {
5251 			softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG;
5252 		}
5253 	} else {
5254 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5255 				SF_RETRY_UA|SF_NO_PRINT);
5256 		if (error == ERESTART)
5257 			return;
5258 		else if (error != 0) {
5259 			/*
5260 			 * If we can't get the ATA log directory,
5261 			 * then ATA logs are effectively not
5262 			 * supported even if the bit is set in the
5263 			 * identify data.
5264 			 */
5265 			softc->flags &= ~(DA_FLAG_CAN_ATA_LOG |
5266 					  DA_FLAG_CAN_ATA_IDLOG);
5267 			if ((done_ccb->ccb_h.status &
5268 			     CAM_DEV_QFRZN) != 0) {
5269 				/* Don't wedge this device's queue */
5270 				cam_release_devq(done_ccb->ccb_h.path,
5271 						 /*relsim_flags*/0,
5272 						 /*reduction*/0,
5273 						 /*timeout*/0,
5274 						 /*getcount_only*/0);
5275 			}
5276 		}
5277 	}
5278 	cam_periph_unlock(periph);
5279 
5280 	free(csio->data_ptr, M_SCSIDA);
5281 
5282 	if ((error == 0)
5283 	 && (softc->flags & DA_FLAG_CAN_ATA_IDLOG)) {
5284 		softc->state = DA_STATE_PROBE_ATA_IDDIR;
5285 		xpt_release_ccb(done_ccb);
5286 		xpt_schedule(periph, priority);
5287 		return;
5288 	}
5289 	daprobedone(periph, done_ccb);
5290 	return;
5291 }
5292 
5293 static void
5294 dadone_probeataiddir(struct cam_periph *periph, union ccb *done_ccb)
5295 {
5296 	struct da_softc *softc;
5297 	struct ccb_scsiio *csio;
5298 	u_int32_t  priority;
5299 	int error;
5300 
5301 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeataiddir\n"));
5302 
5303 	softc = (struct da_softc *)periph->softc;
5304 	priority = done_ccb->ccb_h.pinfo.priority;
5305 	csio = &done_ccb->csio;
5306 
5307 	cam_periph_lock(periph);
5308 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5309 		off_t entries_offset, max_entries;
5310 		error = 0;
5311 
5312 		softc->valid_iddir_len = 0;
5313 		bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
5314 		softc->flags &= ~(DA_FLAG_CAN_ATA_SUPCAP |
5315 				  DA_FLAG_CAN_ATA_ZONE);
5316 		softc->valid_iddir_len = csio->dxfer_len - csio->resid;
5317 		if (softc->valid_iddir_len > 0)
5318 			bcopy(csio->data_ptr, &softc->ata_iddir,
5319 			    min(softc->valid_iddir_len,
5320 				sizeof(softc->ata_iddir)));
5321 
5322 		entries_offset =
5323 		    __offsetof(struct ata_identify_log_pages,entries);
5324 		max_entries = softc->valid_iddir_len - entries_offset;
5325 		if ((softc->valid_iddir_len > (entries_offset + 1))
5326 		 && (le64dec(softc->ata_iddir.header) == ATA_IDLOG_REVISION)
5327 		 && (softc->ata_iddir.entry_count > 0)) {
5328 			int num_entries, i;
5329 
5330 			num_entries = softc->ata_iddir.entry_count;
5331 			num_entries = min(num_entries,
5332 			   softc->valid_iddir_len - entries_offset);
5333 			for (i = 0; i < num_entries && i < max_entries; i++) {
5334 				if (softc->ata_iddir.entries[i] ==
5335 				    ATA_IDL_SUP_CAP)
5336 					softc->flags |= DA_FLAG_CAN_ATA_SUPCAP;
5337 				else if (softc->ata_iddir.entries[i] ==
5338 					 ATA_IDL_ZDI)
5339 					softc->flags |= DA_FLAG_CAN_ATA_ZONE;
5340 
5341 				if ((softc->flags & DA_FLAG_CAN_ATA_SUPCAP)
5342 				 && (softc->flags & DA_FLAG_CAN_ATA_ZONE))
5343 					break;
5344 			}
5345 		}
5346 	} else {
5347 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5348 				SF_RETRY_UA|SF_NO_PRINT);
5349 		if (error == ERESTART)
5350 			return;
5351 		else if (error != 0) {
5352 			/*
5353 			 * If we can't get the ATA Identify Data log
5354 			 * directory, then it effectively isn't
5355 			 * supported even if the ATA Log directory
5356 			 * a non-zero number of pages present for
5357 			 * this log.
5358 			 */
5359 			softc->flags &= ~DA_FLAG_CAN_ATA_IDLOG;
5360 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5361 				/* Don't wedge this device's queue */
5362 				cam_release_devq(done_ccb->ccb_h.path,
5363 						 /*relsim_flags*/0,
5364 						 /*reduction*/0,
5365 						 /*timeout*/0,
5366 						 /*getcount_only*/0);
5367 			}
5368 		}
5369 	}
5370 	cam_periph_unlock(periph);
5371 
5372 	free(csio->data_ptr, M_SCSIDA);
5373 
5374 	if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_SUPCAP)) {
5375 		softc->state = DA_STATE_PROBE_ATA_SUP;
5376 		xpt_release_ccb(done_ccb);
5377 		xpt_schedule(periph, priority);
5378 		return;
5379 	}
5380 	daprobedone(periph, done_ccb);
5381 	return;
5382 }
5383 
5384 static void
5385 dadone_probeatasup(struct cam_periph *periph, union ccb *done_ccb)
5386 {
5387 	struct da_softc *softc;
5388 	struct ccb_scsiio *csio;
5389 	u_int32_t  priority;
5390 	int error;
5391 
5392 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatasup\n"));
5393 
5394 	softc = (struct da_softc *)periph->softc;
5395 	priority = done_ccb->ccb_h.pinfo.priority;
5396 	csio = &done_ccb->csio;
5397 
5398 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5399 		uint32_t valid_len;
5400 		size_t needed_size;
5401 		struct ata_identify_log_sup_cap *sup_cap;
5402 		error = 0;
5403 
5404 		sup_cap = (struct ata_identify_log_sup_cap *)csio->data_ptr;
5405 		valid_len = csio->dxfer_len - csio->resid;
5406 		needed_size = __offsetof(struct ata_identify_log_sup_cap,
5407 		    sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
5408 		if (valid_len >= needed_size) {
5409 			uint64_t zoned, zac_cap;
5410 
5411 			zoned = le64dec(sup_cap->zoned_cap);
5412 			if (zoned & ATA_ZONED_VALID) {
5413 				/*
5414 				 * This should have already been
5415 				 * set, because this is also in the
5416 				 * ATA identify data.
5417 				 */
5418 				if ((zoned & ATA_ZONED_MASK) ==
5419 				    ATA_SUPPORT_ZONE_HOST_AWARE)
5420 					softc->zone_mode = DA_ZONE_HOST_AWARE;
5421 				else if ((zoned & ATA_ZONED_MASK) ==
5422 				    ATA_SUPPORT_ZONE_DEV_MANAGED)
5423 					softc->zone_mode =
5424 					    DA_ZONE_DRIVE_MANAGED;
5425 			}
5426 
5427 			zac_cap = le64dec(sup_cap->sup_zac_cap);
5428 			if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
5429 				if (zac_cap & ATA_REPORT_ZONES_SUP)
5430 					softc->zone_flags |=
5431 					    DA_ZONE_FLAG_RZ_SUP;
5432 				if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
5433 					softc->zone_flags |=
5434 					    DA_ZONE_FLAG_OPEN_SUP;
5435 				if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
5436 					softc->zone_flags |=
5437 					    DA_ZONE_FLAG_CLOSE_SUP;
5438 				if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
5439 					softc->zone_flags |=
5440 					    DA_ZONE_FLAG_FINISH_SUP;
5441 				if (zac_cap & ATA_ND_RWP_SUP)
5442 					softc->zone_flags |=
5443 					    DA_ZONE_FLAG_RWP_SUP;
5444 			} else {
5445 				/*
5446 				 * This field was introduced in
5447 				 * ACS-4, r08 on April 28th, 2015.
5448 				 * If the drive firmware was written
5449 				 * to an earlier spec, it won't have
5450 				 * the field.  So, assume all
5451 				 * commands are supported.
5452 				 */
5453 				softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK;
5454 			}
5455 
5456 		}
5457 	} else {
5458 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5459 				SF_RETRY_UA|SF_NO_PRINT);
5460 		if (error == ERESTART)
5461 			return;
5462 		else if (error != 0) {
5463 			/*
5464 			 * If we can't get the ATA Identify Data
5465 			 * Supported Capabilities page, clear the
5466 			 * flag...
5467 			 */
5468 			cam_periph_lock(periph);
5469 			softc->flags &= ~DA_FLAG_CAN_ATA_SUPCAP;
5470 			cam_periph_unlock(periph);
5471 			/*
5472 			 * And clear zone capabilities.
5473 			 */
5474 			softc->zone_flags &= ~DA_ZONE_FLAG_SUP_MASK;
5475 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5476 				/* Don't wedge this device's queue */
5477 				cam_release_devq(done_ccb->ccb_h.path,
5478 						 /*relsim_flags*/0,
5479 						 /*reduction*/0,
5480 						 /*timeout*/0,
5481 						 /*getcount_only*/0);
5482 			}
5483 		}
5484 	}
5485 
5486 	free(csio->data_ptr, M_SCSIDA);
5487 
5488 	if ((error == 0) && (softc->flags & DA_FLAG_CAN_ATA_ZONE)) {
5489 		softc->state = DA_STATE_PROBE_ATA_ZONE;
5490 		xpt_release_ccb(done_ccb);
5491 		xpt_schedule(periph, priority);
5492 		return;
5493 	}
5494 	daprobedone(periph, done_ccb);
5495 	return;
5496 }
5497 
5498 static void
5499 dadone_probeatazone(struct cam_periph *periph, union ccb *done_ccb)
5500 {
5501 	struct da_softc *softc;
5502 	struct ccb_scsiio *csio;
5503 	int error;
5504 
5505 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probeatazone\n"));
5506 
5507 	softc = (struct da_softc *)periph->softc;
5508 	csio = &done_ccb->csio;
5509 
5510 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5511 		struct ata_zoned_info_log *zi_log;
5512 		uint32_t valid_len;
5513 		size_t needed_size;
5514 
5515 		zi_log = (struct ata_zoned_info_log *)csio->data_ptr;
5516 
5517 		valid_len = csio->dxfer_len - csio->resid;
5518 		needed_size = __offsetof(struct ata_zoned_info_log,
5519 		    version_info) + 1 + sizeof(zi_log->version_info);
5520 		if (valid_len >= needed_size) {
5521 			uint64_t tmpvar;
5522 
5523 			tmpvar = le64dec(zi_log->zoned_cap);
5524 			if (tmpvar & ATA_ZDI_CAP_VALID) {
5525 				if (tmpvar & ATA_ZDI_CAP_URSWRZ)
5526 					softc->zone_flags |=
5527 					    DA_ZONE_FLAG_URSWRZ;
5528 				else
5529 					softc->zone_flags &=
5530 					    ~DA_ZONE_FLAG_URSWRZ;
5531 			}
5532 			tmpvar = le64dec(zi_log->optimal_seq_zones);
5533 			if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
5534 				softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET;
5535 				softc->optimal_seq_zones = (tmpvar &
5536 				    ATA_ZDI_OPT_SEQ_MASK);
5537 			} else {
5538 				softc->zone_flags &= ~DA_ZONE_FLAG_OPT_SEQ_SET;
5539 				softc->optimal_seq_zones = 0;
5540 			}
5541 
5542 			tmpvar =le64dec(zi_log->optimal_nonseq_zones);
5543 			if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
5544 				softc->zone_flags |=
5545 				    DA_ZONE_FLAG_OPT_NONSEQ_SET;
5546 				softc->optimal_nonseq_zones =
5547 				    (tmpvar & ATA_ZDI_OPT_NS_MASK);
5548 			} else {
5549 				softc->zone_flags &=
5550 				    ~DA_ZONE_FLAG_OPT_NONSEQ_SET;
5551 				softc->optimal_nonseq_zones = 0;
5552 			}
5553 
5554 			tmpvar = le64dec(zi_log->max_seq_req_zones);
5555 			if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
5556 				softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET;
5557 				softc->max_seq_zones =
5558 				    (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
5559 			} else {
5560 				softc->zone_flags &= ~DA_ZONE_FLAG_MAX_SEQ_SET;
5561 				softc->max_seq_zones = 0;
5562 			}
5563 		}
5564 	} else {
5565 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5566 				SF_RETRY_UA|SF_NO_PRINT);
5567 		if (error == ERESTART)
5568 			return;
5569 		else if (error != 0) {
5570 			cam_periph_lock(periph);
5571 			softc->flags &= ~DA_FLAG_CAN_ATA_ZONE;
5572 			softc->flags &= ~DA_ZONE_FLAG_SET_MASK;
5573 			cam_periph_unlock(periph);
5574 
5575 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5576 				/* Don't wedge this device's queue */
5577 				cam_release_devq(done_ccb->ccb_h.path,
5578 						 /*relsim_flags*/0,
5579 						 /*reduction*/0,
5580 						 /*timeout*/0,
5581 						 /*getcount_only*/0);
5582 			}
5583 		}
5584 
5585 	}
5586 	free(csio->data_ptr, M_SCSIDA);
5587 
5588 	daprobedone(periph, done_ccb);
5589 	return;
5590 }
5591 
5592 static void
5593 dadone_probezone(struct cam_periph *periph, union ccb *done_ccb)
5594 {
5595 	struct da_softc *softc;
5596 	struct ccb_scsiio *csio;
5597 	int error;
5598 
5599 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_probezone\n"));
5600 
5601 	softc = (struct da_softc *)periph->softc;
5602 	csio = &done_ccb->csio;
5603 
5604 	if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5605 		uint32_t valid_len;
5606 		size_t needed_len;
5607 		struct scsi_vpd_zoned_bdc *zoned_bdc;
5608 
5609 		error = 0;
5610 		zoned_bdc = (struct scsi_vpd_zoned_bdc *)csio->data_ptr;
5611 		valid_len = csio->dxfer_len - csio->resid;
5612 		needed_len = __offsetof(struct scsi_vpd_zoned_bdc,
5613 		    max_seq_req_zones) + 1 +
5614 		    sizeof(zoned_bdc->max_seq_req_zones);
5615 		if ((valid_len >= needed_len)
5616 		 && (scsi_2btoul(zoned_bdc->page_length) >= SVPD_ZBDC_PL)) {
5617 			if (zoned_bdc->flags & SVPD_ZBDC_URSWRZ)
5618 				softc->zone_flags |= DA_ZONE_FLAG_URSWRZ;
5619 			else
5620 				softc->zone_flags &= ~DA_ZONE_FLAG_URSWRZ;
5621 			softc->optimal_seq_zones =
5622 			    scsi_4btoul(zoned_bdc->optimal_seq_zones);
5623 			softc->zone_flags |= DA_ZONE_FLAG_OPT_SEQ_SET;
5624 			softc->optimal_nonseq_zones = scsi_4btoul(
5625 			    zoned_bdc->optimal_nonseq_zones);
5626 			softc->zone_flags |= DA_ZONE_FLAG_OPT_NONSEQ_SET;
5627 			softc->max_seq_zones =
5628 			    scsi_4btoul(zoned_bdc->max_seq_req_zones);
5629 			softc->zone_flags |= DA_ZONE_FLAG_MAX_SEQ_SET;
5630 		}
5631 		/*
5632 		 * All of the zone commands are mandatory for SCSI
5633 		 * devices.
5634 		 *
5635 		 * XXX KDM this is valid as of September 2015.
5636 		 * Re-check this assumption once the SAT spec is
5637 		 * updated to support SCSI ZBC to ATA ZAC mapping.
5638 		 * Since ATA allows zone commands to be reported
5639 		 * as supported or not, this may not necessarily
5640 		 * be true for an ATA device behind a SAT (SCSI to
5641 		 * ATA Translation) layer.
5642 		 */
5643 		softc->zone_flags |= DA_ZONE_FLAG_SUP_MASK;
5644 	} else {
5645 		error = daerror(done_ccb, CAM_RETRY_SELTO,
5646 				SF_RETRY_UA|SF_NO_PRINT);
5647 		if (error == ERESTART)
5648 			return;
5649 		else if (error != 0) {
5650 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5651 				/* Don't wedge this device's queue */
5652 				cam_release_devq(done_ccb->ccb_h.path,
5653 						 /*relsim_flags*/0,
5654 						 /*reduction*/0,
5655 						 /*timeout*/0,
5656 						 /*getcount_only*/0);
5657 			}
5658 		}
5659 	}
5660 	daprobedone(periph, done_ccb);
5661 	return;
5662 }
5663 
5664 static void
5665 dadone_tur(struct cam_periph *periph, union ccb *done_ccb)
5666 {
5667 	struct da_softc *softc;
5668 	struct ccb_scsiio *csio;
5669 
5670 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("dadone_tur\n"));
5671 
5672 	softc = (struct da_softc *)periph->softc;
5673 	csio = &done_ccb->csio;
5674 	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5675 
5676 		if (daerror(done_ccb, CAM_RETRY_SELTO,
5677 		    SF_RETRY_UA | SF_NO_RECOVERY | SF_NO_PRINT) == ERESTART)
5678 			return;	/* Will complete again, keep reference */
5679 		if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
5680 			cam_release_devq(done_ccb->ccb_h.path,
5681 					 /*relsim_flags*/0,
5682 					 /*reduction*/0,
5683 					 /*timeout*/0,
5684 					 /*getcount_only*/0);
5685 	}
5686 	xpt_release_ccb(done_ccb);
5687 	softc->flags &= ~DA_FLAG_TUR_PENDING;
5688 	da_periph_release_locked(periph, DA_REF_TUR);
5689 	return;
5690 }
5691 
5692 static void
5693 dareprobe(struct cam_periph *periph)
5694 {
5695 	struct da_softc	  *softc;
5696 	int status;
5697 
5698 	softc = (struct da_softc *)periph->softc;
5699 
5700 	/* Probe in progress; don't interfere. */
5701 	if (softc->state != DA_STATE_NORMAL)
5702 		return;
5703 
5704 	status = da_periph_acquire(periph, DA_REF_REPROBE);
5705 	KASSERT(status == 0, ("dareprobe: cam_periph_acquire failed"));
5706 
5707 	softc->state = DA_STATE_PROBE_WP;
5708 	xpt_schedule(periph, CAM_PRIORITY_DEV);
5709 }
5710 
5711 static int
5712 daerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
5713 {
5714 	struct da_softc	  *softc;
5715 	struct cam_periph *periph;
5716 	int error, error_code, sense_key, asc, ascq;
5717 
5718 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
5719 	if (ccb->csio.bio != NULL)
5720 		biotrack(ccb->csio.bio, __func__);
5721 #endif
5722 
5723 	periph = xpt_path_periph(ccb->ccb_h.path);
5724 	softc = (struct da_softc *)periph->softc;
5725 
5726 	cam_periph_assert(periph, MA_OWNED);
5727 
5728 	/*
5729 	 * Automatically detect devices that do not support
5730 	 * READ(6)/WRITE(6) and upgrade to using 10 byte cdbs.
5731 	 */
5732 	error = 0;
5733 	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INVALID) {
5734 		error = cmd6workaround(ccb);
5735 	} else if (scsi_extract_sense_ccb(ccb,
5736 	    &error_code, &sense_key, &asc, &ascq)) {
5737 		if (sense_key == SSD_KEY_ILLEGAL_REQUEST)
5738 			error = cmd6workaround(ccb);
5739 		/*
5740 		 * If the target replied with CAPACITY DATA HAS CHANGED UA,
5741 		 * query the capacity and notify upper layers.
5742 		 */
5743 		else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5744 		    asc == 0x2A && ascq == 0x09) {
5745 			xpt_print(periph->path, "Capacity data has changed\n");
5746 			softc->flags &= ~DA_FLAG_PROBED;
5747 			dareprobe(periph);
5748 			sense_flags |= SF_NO_PRINT;
5749 		} else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5750 		    asc == 0x28 && ascq == 0x00) {
5751 			softc->flags &= ~DA_FLAG_PROBED;
5752 			disk_media_changed(softc->disk, M_NOWAIT);
5753 		} else if (sense_key == SSD_KEY_UNIT_ATTENTION &&
5754 		    asc == 0x3F && ascq == 0x03) {
5755 			xpt_print(periph->path, "INQUIRY data has changed\n");
5756 			softc->flags &= ~DA_FLAG_PROBED;
5757 			dareprobe(periph);
5758 			sense_flags |= SF_NO_PRINT;
5759 		} else if (sense_key == SSD_KEY_NOT_READY &&
5760 		    asc == 0x3a && (softc->flags & DA_FLAG_PACK_INVALID) == 0) {
5761 			softc->flags |= DA_FLAG_PACK_INVALID;
5762 			disk_media_gone(softc->disk, M_NOWAIT);
5763 		}
5764 	}
5765 	if (error == ERESTART)
5766 		return (ERESTART);
5767 
5768 #ifdef CAM_IO_STATS
5769 	switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
5770 	case CAM_CMD_TIMEOUT:
5771 		softc->timeouts++;
5772 		break;
5773 	case CAM_REQ_ABORTED:
5774 	case CAM_REQ_CMP_ERR:
5775 	case CAM_REQ_TERMIO:
5776 	case CAM_UNREC_HBA_ERROR:
5777 	case CAM_DATA_RUN_ERR:
5778 		softc->errors++;
5779 		break;
5780 	default:
5781 		break;
5782 	}
5783 #endif
5784 
5785 	/*
5786 	 * XXX
5787 	 * Until we have a better way of doing pack validation,
5788 	 * don't treat UAs as errors.
5789 	 */
5790 	sense_flags |= SF_RETRY_UA;
5791 
5792 	if (softc->quirks & DA_Q_RETRY_BUSY)
5793 		sense_flags |= SF_RETRY_BUSY;
5794 	return(cam_periph_error(ccb, cam_flags, sense_flags));
5795 }
5796 
5797 static void
5798 damediapoll(void *arg)
5799 {
5800 	struct cam_periph *periph = arg;
5801 	struct da_softc *softc = periph->softc;
5802 
5803 	if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR) &&
5804 	    (softc->flags & DA_FLAG_TUR_PENDING) == 0 &&
5805 	    LIST_EMPTY(&softc->pending_ccbs)) {
5806 		if (da_periph_acquire(periph, DA_REF_TUR) == 0) {
5807 			cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR);
5808 			daschedule(periph);
5809 		}
5810 	}
5811 	/* Queue us up again */
5812 	if (da_poll_period != 0)
5813 		callout_schedule(&softc->mediapoll_c, da_poll_period * hz);
5814 }
5815 
5816 static void
5817 daprevent(struct cam_periph *periph, int action)
5818 {
5819 	struct	da_softc *softc;
5820 	union	ccb *ccb;
5821 	int	error;
5822 
5823 	cam_periph_assert(periph, MA_OWNED);
5824 	softc = (struct da_softc *)periph->softc;
5825 
5826 	if (((action == PR_ALLOW)
5827 	  && (softc->flags & DA_FLAG_PACK_LOCKED) == 0)
5828 	 || ((action == PR_PREVENT)
5829 	  && (softc->flags & DA_FLAG_PACK_LOCKED) != 0)) {
5830 		return;
5831 	}
5832 
5833 	ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
5834 
5835 	scsi_prevent(&ccb->csio,
5836 		     /*retries*/1,
5837 		     /*cbcfp*/NULL,
5838 		     MSG_SIMPLE_Q_TAG,
5839 		     action,
5840 		     SSD_FULL_SIZE,
5841 		     5000);
5842 
5843 	error = cam_periph_runccb(ccb, daerror, CAM_RETRY_SELTO,
5844 	    SF_RETRY_UA | SF_NO_PRINT, softc->disk->d_devstat);
5845 
5846 	if (error == 0) {
5847 		if (action == PR_ALLOW)
5848 			softc->flags &= ~DA_FLAG_PACK_LOCKED;
5849 		else
5850 			softc->flags |= DA_FLAG_PACK_LOCKED;
5851 	}
5852 
5853 	xpt_release_ccb(ccb);
5854 }
5855 
5856 static void
5857 dasetgeom(struct cam_periph *periph, uint32_t block_len, uint64_t maxsector,
5858 	  struct scsi_read_capacity_data_long *rcaplong, size_t rcap_len)
5859 {
5860 	struct ccb_calc_geometry ccg;
5861 	struct da_softc *softc;
5862 	struct disk_params *dp;
5863 	u_int lbppbe, lalba;
5864 	int error;
5865 
5866 	softc = (struct da_softc *)periph->softc;
5867 
5868 	dp = &softc->params;
5869 	dp->secsize = block_len;
5870 	dp->sectors = maxsector + 1;
5871 	if (rcaplong != NULL) {
5872 		lbppbe = rcaplong->prot_lbppbe & SRC16_LBPPBE;
5873 		lalba = scsi_2btoul(rcaplong->lalba_lbp);
5874 		lalba &= SRC16_LALBA_A;
5875 	} else {
5876 		lbppbe = 0;
5877 		lalba = 0;
5878 	}
5879 
5880 	if (lbppbe > 0) {
5881 		dp->stripesize = block_len << lbppbe;
5882 		dp->stripeoffset = (dp->stripesize - block_len * lalba) %
5883 		    dp->stripesize;
5884 	} else if (softc->quirks & DA_Q_4K) {
5885 		dp->stripesize = 4096;
5886 		dp->stripeoffset = 0;
5887 	} else if (softc->unmap_gran != 0) {
5888 		dp->stripesize = block_len * softc->unmap_gran;
5889 		dp->stripeoffset = (dp->stripesize - block_len *
5890 		    softc->unmap_gran_align) % dp->stripesize;
5891 	} else {
5892 		dp->stripesize = 0;
5893 		dp->stripeoffset = 0;
5894 	}
5895 	/*
5896 	 * Have the controller provide us with a geometry
5897 	 * for this disk.  The only time the geometry
5898 	 * matters is when we boot and the controller
5899 	 * is the only one knowledgeable enough to come
5900 	 * up with something that will make this a bootable
5901 	 * device.
5902 	 */
5903 	xpt_setup_ccb(&ccg.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
5904 	ccg.ccb_h.func_code = XPT_CALC_GEOMETRY;
5905 	ccg.block_size = dp->secsize;
5906 	ccg.volume_size = dp->sectors;
5907 	ccg.heads = 0;
5908 	ccg.secs_per_track = 0;
5909 	ccg.cylinders = 0;
5910 	xpt_action((union ccb*)&ccg);
5911 	if ((ccg.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5912 		/*
5913 		 * We don't know what went wrong here- but just pick
5914 		 * a geometry so we don't have nasty things like divide
5915 		 * by zero.
5916 		 */
5917 		dp->heads = 255;
5918 		dp->secs_per_track = 255;
5919 		dp->cylinders = dp->sectors / (255 * 255);
5920 		if (dp->cylinders == 0) {
5921 			dp->cylinders = 1;
5922 		}
5923 	} else {
5924 		dp->heads = ccg.heads;
5925 		dp->secs_per_track = ccg.secs_per_track;
5926 		dp->cylinders = ccg.cylinders;
5927 	}
5928 
5929 	/*
5930 	 * If the user supplied a read capacity buffer, and if it is
5931 	 * different than the previous buffer, update the data in the EDT.
5932 	 * If it's the same, we don't bother.  This avoids sending an
5933 	 * update every time someone opens this device.
5934 	 */
5935 	if ((rcaplong != NULL)
5936 	 && (bcmp(rcaplong, &softc->rcaplong,
5937 		  min(sizeof(softc->rcaplong), rcap_len)) != 0)) {
5938 		struct ccb_dev_advinfo cdai;
5939 
5940 		xpt_setup_ccb(&cdai.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
5941 		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
5942 		cdai.buftype = CDAI_TYPE_RCAPLONG;
5943 		cdai.flags = CDAI_FLAG_STORE;
5944 		cdai.bufsiz = rcap_len;
5945 		cdai.buf = (uint8_t *)rcaplong;
5946 		xpt_action((union ccb *)&cdai);
5947 		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
5948 			cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
5949 		if (cdai.ccb_h.status != CAM_REQ_CMP) {
5950 			xpt_print(periph->path, "%s: failed to set read "
5951 				  "capacity advinfo\n", __func__);
5952 			/* Use cam_error_print() to decode the status */
5953 			cam_error_print((union ccb *)&cdai, CAM_ESF_CAM_STATUS,
5954 					CAM_EPF_ALL);
5955 		} else {
5956 			bcopy(rcaplong, &softc->rcaplong,
5957 			      min(sizeof(softc->rcaplong), rcap_len));
5958 		}
5959 	}
5960 
5961 	softc->disk->d_sectorsize = softc->params.secsize;
5962 	softc->disk->d_mediasize = softc->params.secsize * (off_t)softc->params.sectors;
5963 	softc->disk->d_stripesize = softc->params.stripesize;
5964 	softc->disk->d_stripeoffset = softc->params.stripeoffset;
5965 	/* XXX: these are not actually "firmware" values, so they may be wrong */
5966 	softc->disk->d_fwsectors = softc->params.secs_per_track;
5967 	softc->disk->d_fwheads = softc->params.heads;
5968 	softc->disk->d_devstat->block_size = softc->params.secsize;
5969 	softc->disk->d_devstat->flags &= ~DEVSTAT_BS_UNAVAILABLE;
5970 
5971 	error = disk_resize(softc->disk, M_NOWAIT);
5972 	if (error != 0)
5973 		xpt_print(periph->path, "disk_resize(9) failed, error = %d\n", error);
5974 }
5975 
5976 static void
5977 dasendorderedtag(void *arg)
5978 {
5979 	struct cam_periph *periph = arg;
5980 	struct da_softc *softc = periph->softc;
5981 
5982 	cam_periph_assert(periph, MA_OWNED);
5983 	if (da_send_ordered) {
5984 		if (!LIST_EMPTY(&softc->pending_ccbs)) {
5985 			if ((softc->flags & DA_FLAG_WAS_OTAG) == 0)
5986 				softc->flags |= DA_FLAG_NEED_OTAG;
5987 			softc->flags &= ~DA_FLAG_WAS_OTAG;
5988 		}
5989 	}
5990 
5991 	/* Queue us up again */
5992 	callout_reset(&softc->sendordered_c,
5993 	    (da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL,
5994 	    dasendorderedtag, periph);
5995 }
5996 
5997 /*
5998  * Step through all DA peripheral drivers, and if the device is still open,
5999  * sync the disk cache to physical media.
6000  */
6001 static void
6002 dashutdown(void * arg, int howto)
6003 {
6004 	struct cam_periph *periph;
6005 	struct da_softc *softc;
6006 	union ccb *ccb;
6007 	int error;
6008 
6009 	CAM_PERIPH_FOREACH(periph, &dadriver) {
6010 		softc = (struct da_softc *)periph->softc;
6011 		if (SCHEDULER_STOPPED()) {
6012 			/* If we paniced with the lock held, do not recurse. */
6013 			if (!cam_periph_owned(periph) &&
6014 			    (softc->flags & DA_FLAG_OPEN)) {
6015 				dadump(softc->disk, NULL, 0, 0, 0);
6016 			}
6017 			continue;
6018 		}
6019 		cam_periph_lock(periph);
6020 
6021 		/*
6022 		 * We only sync the cache if the drive is still open, and
6023 		 * if the drive is capable of it..
6024 		 */
6025 		if (((softc->flags & DA_FLAG_OPEN) == 0)
6026 		 || (softc->quirks & DA_Q_NO_SYNC_CACHE)) {
6027 			cam_periph_unlock(periph);
6028 			continue;
6029 		}
6030 
6031 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
6032 		scsi_synchronize_cache(&ccb->csio,
6033 				       /*retries*/0,
6034 				       /*cbfcnp*/NULL,
6035 				       MSG_SIMPLE_Q_TAG,
6036 				       /*begin_lba*/0, /* whole disk */
6037 				       /*lb_count*/0,
6038 				       SSD_FULL_SIZE,
6039 				       60 * 60 * 1000);
6040 
6041 		error = cam_periph_runccb(ccb, daerror, /*cam_flags*/0,
6042 		    /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY | SF_QUIET_IR,
6043 		    softc->disk->d_devstat);
6044 		if (error != 0)
6045 			xpt_print(periph->path, "Synchronize cache failed\n");
6046 		xpt_release_ccb(ccb);
6047 		cam_periph_unlock(periph);
6048 	}
6049 }
6050 
6051 #else /* !_KERNEL */
6052 
6053 /*
6054  * XXX These are only left out of the kernel build to silence warnings.  If,
6055  * for some reason these functions are used in the kernel, the ifdefs should
6056  * be moved so they are included both in the kernel and userland.
6057  */
6058 void
6059 scsi_format_unit(struct ccb_scsiio *csio, u_int32_t retries,
6060 		 void (*cbfcnp)(struct cam_periph *, union ccb *),
6061 		 u_int8_t tag_action, u_int8_t byte2, u_int16_t ileave,
6062 		 u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len,
6063 		 u_int32_t timeout)
6064 {
6065 	struct scsi_format_unit *scsi_cmd;
6066 
6067 	scsi_cmd = (struct scsi_format_unit *)&csio->cdb_io.cdb_bytes;
6068 	scsi_cmd->opcode = FORMAT_UNIT;
6069 	scsi_cmd->byte2 = byte2;
6070 	scsi_ulto2b(ileave, scsi_cmd->interleave);
6071 
6072 	cam_fill_csio(csio,
6073 		      retries,
6074 		      cbfcnp,
6075 		      /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6076 		      tag_action,
6077 		      data_ptr,
6078 		      dxfer_len,
6079 		      sense_len,
6080 		      sizeof(*scsi_cmd),
6081 		      timeout);
6082 }
6083 
6084 void
6085 scsi_read_defects(struct ccb_scsiio *csio, uint32_t retries,
6086 		  void (*cbfcnp)(struct cam_periph *, union ccb *),
6087 		  uint8_t tag_action, uint8_t list_format,
6088 		  uint32_t addr_desc_index, uint8_t *data_ptr,
6089 		  uint32_t dxfer_len, int minimum_cmd_size,
6090 		  uint8_t sense_len, uint32_t timeout)
6091 {
6092 	uint8_t cdb_len;
6093 
6094 	/*
6095 	 * These conditions allow using the 10 byte command.  Otherwise we
6096 	 * need to use the 12 byte command.
6097 	 */
6098 	if ((minimum_cmd_size <= 10)
6099 	 && (addr_desc_index == 0)
6100 	 && (dxfer_len <= SRDD10_MAX_LENGTH)) {
6101 		struct scsi_read_defect_data_10 *cdb10;
6102 
6103 		cdb10 = (struct scsi_read_defect_data_10 *)
6104 			&csio->cdb_io.cdb_bytes;
6105 
6106 		cdb_len = sizeof(*cdb10);
6107 		bzero(cdb10, cdb_len);
6108                 cdb10->opcode = READ_DEFECT_DATA_10;
6109                 cdb10->format = list_format;
6110                 scsi_ulto2b(dxfer_len, cdb10->alloc_length);
6111 	} else {
6112 		struct scsi_read_defect_data_12 *cdb12;
6113 
6114 		cdb12 = (struct scsi_read_defect_data_12 *)
6115 			&csio->cdb_io.cdb_bytes;
6116 
6117 		cdb_len = sizeof(*cdb12);
6118 		bzero(cdb12, cdb_len);
6119                 cdb12->opcode = READ_DEFECT_DATA_12;
6120                 cdb12->format = list_format;
6121                 scsi_ulto4b(dxfer_len, cdb12->alloc_length);
6122 		scsi_ulto4b(addr_desc_index, cdb12->address_descriptor_index);
6123 	}
6124 
6125 	cam_fill_csio(csio,
6126 		      retries,
6127 		      cbfcnp,
6128 		      /*flags*/ CAM_DIR_IN,
6129 		      tag_action,
6130 		      data_ptr,
6131 		      dxfer_len,
6132 		      sense_len,
6133 		      cdb_len,
6134 		      timeout);
6135 }
6136 
6137 void
6138 scsi_sanitize(struct ccb_scsiio *csio, u_int32_t retries,
6139 	      void (*cbfcnp)(struct cam_periph *, union ccb *),
6140 	      u_int8_t tag_action, u_int8_t byte2, u_int16_t control,
6141 	      u_int8_t *data_ptr, u_int32_t dxfer_len, u_int8_t sense_len,
6142 	      u_int32_t timeout)
6143 {
6144 	struct scsi_sanitize *scsi_cmd;
6145 
6146 	scsi_cmd = (struct scsi_sanitize *)&csio->cdb_io.cdb_bytes;
6147 	scsi_cmd->opcode = SANITIZE;
6148 	scsi_cmd->byte2 = byte2;
6149 	scsi_cmd->control = control;
6150 	scsi_ulto2b(dxfer_len, scsi_cmd->length);
6151 
6152 	cam_fill_csio(csio,
6153 		      retries,
6154 		      cbfcnp,
6155 		      /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6156 		      tag_action,
6157 		      data_ptr,
6158 		      dxfer_len,
6159 		      sense_len,
6160 		      sizeof(*scsi_cmd),
6161 		      timeout);
6162 }
6163 
6164 #endif /* _KERNEL */
6165 
6166 void
6167 scsi_zbc_out(struct ccb_scsiio *csio, uint32_t retries,
6168 	     void (*cbfcnp)(struct cam_periph *, union ccb *),
6169 	     uint8_t tag_action, uint8_t service_action, uint64_t zone_id,
6170 	     uint8_t zone_flags, uint8_t *data_ptr, uint32_t dxfer_len,
6171 	     uint8_t sense_len, uint32_t timeout)
6172 {
6173 	struct scsi_zbc_out *scsi_cmd;
6174 
6175 	scsi_cmd = (struct scsi_zbc_out *)&csio->cdb_io.cdb_bytes;
6176 	scsi_cmd->opcode = ZBC_OUT;
6177 	scsi_cmd->service_action = service_action;
6178 	scsi_u64to8b(zone_id, scsi_cmd->zone_id);
6179 	scsi_cmd->zone_flags = zone_flags;
6180 
6181 	cam_fill_csio(csio,
6182 		      retries,
6183 		      cbfcnp,
6184 		      /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6185 		      tag_action,
6186 		      data_ptr,
6187 		      dxfer_len,
6188 		      sense_len,
6189 		      sizeof(*scsi_cmd),
6190 		      timeout);
6191 }
6192 
6193 void
6194 scsi_zbc_in(struct ccb_scsiio *csio, uint32_t retries,
6195 	    void (*cbfcnp)(struct cam_periph *, union ccb *),
6196 	    uint8_t tag_action, uint8_t service_action, uint64_t zone_start_lba,
6197 	    uint8_t zone_options, uint8_t *data_ptr, uint32_t dxfer_len,
6198 	    uint8_t sense_len, uint32_t timeout)
6199 {
6200 	struct scsi_zbc_in *scsi_cmd;
6201 
6202 	scsi_cmd = (struct scsi_zbc_in *)&csio->cdb_io.cdb_bytes;
6203 	scsi_cmd->opcode = ZBC_IN;
6204 	scsi_cmd->service_action = service_action;
6205 	scsi_ulto4b(dxfer_len, scsi_cmd->length);
6206 	scsi_u64to8b(zone_start_lba, scsi_cmd->zone_start_lba);
6207 	scsi_cmd->zone_options = zone_options;
6208 
6209 	cam_fill_csio(csio,
6210 		      retries,
6211 		      cbfcnp,
6212 		      /*flags*/ (dxfer_len > 0) ? CAM_DIR_IN : CAM_DIR_NONE,
6213 		      tag_action,
6214 		      data_ptr,
6215 		      dxfer_len,
6216 		      sense_len,
6217 		      sizeof(*scsi_cmd),
6218 		      timeout);
6219 
6220 }
6221 
6222 int
6223 scsi_ata_zac_mgmt_out(struct ccb_scsiio *csio, uint32_t retries,
6224 		      void (*cbfcnp)(struct cam_periph *, union ccb *),
6225 		      uint8_t tag_action, int use_ncq,
6226 		      uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags,
6227 		      uint8_t *data_ptr, uint32_t dxfer_len,
6228 		      uint8_t *cdb_storage, size_t cdb_storage_len,
6229 		      uint8_t sense_len, uint32_t timeout)
6230 {
6231 	uint8_t command_out, protocol, ata_flags;
6232 	uint16_t features_out;
6233 	uint32_t sectors_out, auxiliary;
6234 	int retval;
6235 
6236 	retval = 0;
6237 
6238 	if (use_ncq == 0) {
6239 		command_out = ATA_ZAC_MANAGEMENT_OUT;
6240 		features_out = (zm_action & 0xf) | (zone_flags << 8);
6241 		ata_flags = AP_FLAG_BYT_BLOK_BLOCKS;
6242 		if (dxfer_len == 0) {
6243 			protocol = AP_PROTO_NON_DATA;
6244 			ata_flags |= AP_FLAG_TLEN_NO_DATA;
6245 			sectors_out = 0;
6246 		} else {
6247 			protocol = AP_PROTO_DMA;
6248 			ata_flags |= AP_FLAG_TLEN_SECT_CNT |
6249 				     AP_FLAG_TDIR_TO_DEV;
6250 			sectors_out = ((dxfer_len >> 9) & 0xffff);
6251 		}
6252 		auxiliary = 0;
6253 	} else {
6254 		ata_flags = AP_FLAG_BYT_BLOK_BLOCKS;
6255 		if (dxfer_len == 0) {
6256 			command_out = ATA_NCQ_NON_DATA;
6257 			features_out = ATA_NCQ_ZAC_MGMT_OUT;
6258 			/*
6259 			 * We're assuming the SCSI to ATA translation layer
6260 			 * will set the NCQ tag number in the tag field.
6261 			 * That isn't clear from the SAT-4 spec (as of rev 05).
6262 			 */
6263 			sectors_out = 0;
6264 			ata_flags |= AP_FLAG_TLEN_NO_DATA;
6265 		} else {
6266 			command_out = ATA_SEND_FPDMA_QUEUED;
6267 			/*
6268 			 * Note that we're defaulting to normal priority,
6269 			 * and assuming that the SCSI to ATA translation
6270 			 * layer will insert the NCQ tag number in the tag
6271 			 * field.  That isn't clear in the SAT-4 spec (as
6272 			 * of rev 05).
6273 			 */
6274 			sectors_out = ATA_SFPDMA_ZAC_MGMT_OUT << 8;
6275 
6276 			ata_flags |= AP_FLAG_TLEN_FEAT |
6277 				     AP_FLAG_TDIR_TO_DEV;
6278 
6279 			/*
6280 			 * For SEND FPDMA QUEUED, the transfer length is
6281 			 * encoded in the FEATURE register, and 0 means
6282 			 * that 65536 512 byte blocks are to be tranferred.
6283 			 * In practice, it seems unlikely that we'll see
6284 			 * a transfer that large, and it may confuse the
6285 			 * the SAT layer, because generally that means that
6286 			 * 0 bytes should be transferred.
6287 			 */
6288 			if (dxfer_len == (65536 * 512)) {
6289 				features_out = 0;
6290 			} else if (dxfer_len <= (65535 * 512)) {
6291 				features_out = ((dxfer_len >> 9) & 0xffff);
6292 			} else {
6293 				/* The transfer is too big. */
6294 				retval = 1;
6295 				goto bailout;
6296 			}
6297 
6298 		}
6299 
6300 		auxiliary = (zm_action & 0xf) | (zone_flags << 8);
6301 		protocol = AP_PROTO_FPDMA;
6302 	}
6303 
6304 	protocol |= AP_EXTEND;
6305 
6306 	retval = scsi_ata_pass(csio,
6307 	    retries,
6308 	    cbfcnp,
6309 	    /*flags*/ (dxfer_len > 0) ? CAM_DIR_OUT : CAM_DIR_NONE,
6310 	    tag_action,
6311 	    /*protocol*/ protocol,
6312 	    /*ata_flags*/ ata_flags,
6313 	    /*features*/ features_out,
6314 	    /*sector_count*/ sectors_out,
6315 	    /*lba*/ zone_id,
6316 	    /*command*/ command_out,
6317 	    /*device*/ 0,
6318 	    /*icc*/ 0,
6319 	    /*auxiliary*/ auxiliary,
6320 	    /*control*/ 0,
6321 	    /*data_ptr*/ data_ptr,
6322 	    /*dxfer_len*/ dxfer_len,
6323 	    /*cdb_storage*/ cdb_storage,
6324 	    /*cdb_storage_len*/ cdb_storage_len,
6325 	    /*minimum_cmd_size*/ 0,
6326 	    /*sense_len*/ SSD_FULL_SIZE,
6327 	    /*timeout*/ timeout);
6328 
6329 bailout:
6330 
6331 	return (retval);
6332 }
6333 
6334 int
6335 scsi_ata_zac_mgmt_in(struct ccb_scsiio *csio, uint32_t retries,
6336 		     void (*cbfcnp)(struct cam_periph *, union ccb *),
6337 		     uint8_t tag_action, int use_ncq,
6338 		     uint8_t zm_action, uint64_t zone_id, uint8_t zone_flags,
6339 		     uint8_t *data_ptr, uint32_t dxfer_len,
6340 		     uint8_t *cdb_storage, size_t cdb_storage_len,
6341 		     uint8_t sense_len, uint32_t timeout)
6342 {
6343 	uint8_t command_out, protocol;
6344 	uint16_t features_out, sectors_out;
6345 	uint32_t auxiliary;
6346 	int ata_flags;
6347 	int retval;
6348 
6349 	retval = 0;
6350 	ata_flags = AP_FLAG_TDIR_FROM_DEV | AP_FLAG_BYT_BLOK_BLOCKS;
6351 
6352 	if (use_ncq == 0) {
6353 		command_out = ATA_ZAC_MANAGEMENT_IN;
6354 		/* XXX KDM put a macro here */
6355 		features_out = (zm_action & 0xf) | (zone_flags << 8);
6356 		sectors_out = dxfer_len >> 9; /* XXX KDM macro */
6357 		protocol = AP_PROTO_DMA;
6358 		ata_flags |= AP_FLAG_TLEN_SECT_CNT;
6359 		auxiliary = 0;
6360 	} else {
6361 		ata_flags |= AP_FLAG_TLEN_FEAT;
6362 
6363 		command_out = ATA_RECV_FPDMA_QUEUED;
6364 		sectors_out = ATA_RFPDMA_ZAC_MGMT_IN << 8;
6365 
6366 		/*
6367 		 * For RECEIVE FPDMA QUEUED, the transfer length is
6368 		 * encoded in the FEATURE register, and 0 means
6369 		 * that 65536 512 byte blocks are to be tranferred.
6370 		 * In practice, it seems unlikely that we'll see
6371 		 * a transfer that large, and it may confuse the
6372 		 * the SAT layer, because generally that means that
6373 		 * 0 bytes should be transferred.
6374 		 */
6375 		if (dxfer_len == (65536 * 512)) {
6376 			features_out = 0;
6377 		} else if (dxfer_len <= (65535 * 512)) {
6378 			features_out = ((dxfer_len >> 9) & 0xffff);
6379 		} else {
6380 			/* The transfer is too big. */
6381 			retval = 1;
6382 			goto bailout;
6383 		}
6384 		auxiliary = (zm_action & 0xf) | (zone_flags << 8),
6385 		protocol = AP_PROTO_FPDMA;
6386 	}
6387 
6388 	protocol |= AP_EXTEND;
6389 
6390 	retval = scsi_ata_pass(csio,
6391 	    retries,
6392 	    cbfcnp,
6393 	    /*flags*/ CAM_DIR_IN,
6394 	    tag_action,
6395 	    /*protocol*/ protocol,
6396 	    /*ata_flags*/ ata_flags,
6397 	    /*features*/ features_out,
6398 	    /*sector_count*/ sectors_out,
6399 	    /*lba*/ zone_id,
6400 	    /*command*/ command_out,
6401 	    /*device*/ 0,
6402 	    /*icc*/ 0,
6403 	    /*auxiliary*/ auxiliary,
6404 	    /*control*/ 0,
6405 	    /*data_ptr*/ data_ptr,
6406 	    /*dxfer_len*/ (dxfer_len >> 9) * 512, /* XXX KDM */
6407 	    /*cdb_storage*/ cdb_storage,
6408 	    /*cdb_storage_len*/ cdb_storage_len,
6409 	    /*minimum_cmd_size*/ 0,
6410 	    /*sense_len*/ SSD_FULL_SIZE,
6411 	    /*timeout*/ timeout);
6412 
6413 bailout:
6414 	return (retval);
6415 }
6416