xref: /freebsd/sys/cam/nvme/nvme_da.c (revision 36712a94975f5bd0d26c85377283b49a2369c82f)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2015 Netflix, Inc.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer,
11  *    without modification, immediately at the beginning of the file.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  *
27  * Derived from ata_da.c:
28  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 
36 #ifdef _KERNEL
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/bio.h>
40 #include <sys/sysctl.h>
41 #include <sys/taskqueue.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/conf.h>
45 #include <sys/devicestat.h>
46 #include <sys/eventhandler.h>
47 #include <sys/malloc.h>
48 #include <sys/cons.h>
49 #include <sys/proc.h>
50 #include <sys/reboot.h>
51 #include <geom/geom_disk.h>
52 #endif /* _KERNEL */
53 
54 #ifndef _KERNEL
55 #include <stdio.h>
56 #include <string.h>
57 #endif /* _KERNEL */
58 
59 #include <cam/cam.h>
60 #include <cam/cam_ccb.h>
61 #include <cam/cam_periph.h>
62 #include <cam/cam_xpt_periph.h>
63 #include <cam/cam_sim.h>
64 #include <cam/cam_iosched.h>
65 
66 #include <cam/nvme/nvme_all.h>
67 
68 typedef enum {
69 	NDA_STATE_NORMAL
70 } nda_state;
71 
72 typedef enum {
73 	NDA_FLAG_OPEN		= 0x0001,
74 	NDA_FLAG_DIRTY		= 0x0002,
75 	NDA_FLAG_SCTX_INIT	= 0x0004,
76 } nda_flags;
77 
78 typedef enum {
79 	NDA_Q_4K   = 0x01,
80 	NDA_Q_NONE = 0x00,
81 } nda_quirks;
82 
83 #define NDA_Q_BIT_STRING	\
84 	"\020"			\
85 	"\001Bit 0"
86 
87 typedef enum {
88 	NDA_CCB_BUFFER_IO	= 0x01,
89 	NDA_CCB_DUMP            = 0x02,
90 	NDA_CCB_TRIM            = 0x03,
91 	NDA_CCB_TYPE_MASK	= 0x0F,
92 } nda_ccb_state;
93 
94 /* Offsets into our private area for storing information */
95 #define ccb_state	ccb_h.ppriv_field0
96 #define ccb_bp		ccb_h.ppriv_ptr1	/* For NDA_CCB_BUFFER_IO */
97 #define ccb_trim	ccb_h.ppriv_ptr1	/* For NDA_CCB_TRIM */
98 
99 struct nda_softc {
100 	struct   cam_iosched_softc *cam_iosched;
101 	int			outstanding_cmds;	/* Number of active commands */
102 	int			refcount;		/* Active xpt_action() calls */
103 	nda_state		state;
104 	nda_flags		flags;
105 	nda_quirks		quirks;
106 	int			unmappedio;
107 	quad_t			deletes;
108 	uint32_t		nsid;			/* Namespace ID for this nda device */
109 	struct disk		*disk;
110 	struct task		sysctl_task;
111 	struct sysctl_ctx_list	sysctl_ctx;
112 	struct sysctl_oid	*sysctl_tree;
113 	uint64_t		trim_count;
114 	uint64_t		trim_ranges;
115 	uint64_t		trim_lbas;
116 #ifdef CAM_TEST_FAILURE
117 	int			force_read_error;
118 	int			force_write_error;
119 	int			periodic_read_error;
120 	int			periodic_read_count;
121 #endif
122 #ifdef CAM_IO_STATS
123 	struct sysctl_ctx_list	sysctl_stats_ctx;
124 	struct sysctl_oid	*sysctl_stats_tree;
125 	u_int			timeouts;
126 	u_int			errors;
127 	u_int			invalidations;
128 #endif
129 };
130 
131 struct nda_trim_request {
132 	struct nvme_dsm_range	dsm[NVME_MAX_DSM_TRIM / sizeof(struct nvme_dsm_range)];
133 	TAILQ_HEAD(, bio) bps;
134 };
135 _Static_assert(NVME_MAX_DSM_TRIM % sizeof(struct nvme_dsm_range) == 0,
136     "NVME_MAX_DSM_TRIM must be an integral number of ranges");
137 
138 /* Need quirk table */
139 
140 static	disk_strategy_t	ndastrategy;
141 static	dumper_t	ndadump;
142 static	periph_init_t	ndainit;
143 static	void		ndaasync(void *callback_arg, u_int32_t code,
144 				struct cam_path *path, void *arg);
145 static	void		ndasysctlinit(void *context, int pending);
146 static	periph_ctor_t	ndaregister;
147 static	periph_dtor_t	ndacleanup;
148 static	periph_start_t	ndastart;
149 static	periph_oninv_t	ndaoninvalidate;
150 static	void		ndadone(struct cam_periph *periph,
151 			       union ccb *done_ccb);
152 static  int		ndaerror(union ccb *ccb, u_int32_t cam_flags,
153 				u_int32_t sense_flags);
154 static void		ndashutdown(void *arg, int howto);
155 static void		ndasuspend(void *arg);
156 
157 #ifndef	NDA_DEFAULT_SEND_ORDERED
158 #define	NDA_DEFAULT_SEND_ORDERED	1
159 #endif
160 #ifndef NDA_DEFAULT_TIMEOUT
161 #define NDA_DEFAULT_TIMEOUT 30	/* Timeout in seconds */
162 #endif
163 #ifndef	NDA_DEFAULT_RETRY
164 #define	NDA_DEFAULT_RETRY	4
165 #endif
166 #ifndef NDA_MAX_TRIM_ENTRIES
167 #define NDA_MAX_TRIM_ENTRIES  (NVME_MAX_DSM_TRIM / sizeof(struct nvme_dsm_range))/* Number of DSM trims to use, max 256 */
168 #endif
169 
170 static SYSCTL_NODE(_kern_cam, OID_AUTO, nda, CTLFLAG_RD, 0,
171             "CAM Direct Access Disk driver");
172 
173 //static int nda_retry_count = NDA_DEFAULT_RETRY;
174 static int nda_send_ordered = NDA_DEFAULT_SEND_ORDERED;
175 static int nda_default_timeout = NDA_DEFAULT_TIMEOUT;
176 static int nda_max_trim_entries = NDA_MAX_TRIM_ENTRIES;
177 SYSCTL_INT(_kern_cam_nda, OID_AUTO, max_trim, CTLFLAG_RDTUN,
178     &nda_max_trim_entries, NDA_MAX_TRIM_ENTRIES,
179     "Maximum number of BIO_DELETE to send down as a DSM TRIM.");
180 
181 /*
182  * All NVMe media is non-rotational, so all nvme device instances
183  * share this to implement the sysctl.
184  */
185 static int nda_rotating_media = 0;
186 
187 static struct periph_driver ndadriver =
188 {
189 	ndainit, "nda",
190 	TAILQ_HEAD_INITIALIZER(ndadriver.units), /* generation */ 0
191 };
192 
193 PERIPHDRIVER_DECLARE(nda, ndadriver);
194 
195 static MALLOC_DEFINE(M_NVMEDA, "nvme_da", "nvme_da buffers");
196 
197 /*
198  * nice wrappers. Maybe these belong in nvme_all.c instead of
199  * here, but this is the only place that uses these. Should
200  * we ever grow another NVME periph, we should move them
201  * all there wholesale.
202  */
203 
204 static void
205 nda_nvme_flush(struct nda_softc *softc, struct ccb_nvmeio *nvmeio)
206 {
207 	cam_fill_nvmeio(nvmeio,
208 	    0,			/* retries */
209 	    ndadone,		/* cbfcnp */
210 	    CAM_DIR_NONE,	/* flags */
211 	    NULL,		/* data_ptr */
212 	    0,			/* dxfer_len */
213 	    nda_default_timeout * 1000); /* timeout 30s */
214 	nvme_ns_flush_cmd(&nvmeio->cmd, softc->nsid);
215 }
216 
217 static void
218 nda_nvme_trim(struct nda_softc *softc, struct ccb_nvmeio *nvmeio,
219     void *payload, uint32_t num_ranges)
220 {
221 	cam_fill_nvmeio(nvmeio,
222 	    0,			/* retries */
223 	    ndadone,		/* cbfcnp */
224 	    CAM_DIR_OUT,	/* flags */
225 	    payload,		/* data_ptr */
226 	    num_ranges * sizeof(struct nvme_dsm_range), /* dxfer_len */
227 	    nda_default_timeout * 1000); /* timeout 30s */
228 	nvme_ns_trim_cmd(&nvmeio->cmd, softc->nsid, num_ranges);
229 }
230 
231 static void
232 nda_nvme_write(struct nda_softc *softc, struct ccb_nvmeio *nvmeio,
233     void *payload, uint64_t lba, uint32_t len, uint32_t count)
234 {
235 	cam_fill_nvmeio(nvmeio,
236 	    0,			/* retries */
237 	    ndadone,		/* cbfcnp */
238 	    CAM_DIR_OUT,	/* flags */
239 	    payload,		/* data_ptr */
240 	    len,		/* dxfer_len */
241 	    nda_default_timeout * 1000); /* timeout 30s */
242 	nvme_ns_write_cmd(&nvmeio->cmd, softc->nsid, lba, count);
243 }
244 
245 static void
246 nda_nvme_rw_bio(struct nda_softc *softc, struct ccb_nvmeio *nvmeio,
247     struct bio *bp, uint32_t rwcmd)
248 {
249 	int flags = rwcmd == NVME_OPC_READ ? CAM_DIR_IN : CAM_DIR_OUT;
250 	void *payload;
251 	uint64_t lba;
252 	uint32_t count;
253 
254 	if (bp->bio_flags & BIO_UNMAPPED) {
255 		flags |= CAM_DATA_BIO;
256 		payload = bp;
257 	} else {
258 		payload = bp->bio_data;
259 	}
260 
261 	lba = bp->bio_pblkno;
262 	count = bp->bio_bcount / softc->disk->d_sectorsize;
263 
264 	cam_fill_nvmeio(nvmeio,
265 	    0,			/* retries */
266 	    ndadone,		/* cbfcnp */
267 	    flags,		/* flags */
268 	    payload,		/* data_ptr */
269 	    bp->bio_bcount,	/* dxfer_len */
270 	    nda_default_timeout * 1000); /* timeout 30s */
271 	nvme_ns_rw_cmd(&nvmeio->cmd, rwcmd, softc->nsid, lba, count);
272 }
273 
274 static int
275 ndaopen(struct disk *dp)
276 {
277 	struct cam_periph *periph;
278 	struct nda_softc *softc;
279 	int error;
280 
281 	periph = (struct cam_periph *)dp->d_drv1;
282 	if (cam_periph_acquire(periph) != 0) {
283 		return(ENXIO);
284 	}
285 
286 	cam_periph_lock(periph);
287 	if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
288 		cam_periph_unlock(periph);
289 		cam_periph_release(periph);
290 		return (error);
291 	}
292 
293 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
294 	    ("ndaopen\n"));
295 
296 	softc = (struct nda_softc *)periph->softc;
297 	softc->flags |= NDA_FLAG_OPEN;
298 
299 	cam_periph_unhold(periph);
300 	cam_periph_unlock(periph);
301 	return (0);
302 }
303 
304 static int
305 ndaclose(struct disk *dp)
306 {
307 	struct	cam_periph *periph;
308 	struct	nda_softc *softc;
309 	union ccb *ccb;
310 	int error;
311 
312 	periph = (struct cam_periph *)dp->d_drv1;
313 	softc = (struct nda_softc *)periph->softc;
314 	cam_periph_lock(periph);
315 
316 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
317 	    ("ndaclose\n"));
318 
319 	if ((softc->flags & NDA_FLAG_DIRTY) != 0 &&
320 	    (periph->flags & CAM_PERIPH_INVALID) == 0 &&
321 	    cam_periph_hold(periph, PRIBIO) == 0) {
322 
323 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
324 		nda_nvme_flush(softc, &ccb->nvmeio);
325 		error = cam_periph_runccb(ccb, ndaerror, /*cam_flags*/0,
326 		    /*sense_flags*/0, softc->disk->d_devstat);
327 
328 		if (error != 0)
329 			xpt_print(periph->path, "Synchronize cache failed\n");
330 		else
331 			softc->flags &= ~NDA_FLAG_DIRTY;
332 		xpt_release_ccb(ccb);
333 		cam_periph_unhold(periph);
334 	}
335 
336 	softc->flags &= ~NDA_FLAG_OPEN;
337 
338 	while (softc->refcount != 0)
339 		cam_periph_sleep(periph, &softc->refcount, PRIBIO, "ndaclose", 1);
340 	KASSERT(softc->outstanding_cmds == 0,
341 	    ("nda %d outstanding commands", softc->outstanding_cmds));
342 	cam_periph_unlock(periph);
343 	cam_periph_release(periph);
344 	return (0);
345 }
346 
347 static void
348 ndaschedule(struct cam_periph *periph)
349 {
350 	struct nda_softc *softc = (struct nda_softc *)periph->softc;
351 
352 	if (softc->state != NDA_STATE_NORMAL)
353 		return;
354 
355 	cam_iosched_schedule(softc->cam_iosched, periph);
356 }
357 
358 /*
359  * Actually translate the requested transfer into one the physical driver
360  * can understand.  The transfer is described by a buf and will include
361  * only one physical transfer.
362  */
363 static void
364 ndastrategy(struct bio *bp)
365 {
366 	struct cam_periph *periph;
367 	struct nda_softc *softc;
368 
369 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
370 	softc = (struct nda_softc *)periph->softc;
371 
372 	cam_periph_lock(periph);
373 
374 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastrategy(%p)\n", bp));
375 
376 	/*
377 	 * If the device has been made invalid, error out
378 	 */
379 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
380 		cam_periph_unlock(periph);
381 		biofinish(bp, NULL, ENXIO);
382 		return;
383 	}
384 
385 	if (bp->bio_cmd == BIO_DELETE)
386 		softc->deletes++;
387 
388 	/*
389 	 * Place it in the queue of disk activities for this disk
390 	 */
391 	cam_iosched_queue_work(softc->cam_iosched, bp);
392 
393 	/*
394 	 * Schedule ourselves for performing the work.
395 	 */
396 	ndaschedule(periph);
397 	cam_periph_unlock(periph);
398 
399 	return;
400 }
401 
402 static int
403 ndadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
404 {
405 	struct	    cam_periph *periph;
406 	struct	    nda_softc *softc;
407 	u_int	    secsize;
408 	struct ccb_nvmeio nvmeio;
409 	struct	    disk *dp;
410 	uint64_t    lba;
411 	uint32_t    count;
412 	int	    error = 0;
413 
414 	dp = arg;
415 	periph = dp->d_drv1;
416 	softc = (struct nda_softc *)periph->softc;
417 	secsize = softc->disk->d_sectorsize;
418 	lba = offset / secsize;
419 	count = length / secsize;
420 
421 	if ((periph->flags & CAM_PERIPH_INVALID) != 0)
422 		return (ENXIO);
423 
424 	/* xpt_get_ccb returns a zero'd allocation for the ccb, mimic that here */
425 	memset(&nvmeio, 0, sizeof(nvmeio));
426 	if (length > 0) {
427 		xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
428 		nvmeio.ccb_state = NDA_CCB_DUMP;
429 		nda_nvme_write(softc, &nvmeio, virtual, lba, length, count);
430 		error = cam_periph_runccb((union ccb *)&nvmeio, cam_periph_error,
431 		    0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
432 		if (error != 0)
433 			printf("Aborting dump due to I/O error %d.\n", error);
434 
435 		return (error);
436 	}
437 
438 	/* Flush */
439 	xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
440 
441 	nvmeio.ccb_state = NDA_CCB_DUMP;
442 	nda_nvme_flush(softc, &nvmeio);
443 	error = cam_periph_runccb((union ccb *)&nvmeio, cam_periph_error,
444 	    0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
445 	if (error != 0)
446 		xpt_print(periph->path, "flush cmd failed\n");
447 	return (error);
448 }
449 
450 static void
451 ndainit(void)
452 {
453 	cam_status status;
454 
455 	/*
456 	 * Install a global async callback.  This callback will
457 	 * receive async callbacks like "new device found".
458 	 */
459 	status = xpt_register_async(AC_FOUND_DEVICE, ndaasync, NULL, NULL);
460 
461 	if (status != CAM_REQ_CMP) {
462 		printf("nda: Failed to attach master async callback "
463 		       "due to status 0x%x!\n", status);
464 	} else if (nda_send_ordered) {
465 
466 		/* Register our event handlers */
467 		if ((EVENTHANDLER_REGISTER(power_suspend, ndasuspend,
468 					   NULL, EVENTHANDLER_PRI_LAST)) == NULL)
469 		    printf("ndainit: power event registration failed!\n");
470 		if ((EVENTHANDLER_REGISTER(shutdown_post_sync, ndashutdown,
471 					   NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
472 		    printf("ndainit: shutdown event registration failed!\n");
473 	}
474 }
475 
476 /*
477  * Callback from GEOM, called when it has finished cleaning up its
478  * resources.
479  */
480 static void
481 ndadiskgonecb(struct disk *dp)
482 {
483 	struct cam_periph *periph;
484 
485 	periph = (struct cam_periph *)dp->d_drv1;
486 
487 	cam_periph_release(periph);
488 }
489 
490 static void
491 ndaoninvalidate(struct cam_periph *periph)
492 {
493 	struct nda_softc *softc;
494 
495 	softc = (struct nda_softc *)periph->softc;
496 
497 	/*
498 	 * De-register any async callbacks.
499 	 */
500 	xpt_register_async(0, ndaasync, periph, periph->path);
501 #ifdef CAM_IO_STATS
502 	softc->invalidations++;
503 #endif
504 
505 	/*
506 	 * Return all queued I/O with ENXIO.
507 	 * XXX Handle any transactions queued to the card
508 	 *     with XPT_ABORT_CCB.
509 	 */
510 	cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
511 
512 	disk_gone(softc->disk);
513 }
514 
515 static void
516 ndacleanup(struct cam_periph *periph)
517 {
518 	struct nda_softc *softc;
519 
520 	softc = (struct nda_softc *)periph->softc;
521 
522 	cam_periph_unlock(periph);
523 
524 	cam_iosched_fini(softc->cam_iosched);
525 
526 	/*
527 	 * If we can't free the sysctl tree, oh well...
528 	 */
529 	if ((softc->flags & NDA_FLAG_SCTX_INIT) != 0) {
530 #ifdef CAM_IO_STATS
531 		if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
532 			xpt_print(periph->path,
533 			    "can't remove sysctl stats context\n");
534 #endif
535 		if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
536 			xpt_print(periph->path,
537 			    "can't remove sysctl context\n");
538 	}
539 
540 	disk_destroy(softc->disk);
541 	free(softc, M_DEVBUF);
542 	cam_periph_lock(periph);
543 }
544 
545 static void
546 ndaasync(void *callback_arg, u_int32_t code,
547 	struct cam_path *path, void *arg)
548 {
549 	struct cam_periph *periph;
550 
551 	periph = (struct cam_periph *)callback_arg;
552 	switch (code) {
553 	case AC_FOUND_DEVICE:
554 	{
555 		struct ccb_getdev *cgd;
556 		cam_status status;
557 
558 		cgd = (struct ccb_getdev *)arg;
559 		if (cgd == NULL)
560 			break;
561 
562 		if (cgd->protocol != PROTO_NVME)
563 			break;
564 
565 		/*
566 		 * Allocate a peripheral instance for
567 		 * this device and start the probe
568 		 * process.
569 		 */
570 		status = cam_periph_alloc(ndaregister, ndaoninvalidate,
571 					  ndacleanup, ndastart,
572 					  "nda", CAM_PERIPH_BIO,
573 					  path, ndaasync,
574 					  AC_FOUND_DEVICE, cgd);
575 
576 		if (status != CAM_REQ_CMP
577 		 && status != CAM_REQ_INPROG)
578 			printf("ndaasync: Unable to attach to new device "
579 				"due to status 0x%x\n", status);
580 		break;
581 	}
582 	case AC_ADVINFO_CHANGED:
583 	{
584 		uintptr_t buftype;
585 
586 		buftype = (uintptr_t)arg;
587 		if (buftype == CDAI_TYPE_PHYS_PATH) {
588 			struct nda_softc *softc;
589 
590 			softc = periph->softc;
591 			disk_attr_changed(softc->disk, "GEOM::physpath",
592 					  M_NOWAIT);
593 		}
594 		break;
595 	}
596 	case AC_LOST_DEVICE:
597 	default:
598 		cam_periph_async(periph, code, path, arg);
599 		break;
600 	}
601 }
602 
603 static void
604 ndasysctlinit(void *context, int pending)
605 {
606 	struct cam_periph *periph;
607 	struct nda_softc *softc;
608 	char tmpstr[32], tmpstr2[16];
609 
610 	periph = (struct cam_periph *)context;
611 
612 	/* periph was held for us when this task was enqueued */
613 	if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
614 		cam_periph_release(periph);
615 		return;
616 	}
617 
618 	softc = (struct nda_softc *)periph->softc;
619 	snprintf(tmpstr, sizeof(tmpstr), "CAM NDA unit %d", periph->unit_number);
620 	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
621 
622 	sysctl_ctx_init(&softc->sysctl_ctx);
623 	softc->flags |= NDA_FLAG_SCTX_INIT;
624 	softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
625 		SYSCTL_STATIC_CHILDREN(_kern_cam_nda), OID_AUTO, tmpstr2,
626 		CTLFLAG_RD, 0, tmpstr, "device_index");
627 	if (softc->sysctl_tree == NULL) {
628 		printf("ndasysctlinit: unable to allocate sysctl tree\n");
629 		cam_periph_release(periph);
630 		return;
631 	}
632 
633 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
634 	    OID_AUTO, "unmapped_io", CTLFLAG_RD,
635 	    &softc->unmappedio, 0, "Unmapped I/O leaf");
636 
637 	SYSCTL_ADD_QUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
638 	    OID_AUTO, "deletes", CTLFLAG_RD,
639 	    &softc->deletes, "Number of BIO_DELETE requests");
640 
641 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
642 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
643 		"trim_count", CTLFLAG_RD, &softc->trim_count,
644 		"Total number of unmap/dsm commands sent");
645 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
646 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
647 		"trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
648 		"Total number of ranges in unmap/dsm commands");
649 	SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
650 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
651 		"trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
652 		"Total lbas in the unmap/dsm commands sent");
653 
654 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
655 	    OID_AUTO, "rotating", CTLFLAG_RD, &nda_rotating_media, 1,
656 	    "Rotating media");
657 
658 #ifdef CAM_IO_STATS
659 	softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
660 		SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
661 		CTLFLAG_RD, 0, "Statistics");
662 	if (softc->sysctl_stats_tree == NULL) {
663 		printf("ndasysctlinit: unable to allocate sysctl tree for stats\n");
664 		cam_periph_release(periph);
665 		return;
666 	}
667 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
668 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
669 		OID_AUTO, "timeouts", CTLFLAG_RD,
670 		&softc->timeouts, 0,
671 		"Device timeouts reported by the SIM");
672 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
673 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
674 		OID_AUTO, "errors", CTLFLAG_RD,
675 		&softc->errors, 0,
676 		"Transport errors reported by the SIM.");
677 	SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
678 		SYSCTL_CHILDREN(softc->sysctl_stats_tree),
679 		OID_AUTO, "pack_invalidations", CTLFLAG_RD,
680 		&softc->invalidations, 0,
681 		"Device pack invalidations.");
682 #endif
683 
684 #ifdef CAM_TEST_FAILURE
685 	SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
686 		OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
687 		periph, 0, cam_periph_invalidate_sysctl, "I",
688 		"Write 1 to invalidate the drive immediately");
689 #endif
690 
691 	cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
692 	    softc->sysctl_tree);
693 
694 	cam_periph_release(periph);
695 }
696 
697 static int
698 ndagetattr(struct bio *bp)
699 {
700 	int ret;
701 	struct cam_periph *periph;
702 
703 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
704 	cam_periph_lock(periph);
705 	ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
706 	    periph->path);
707 	cam_periph_unlock(periph);
708 	if (ret == 0)
709 		bp->bio_completed = bp->bio_length;
710 	return ret;
711 }
712 
713 static cam_status
714 ndaregister(struct cam_periph *periph, void *arg)
715 {
716 	struct nda_softc *softc;
717 	struct disk *disk;
718 	struct ccb_pathinq cpi;
719 	const struct nvme_namespace_data *nsd;
720 	const struct nvme_controller_data *cd;
721 	char   announce_buf[80];
722 	uint8_t flbas_fmt, lbads, vwc_present;
723 	u_int maxio;
724 	int quirks;
725 
726 	nsd = nvme_get_identify_ns(periph);
727 	cd = nvme_get_identify_cntrl(periph);
728 
729 	softc = (struct nda_softc *)malloc(sizeof(*softc), M_DEVBUF,
730 	    M_NOWAIT | M_ZERO);
731 
732 	if (softc == NULL) {
733 		printf("ndaregister: Unable to probe new device. "
734 		    "Unable to allocate softc\n");
735 		return(CAM_REQ_CMP_ERR);
736 	}
737 
738 	if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
739 		printf("ndaregister: Unable to probe new device. "
740 		       "Unable to allocate iosched memory\n");
741 		free(softc, M_DEVBUF);
742 		return(CAM_REQ_CMP_ERR);
743 	}
744 
745 	/* ident_data parsing */
746 
747 	periph->softc = softc;
748 
749 	softc->quirks = NDA_Q_NONE;
750 
751 	xpt_path_inq(&cpi, periph->path);
752 
753 	TASK_INIT(&softc->sysctl_task, 0, ndasysctlinit, periph);
754 
755 	/*
756 	 * The name space ID is the lun, save it for later I/O
757 	 */
758 	softc->nsid = (uint32_t)xpt_path_lun_id(periph->path);
759 
760 	/*
761 	 * Register this media as a disk
762 	 */
763 	(void)cam_periph_hold(periph, PRIBIO);
764 	cam_periph_unlock(periph);
765 	snprintf(announce_buf, sizeof(announce_buf),
766 	    "kern.cam.nda.%d.quirks", periph->unit_number);
767 	quirks = softc->quirks;
768 	TUNABLE_INT_FETCH(announce_buf, &quirks);
769 	softc->quirks = quirks;
770 	cam_iosched_set_sort_queue(softc->cam_iosched, 0);
771 	softc->disk = disk = disk_alloc();
772 	disk->d_rotation_rate = DISK_RR_NON_ROTATING;
773 	disk->d_open = ndaopen;
774 	disk->d_close = ndaclose;
775 	disk->d_strategy = ndastrategy;
776 	disk->d_getattr = ndagetattr;
777 	disk->d_dump = ndadump;
778 	disk->d_gone = ndadiskgonecb;
779 	disk->d_name = "nda";
780 	disk->d_drv1 = periph;
781 	disk->d_unit = periph->unit_number;
782 	maxio = cpi.maxio;		/* Honor max I/O size of SIM */
783 	if (maxio == 0)
784 		maxio = DFLTPHYS;	/* traditional default */
785 	else if (maxio > MAXPHYS)
786 		maxio = MAXPHYS;	/* for safety */
787 	disk->d_maxsize = maxio;
788 	flbas_fmt = (nsd->flbas >> NVME_NS_DATA_FLBAS_FORMAT_SHIFT) &
789 		NVME_NS_DATA_FLBAS_FORMAT_MASK;
790 	lbads = (nsd->lbaf[flbas_fmt] >> NVME_NS_DATA_LBAF_LBADS_SHIFT) &
791 		NVME_NS_DATA_LBAF_LBADS_MASK;
792 	disk->d_sectorsize = 1 << lbads;
793 	disk->d_mediasize = (off_t)(disk->d_sectorsize * nsd->nsze);
794 	disk->d_delmaxsize = disk->d_mediasize;
795 	disk->d_flags = DISKFLAG_DIRECT_COMPLETION;
796 	if (nvme_ctrlr_has_dataset_mgmt(cd))
797 		disk->d_flags |= DISKFLAG_CANDELETE;
798 	vwc_present = (cd->vwc >> NVME_CTRLR_DATA_VWC_PRESENT_SHIFT) &
799 		NVME_CTRLR_DATA_VWC_PRESENT_MASK;
800 	if (vwc_present)
801 		disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
802 	if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
803 		disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
804 		softc->unmappedio = 1;
805 	}
806 	/*
807 	 * d_ident and d_descr are both far bigger than the length of either
808 	 *  the serial or model number strings.
809 	 */
810 	cam_strvis(disk->d_descr, cd->mn,
811 	    sizeof(disk->d_descr), NVME_MODEL_NUMBER_LENGTH);
812 	cam_strvis(disk->d_ident, cd->sn,
813 	    sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH);
814 	disk->d_hba_vendor = cpi.hba_vendor;
815 	disk->d_hba_device = cpi.hba_device;
816 	disk->d_hba_subvendor = cpi.hba_subvendor;
817 	disk->d_hba_subdevice = cpi.hba_subdevice;
818 	disk->d_stripesize = disk->d_sectorsize;
819 	disk->d_stripeoffset = 0;
820 	disk->d_devstat = devstat_new_entry(periph->periph_name,
821 	    periph->unit_number, disk->d_sectorsize,
822 	    DEVSTAT_ALL_SUPPORTED,
823 	    DEVSTAT_TYPE_DIRECT | XPORT_DEVSTAT_TYPE(cpi.transport),
824 	    DEVSTAT_PRIORITY_DISK);
825 	/*
826 	 * Add alias for older nvd drives to ease transition.
827 	 */
828 	/* disk_add_alias(disk, "nvd"); Have reports of this causing problems */
829 
830 	/*
831 	 * Acquire a reference to the periph before we register with GEOM.
832 	 * We'll release this reference once GEOM calls us back (via
833 	 * ndadiskgonecb()) telling us that our provider has been freed.
834 	 */
835 	if (cam_periph_acquire(periph) != 0) {
836 		xpt_print(periph->path, "%s: lost periph during "
837 			  "registration!\n", __func__);
838 		cam_periph_lock(periph);
839 		return (CAM_REQ_CMP_ERR);
840 	}
841 	disk_create(softc->disk, DISK_VERSION);
842 	cam_periph_lock(periph);
843 	cam_periph_unhold(periph);
844 
845 	snprintf(announce_buf, sizeof(announce_buf),
846 		"%juMB (%ju %u byte sectors)",
847 	    (uintmax_t)((uintmax_t)disk->d_mediasize / (1024*1024)),
848 		(uintmax_t)disk->d_mediasize / disk->d_sectorsize,
849 		disk->d_sectorsize);
850 	xpt_announce_periph(periph, announce_buf);
851 	xpt_announce_quirks(periph, softc->quirks, NDA_Q_BIT_STRING);
852 
853 	/*
854 	 * Create our sysctl variables, now that we know
855 	 * we have successfully attached.
856 	 */
857 	if (cam_periph_acquire(periph) == 0)
858 		taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
859 
860 	/*
861 	 * Register for device going away and info about the drive
862 	 * changing (though with NVMe, it can't)
863 	 */
864 	xpt_register_async(AC_LOST_DEVICE | AC_ADVINFO_CHANGED,
865 	    ndaasync, periph, periph->path);
866 
867 	softc->state = NDA_STATE_NORMAL;
868 	return(CAM_REQ_CMP);
869 }
870 
871 static void
872 ndastart(struct cam_periph *periph, union ccb *start_ccb)
873 {
874 	struct nda_softc *softc = (struct nda_softc *)periph->softc;
875 	struct ccb_nvmeio *nvmeio = &start_ccb->nvmeio;
876 
877 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart\n"));
878 
879 	switch (softc->state) {
880 	case NDA_STATE_NORMAL:
881 	{
882 		struct bio *bp;
883 
884 		bp = cam_iosched_next_bio(softc->cam_iosched);
885 		CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart: bio %p\n", bp));
886 		if (bp == NULL) {
887 			xpt_release_ccb(start_ccb);
888 			break;
889 		}
890 
891 		switch (bp->bio_cmd) {
892 		case BIO_WRITE:
893 			softc->flags |= NDA_FLAG_DIRTY;
894 			/* FALLTHROUGH */
895 		case BIO_READ:
896 		{
897 #ifdef CAM_TEST_FAILURE
898 			int fail = 0;
899 
900 			/*
901 			 * Support the failure ioctls.  If the command is a
902 			 * read, and there are pending forced read errors, or
903 			 * if a write and pending write errors, then fail this
904 			 * operation with EIO.  This is useful for testing
905 			 * purposes.  Also, support having every Nth read fail.
906 			 *
907 			 * This is a rather blunt tool.
908 			 */
909 			if (bp->bio_cmd == BIO_READ) {
910 				if (softc->force_read_error) {
911 					softc->force_read_error--;
912 					fail = 1;
913 				}
914 				if (softc->periodic_read_error > 0) {
915 					if (++softc->periodic_read_count >=
916 					    softc->periodic_read_error) {
917 						softc->periodic_read_count = 0;
918 						fail = 1;
919 					}
920 				}
921 			} else {
922 				if (softc->force_write_error) {
923 					softc->force_write_error--;
924 					fail = 1;
925 				}
926 			}
927 			if (fail) {
928 				biofinish(bp, NULL, EIO);
929 				xpt_release_ccb(start_ccb);
930 				ndaschedule(periph);
931 				return;
932 			}
933 #endif
934 			KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
935 			    round_page(bp->bio_bcount + bp->bio_ma_offset) /
936 			    PAGE_SIZE == bp->bio_ma_n,
937 			    ("Short bio %p", bp));
938 			nda_nvme_rw_bio(softc, &start_ccb->nvmeio, bp, bp->bio_cmd == BIO_READ ?
939 			    NVME_OPC_READ : NVME_OPC_WRITE);
940 			break;
941 		}
942 		case BIO_DELETE:
943 		{
944 			struct nvme_dsm_range *dsm_range, *dsm_end;
945 			struct nda_trim_request *trim;
946 			struct bio *bp1;
947 			int ents;
948 			uint32_t totalcount = 0, ranges = 0;
949 
950 			trim = malloc(sizeof(*trim), M_NVMEDA, M_ZERO | M_NOWAIT);
951 			if (trim == NULL) {
952 				biofinish(bp, NULL, ENOMEM);
953 				xpt_release_ccb(start_ccb);
954 				ndaschedule(periph);
955 				return;
956 			}
957 			TAILQ_INIT(&trim->bps);
958 			bp1 = bp;
959 			ents = min(nitems(trim->dsm), nda_max_trim_entries);
960 			dsm_range = trim->dsm;
961 			dsm_end = dsm_range + ents;
962 			do {
963 				TAILQ_INSERT_TAIL(&trim->bps, bp1, bio_queue);
964 				dsm_range->length =
965 				    htole32(bp1->bio_bcount / softc->disk->d_sectorsize);
966 				dsm_range->starting_lba =
967 				    htole64(bp1->bio_offset / softc->disk->d_sectorsize);
968 				ranges++;
969 				totalcount += dsm_range->length;
970 				dsm_range++;
971 				if (dsm_range >= dsm_end)
972 					break;
973 				bp1 = cam_iosched_next_trim(softc->cam_iosched);
974 				/* XXX -- Could collapse adjacent ranges, but we don't for now */
975 				/* XXX -- Could limit based on total payload size */
976 			} while (bp1 != NULL);
977 			start_ccb->ccb_trim = trim;
978 			nda_nvme_trim(softc, &start_ccb->nvmeio, trim->dsm,
979 			    dsm_range - trim->dsm);
980 			start_ccb->ccb_state = NDA_CCB_TRIM;
981 			softc->trim_count++;
982 			softc->trim_ranges += ranges;
983 			softc->trim_lbas += totalcount;
984 			/*
985 			 * Note: We can have multiple TRIMs in flight, so we don't call
986 			 * cam_iosched_submit_trim(softc->cam_iosched);
987 			 * since that forces the I/O scheduler to only schedule one at a time.
988 			 * On NVMe drives, this is a performance disaster.
989 			 */
990 			goto out;
991 		}
992 		case BIO_FLUSH:
993 			nda_nvme_flush(softc, nvmeio);
994 			break;
995 		}
996 		start_ccb->ccb_state = NDA_CCB_BUFFER_IO;
997 		start_ccb->ccb_bp = bp;
998 out:
999 		start_ccb->ccb_h.flags |= CAM_UNLOCKED;
1000 		softc->outstanding_cmds++;
1001 		softc->refcount++;			/* For submission only */
1002 		cam_periph_unlock(periph);
1003 		xpt_action(start_ccb);
1004 		cam_periph_lock(periph);
1005 		softc->refcount--;			/* Submission done */
1006 
1007 		/* May have more work to do, so ensure we stay scheduled */
1008 		ndaschedule(periph);
1009 		break;
1010 		}
1011 	}
1012 }
1013 
1014 static void
1015 ndadone(struct cam_periph *periph, union ccb *done_ccb)
1016 {
1017 	struct nda_softc *softc;
1018 	struct ccb_nvmeio *nvmeio = &done_ccb->nvmeio;
1019 	struct cam_path *path;
1020 	int state;
1021 
1022 	softc = (struct nda_softc *)periph->softc;
1023 	path = done_ccb->ccb_h.path;
1024 
1025 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("ndadone\n"));
1026 
1027 	state = nvmeio->ccb_state & NDA_CCB_TYPE_MASK;
1028 	switch (state) {
1029 	case NDA_CCB_BUFFER_IO:
1030 	case NDA_CCB_TRIM:
1031 	{
1032 		int error;
1033 
1034 		cam_periph_lock(periph);
1035 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1036 			error = ndaerror(done_ccb, 0, 0);
1037 			if (error == ERESTART) {
1038 				/* A retry was scheduled, so just return. */
1039 				cam_periph_unlock(periph);
1040 				return;
1041 			}
1042 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1043 				cam_release_devq(path,
1044 						 /*relsim_flags*/0,
1045 						 /*reduction*/0,
1046 						 /*timeout*/0,
1047 						 /*getcount_only*/0);
1048 		} else {
1049 			if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1050 				panic("REQ_CMP with QFRZN");
1051 			error = 0;
1052 		}
1053 		if (state == NDA_CCB_BUFFER_IO) {
1054 			struct bio *bp;
1055 
1056 			bp = (struct bio *)done_ccb->ccb_bp;
1057 			bp->bio_error = error;
1058 			if (error != 0) {
1059 				bp->bio_resid = bp->bio_bcount;
1060 				bp->bio_flags |= BIO_ERROR;
1061 			} else {
1062 				bp->bio_resid = 0;
1063 			}
1064 			softc->outstanding_cmds--;
1065 
1066 			/*
1067 			 * We need to call cam_iosched before we call biodone so that we
1068 			 * don't measure any activity that happens in the completion
1069 			 * routine, which in the case of sendfile can be quite
1070 			 * extensive.
1071 			 */
1072 			cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
1073 			xpt_release_ccb(done_ccb);
1074 			ndaschedule(periph);
1075 			cam_periph_unlock(periph);
1076 			biodone(bp);
1077 		} else { /* state == NDA_CCB_TRIM */
1078 			struct nda_trim_request *trim;
1079 			struct bio *bp1, *bp2;
1080 			TAILQ_HEAD(, bio) queue;
1081 
1082 			trim = nvmeio->ccb_trim;
1083 			TAILQ_INIT(&queue);
1084 			TAILQ_CONCAT(&queue, &trim->bps, bio_queue);
1085 			free(trim, M_NVMEDA);
1086 
1087 			/*
1088 			 * Since we can have multiple trims in flight, we don't
1089 			 * need to call this here.
1090 			 * cam_iosched_trim_done(softc->cam_iosched);
1091 			 */
1092 			/*
1093 			 * The the I/O scheduler that we're finishing the I/O
1094 			 * so we can keep book. The first one we pass in the CCB
1095 			 * which has the timing information. The rest we pass in NULL
1096 			 * so we can keep proper counts.
1097 			 */
1098 			bp1 = TAILQ_FIRST(&queue);
1099 			cam_iosched_bio_complete(softc->cam_iosched, bp1, done_ccb);
1100 			xpt_release_ccb(done_ccb);
1101 			softc->outstanding_cmds--;
1102 			ndaschedule(periph);
1103 			cam_periph_unlock(periph);
1104 			while ((bp2 = TAILQ_FIRST(&queue)) != NULL) {
1105 				TAILQ_REMOVE(&queue, bp2, bio_queue);
1106 				bp2->bio_error = error;
1107 				if (error != 0) {
1108 					bp2->bio_flags |= BIO_ERROR;
1109 					bp2->bio_resid = bp1->bio_bcount;
1110 				} else
1111 					bp2->bio_resid = 0;
1112 				if (bp1 != bp2)
1113 					cam_iosched_bio_complete(softc->cam_iosched, bp2, NULL);
1114 				biodone(bp2);
1115 			}
1116 		}
1117 		return;
1118 	}
1119 	case NDA_CCB_DUMP:
1120 		/* No-op.  We're polling */
1121 		return;
1122 	default:
1123 		break;
1124 	}
1125 	xpt_release_ccb(done_ccb);
1126 }
1127 
1128 static int
1129 ndaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
1130 {
1131 	struct nda_softc *softc;
1132 	struct cam_periph *periph;
1133 
1134 	periph = xpt_path_periph(ccb->ccb_h.path);
1135 	softc = (struct nda_softc *)periph->softc;
1136 
1137 	switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
1138 	case CAM_CMD_TIMEOUT:
1139 #ifdef CAM_IO_STATS
1140 		softc->timeouts++;
1141 #endif
1142 		break;
1143 	case CAM_REQ_ABORTED:
1144 	case CAM_REQ_CMP_ERR:
1145 	case CAM_REQ_TERMIO:
1146 	case CAM_UNREC_HBA_ERROR:
1147 	case CAM_DATA_RUN_ERR:
1148 	case CAM_ATA_STATUS_ERROR:
1149 #ifdef CAM_IO_STATS
1150 		softc->errors++;
1151 #endif
1152 		break;
1153 	default:
1154 		break;
1155 	}
1156 
1157 	return(cam_periph_error(ccb, cam_flags, sense_flags));
1158 }
1159 
1160 /*
1161  * Step through all NDA peripheral drivers, and if the device is still open,
1162  * sync the disk cache to physical media.
1163  */
1164 static void
1165 ndaflush(void)
1166 {
1167 	struct cam_periph *periph;
1168 	struct nda_softc *softc;
1169 	union ccb *ccb;
1170 	int error;
1171 
1172 	CAM_PERIPH_FOREACH(periph, &ndadriver) {
1173 		softc = (struct nda_softc *)periph->softc;
1174 
1175 		if (SCHEDULER_STOPPED()) {
1176 			/*
1177 			 * If we paniced with the lock held or the periph is not
1178 			 * open, do not recurse.  Otherwise, call ndadump since
1179 			 * that avoids the sleeping cam_periph_getccb does if no
1180 			 * CCBs are available.
1181 			 */
1182 			if (!cam_periph_owned(periph) &&
1183 			    (softc->flags & NDA_FLAG_OPEN)) {
1184 				ndadump(softc->disk, NULL, 0, 0, 0);
1185 			}
1186 			continue;
1187 		}
1188 
1189 		/*
1190 		 * We only sync the cache if the drive is still open
1191 		 */
1192 		cam_periph_lock(periph);
1193 		if ((softc->flags & NDA_FLAG_OPEN) == 0) {
1194 			cam_periph_unlock(periph);
1195 			continue;
1196 		}
1197 
1198 		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1199 		nda_nvme_flush(softc, &ccb->nvmeio);
1200 		error = cam_periph_runccb(ccb, ndaerror, /*cam_flags*/0,
1201 		    /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
1202 		    softc->disk->d_devstat);
1203 		if (error != 0)
1204 			xpt_print(periph->path, "Synchronize cache failed\n");
1205 		xpt_release_ccb(ccb);
1206 		cam_periph_unlock(periph);
1207 	}
1208 }
1209 
1210 static void
1211 ndashutdown(void *arg, int howto)
1212 {
1213 
1214 	ndaflush();
1215 }
1216 
1217 static void
1218 ndasuspend(void *arg)
1219 {
1220 
1221 	ndaflush();
1222 }
1223