xref: /freebsd/sys/dev/nvd/nvd.c (revision 232e2edb6cd50465bd62b9f7cbc9fd83531c02ca)
1bb0ec6b3SJim Harris /*-
2bb0ec6b3SJim Harris  * Copyright (C) 2012 Intel Corporation
3bb0ec6b3SJim Harris  * All rights reserved.
4bb0ec6b3SJim Harris  *
5bb0ec6b3SJim Harris  * Redistribution and use in source and binary forms, with or without
6bb0ec6b3SJim Harris  * modification, are permitted provided that the following conditions
7bb0ec6b3SJim Harris  * are met:
8bb0ec6b3SJim Harris  * 1. Redistributions of source code must retain the above copyright
9bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer.
10bb0ec6b3SJim Harris  * 2. Redistributions in binary form must reproduce the above copyright
11bb0ec6b3SJim Harris  *    notice, this list of conditions and the following disclaimer in the
12bb0ec6b3SJim Harris  *    documentation and/or other materials provided with the distribution.
13bb0ec6b3SJim Harris  *
14bb0ec6b3SJim Harris  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15bb0ec6b3SJim Harris  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16bb0ec6b3SJim Harris  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17bb0ec6b3SJim Harris  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18bb0ec6b3SJim Harris  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19bb0ec6b3SJim Harris  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20bb0ec6b3SJim Harris  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21bb0ec6b3SJim Harris  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22bb0ec6b3SJim Harris  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23bb0ec6b3SJim Harris  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24bb0ec6b3SJim Harris  * SUCH DAMAGE.
25bb0ec6b3SJim Harris  */
26bb0ec6b3SJim Harris 
27bb0ec6b3SJim Harris #include <sys/cdefs.h>
28bb0ec6b3SJim Harris __FBSDID("$FreeBSD$");
29bb0ec6b3SJim Harris 
30bb0ec6b3SJim Harris #include <sys/param.h>
31bb0ec6b3SJim Harris #include <sys/bio.h>
32bb0ec6b3SJim Harris #include <sys/kernel.h>
33bb0ec6b3SJim Harris #include <sys/malloc.h>
34bb0ec6b3SJim Harris #include <sys/module.h>
35bb0ec6b3SJim Harris #include <sys/systm.h>
36bb0ec6b3SJim Harris #include <sys/taskqueue.h>
37bb0ec6b3SJim Harris 
38bb0ec6b3SJim Harris #include <geom/geom.h>
39bb0ec6b3SJim Harris #include <geom/geom_disk.h>
40bb0ec6b3SJim Harris 
41bb0ec6b3SJim Harris #include <dev/nvme/nvme.h>
42bb0ec6b3SJim Harris 
43bb0ec6b3SJim Harris struct nvd_disk;
44bb0ec6b3SJim Harris 
45bb0ec6b3SJim Harris static disk_ioctl_t nvd_ioctl;
46bb0ec6b3SJim Harris static disk_strategy_t nvd_strategy;
47bb0ec6b3SJim Harris 
48ec84ecbbSJim Harris static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr);
49bb0ec6b3SJim Harris static void destroy_geom_disk(struct nvd_disk *ndisk);
50bb0ec6b3SJim Harris 
51ec84ecbbSJim Harris static void *nvd_new_controller(struct nvme_controller *ctrlr);
52*232e2edbSJim Harris static void nvd_controller_fail(void *ctrlr);
53ec84ecbbSJim Harris 
54bb0ec6b3SJim Harris static int nvd_load(void);
55bb0ec6b3SJim Harris static void nvd_unload(void);
56bb0ec6b3SJim Harris 
57bb0ec6b3SJim Harris MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations");
58bb0ec6b3SJim Harris 
59bb0ec6b3SJim Harris struct nvme_consumer *consumer_handle;
60bb0ec6b3SJim Harris 
61bb0ec6b3SJim Harris struct nvd_disk {
62bb0ec6b3SJim Harris 
63bb0ec6b3SJim Harris 	struct bio_queue_head	bioq;
64bb0ec6b3SJim Harris 	struct task		bioqtask;
65bb0ec6b3SJim Harris 	struct mtx		bioqlock;
66bb0ec6b3SJim Harris 
67bb0ec6b3SJim Harris 	struct disk		*disk;
68bb0ec6b3SJim Harris 	struct taskqueue	*tq;
69bb0ec6b3SJim Harris 	struct nvme_namespace	*ns;
70bb0ec6b3SJim Harris 
71bb0ec6b3SJim Harris 	uint32_t		cur_depth;
72bb0ec6b3SJim Harris 
73ec84ecbbSJim Harris 	TAILQ_ENTRY(nvd_disk)	global_tailq;
74ec84ecbbSJim Harris 	TAILQ_ENTRY(nvd_disk)	ctrlr_tailq;
75bb0ec6b3SJim Harris };
76bb0ec6b3SJim Harris 
77ec84ecbbSJim Harris struct nvd_controller {
78ec84ecbbSJim Harris 
79ec84ecbbSJim Harris 	TAILQ_ENTRY(nvd_controller)	tailq;
80ec84ecbbSJim Harris 	TAILQ_HEAD(, nvd_disk)		disk_head;
81ec84ecbbSJim Harris };
82ec84ecbbSJim Harris 
83ec84ecbbSJim Harris static TAILQ_HEAD(, nvd_controller)	ctrlr_head;
84ec84ecbbSJim Harris static TAILQ_HEAD(disk_list, nvd_disk)	disk_head;
85bb0ec6b3SJim Harris 
86bb0ec6b3SJim Harris static int nvd_modevent(module_t mod, int type, void *arg)
87bb0ec6b3SJim Harris {
88bb0ec6b3SJim Harris 	int error = 0;
89bb0ec6b3SJim Harris 
90bb0ec6b3SJim Harris 	switch (type) {
91bb0ec6b3SJim Harris 	case MOD_LOAD:
92bb0ec6b3SJim Harris 		error = nvd_load();
93bb0ec6b3SJim Harris 		break;
94bb0ec6b3SJim Harris 	case MOD_UNLOAD:
95bb0ec6b3SJim Harris 		nvd_unload();
96bb0ec6b3SJim Harris 		break;
97bb0ec6b3SJim Harris 	default:
98bb0ec6b3SJim Harris 		break;
99bb0ec6b3SJim Harris 	}
100bb0ec6b3SJim Harris 
101bb0ec6b3SJim Harris 	return (error);
102bb0ec6b3SJim Harris }
103bb0ec6b3SJim Harris 
104bb0ec6b3SJim Harris moduledata_t nvd_mod = {
105bb0ec6b3SJim Harris 	"nvd",
106bb0ec6b3SJim Harris 	(modeventhand_t)nvd_modevent,
1079823d527SKevin Lo 	0
108bb0ec6b3SJim Harris };
109bb0ec6b3SJim Harris 
110bb0ec6b3SJim Harris DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
111bb0ec6b3SJim Harris MODULE_VERSION(nvd, 1);
112bb0ec6b3SJim Harris MODULE_DEPEND(nvd, nvme, 1, 1, 1);
113bb0ec6b3SJim Harris 
114bb0ec6b3SJim Harris static int
115bb0ec6b3SJim Harris nvd_load()
116bb0ec6b3SJim Harris {
117bb0ec6b3SJim Harris 
118ec84ecbbSJim Harris 	TAILQ_INIT(&ctrlr_head);
119ec84ecbbSJim Harris 	TAILQ_INIT(&disk_head);
120ec84ecbbSJim Harris 
121ec84ecbbSJim Harris 	consumer_handle = nvme_register_consumer(nvd_new_disk,
122*232e2edbSJim Harris 	    nvd_new_controller, NULL, nvd_controller_fail);
123bb0ec6b3SJim Harris 
124bb0ec6b3SJim Harris 	return (consumer_handle != NULL ? 0 : -1);
125bb0ec6b3SJim Harris }
126bb0ec6b3SJim Harris 
127bb0ec6b3SJim Harris static void
128bb0ec6b3SJim Harris nvd_unload()
129bb0ec6b3SJim Harris {
130ec84ecbbSJim Harris 	struct nvd_controller	*ctrlr;
131ec84ecbbSJim Harris 	struct nvd_disk		*disk;
132bb0ec6b3SJim Harris 
133ec84ecbbSJim Harris 	while (!TAILQ_EMPTY(&ctrlr_head)) {
134ec84ecbbSJim Harris 		ctrlr = TAILQ_FIRST(&ctrlr_head);
135ec84ecbbSJim Harris 		TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
136ec84ecbbSJim Harris 		free(ctrlr, M_NVD);
137ec84ecbbSJim Harris 	}
138ec84ecbbSJim Harris 
139ec84ecbbSJim Harris 	while (!TAILQ_EMPTY(&disk_head)) {
140ec84ecbbSJim Harris 		disk = TAILQ_FIRST(&disk_head);
141ec84ecbbSJim Harris 		TAILQ_REMOVE(&disk_head, disk, global_tailq);
142ec84ecbbSJim Harris 		destroy_geom_disk(disk);
143ec84ecbbSJim Harris 		free(disk, M_NVD);
144bb0ec6b3SJim Harris 	}
145bb0ec6b3SJim Harris 
146bb0ec6b3SJim Harris 	nvme_unregister_consumer(consumer_handle);
147bb0ec6b3SJim Harris }
148bb0ec6b3SJim Harris 
149bb0ec6b3SJim Harris static void
150bb0ec6b3SJim Harris nvd_strategy(struct bio *bp)
151bb0ec6b3SJim Harris {
152bb0ec6b3SJim Harris 	struct nvd_disk *ndisk;
153bb0ec6b3SJim Harris 
154bb0ec6b3SJim Harris 	ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
155bb0ec6b3SJim Harris 
156bb0ec6b3SJim Harris 	mtx_lock(&ndisk->bioqlock);
157bb0ec6b3SJim Harris 	bioq_insert_tail(&ndisk->bioq, bp);
158bb0ec6b3SJim Harris 	mtx_unlock(&ndisk->bioqlock);
159bb0ec6b3SJim Harris 	taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask);
160bb0ec6b3SJim Harris }
161bb0ec6b3SJim Harris 
162bb0ec6b3SJim Harris static int
163bb0ec6b3SJim Harris nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag,
164bb0ec6b3SJim Harris     struct thread *td)
165bb0ec6b3SJim Harris {
166bb0ec6b3SJim Harris 	int ret = 0;
167bb0ec6b3SJim Harris 
168bb0ec6b3SJim Harris 	switch (cmd) {
169bb0ec6b3SJim Harris 	default:
170bb0ec6b3SJim Harris 		ret = EIO;
171bb0ec6b3SJim Harris 	}
172bb0ec6b3SJim Harris 
173bb0ec6b3SJim Harris 	return (ret);
174bb0ec6b3SJim Harris }
175bb0ec6b3SJim Harris 
176bb0ec6b3SJim Harris static void
177cf81529cSJim Harris nvd_done(void *arg, const struct nvme_completion *cpl)
178bb0ec6b3SJim Harris {
179bb0ec6b3SJim Harris 	struct bio *bp;
180bb0ec6b3SJim Harris 	struct nvd_disk *ndisk;
181bb0ec6b3SJim Harris 
182bb0ec6b3SJim Harris 	bp = (struct bio *)arg;
183bb0ec6b3SJim Harris 
184bb0ec6b3SJim Harris 	ndisk = bp->bio_disk->d_drv1;
185bb0ec6b3SJim Harris 
1860f71ecf7SJim Harris 	atomic_add_int(&ndisk->cur_depth, -1);
187bb0ec6b3SJim Harris 
188bb0ec6b3SJim Harris 	/*
189bb0ec6b3SJim Harris 	 * TODO: add more extensive translation of NVMe status codes
190bb0ec6b3SJim Harris 	 *  to different bio error codes (i.e. EIO, EINVAL, etc.)
191bb0ec6b3SJim Harris 	 */
192cf81529cSJim Harris 	if (nvme_completion_is_error(cpl)) {
193bb0ec6b3SJim Harris 		bp->bio_error = EIO;
194bb0ec6b3SJim Harris 		bp->bio_flags |= BIO_ERROR;
195bb0ec6b3SJim Harris 		bp->bio_resid = bp->bio_bcount;
196bb0ec6b3SJim Harris 	} else
197bb0ec6b3SJim Harris 		bp->bio_resid = 0;
198bb0ec6b3SJim Harris 
199bb0ec6b3SJim Harris 	biodone(bp);
200bb0ec6b3SJim Harris }
201bb0ec6b3SJim Harris 
202bb0ec6b3SJim Harris static void
203bb0ec6b3SJim Harris nvd_bioq_process(void *arg, int pending)
204bb0ec6b3SJim Harris {
205bb0ec6b3SJim Harris 	struct nvd_disk *ndisk = arg;
206bb0ec6b3SJim Harris 	struct bio *bp;
207bb0ec6b3SJim Harris 	int err;
208bb0ec6b3SJim Harris 
209bb0ec6b3SJim Harris 	for (;;) {
210bb0ec6b3SJim Harris 		mtx_lock(&ndisk->bioqlock);
211bb0ec6b3SJim Harris 		bp = bioq_takefirst(&ndisk->bioq);
212bb0ec6b3SJim Harris 		mtx_unlock(&ndisk->bioqlock);
213bb0ec6b3SJim Harris 		if (bp == NULL)
214bb0ec6b3SJim Harris 			break;
215bb0ec6b3SJim Harris 
216bb0ec6b3SJim Harris #ifdef BIO_ORDERED
217bb0ec6b3SJim Harris 		/*
218bb0ec6b3SJim Harris 		 * BIO_ORDERED flag dictates that all outstanding bios
219bb0ec6b3SJim Harris 		 *  must be completed before processing the bio with
220bb0ec6b3SJim Harris 		 *  BIO_ORDERED flag set.
221bb0ec6b3SJim Harris 		 */
222bb0ec6b3SJim Harris 		if (bp->bio_flags & BIO_ORDERED) {
223bb0ec6b3SJim Harris 			while (ndisk->cur_depth > 0) {
224bb0ec6b3SJim Harris 				pause("nvd flush", 1);
225bb0ec6b3SJim Harris 			}
226bb0ec6b3SJim Harris 		}
227bb0ec6b3SJim Harris #endif
228bb0ec6b3SJim Harris 
229bb0ec6b3SJim Harris 		bp->bio_driver1 = NULL;
2300f71ecf7SJim Harris 		atomic_add_int(&ndisk->cur_depth, 1);
231bb0ec6b3SJim Harris 
232bb0ec6b3SJim Harris 		err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done);
233bb0ec6b3SJim Harris 
234bb0ec6b3SJim Harris 		if (err) {
2350f71ecf7SJim Harris 			atomic_add_int(&ndisk->cur_depth, -1);
2369eb93f29SJim Harris 			bp->bio_error = err;
237bb0ec6b3SJim Harris 			bp->bio_flags |= BIO_ERROR;
238bb0ec6b3SJim Harris 			bp->bio_resid = bp->bio_bcount;
239bb0ec6b3SJim Harris 			biodone(bp);
240bb0ec6b3SJim Harris 		}
241bb0ec6b3SJim Harris 
242bb0ec6b3SJim Harris #ifdef BIO_ORDERED
243bb0ec6b3SJim Harris 		/*
244bb0ec6b3SJim Harris 		 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED
245bb0ec6b3SJim Harris 		 *  flag set must be completed before proceeding with
246bb0ec6b3SJim Harris 		 *  additional bios.
247bb0ec6b3SJim Harris 		 */
248bb0ec6b3SJim Harris 		if (bp->bio_flags & BIO_ORDERED) {
249bb0ec6b3SJim Harris 			while (ndisk->cur_depth > 0) {
250bb0ec6b3SJim Harris 				pause("nvd flush", 1);
251bb0ec6b3SJim Harris 			}
252bb0ec6b3SJim Harris 		}
253bb0ec6b3SJim Harris #endif
254bb0ec6b3SJim Harris 	}
255bb0ec6b3SJim Harris }
256bb0ec6b3SJim Harris 
257038a5ee4SJim Harris static void *
258ec84ecbbSJim Harris nvd_new_controller(struct nvme_controller *ctrlr)
259ec84ecbbSJim Harris {
260ec84ecbbSJim Harris 	struct nvd_controller	*nvd_ctrlr;
261ec84ecbbSJim Harris 
262ec84ecbbSJim Harris 	nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD,
263ec84ecbbSJim Harris 	    M_ZERO | M_NOWAIT);
264ec84ecbbSJim Harris 
265ec84ecbbSJim Harris 	TAILQ_INIT(&nvd_ctrlr->disk_head);
266ec84ecbbSJim Harris 	TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq);
267ec84ecbbSJim Harris 
268ec84ecbbSJim Harris 	return (nvd_ctrlr);
269ec84ecbbSJim Harris }
270ec84ecbbSJim Harris 
271ec84ecbbSJim Harris static void *
272ec84ecbbSJim Harris nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg)
273bb0ec6b3SJim Harris {
274bb0ec6b3SJim Harris 	struct nvd_disk		*ndisk;
275bb0ec6b3SJim Harris 	struct disk		*disk;
276ec84ecbbSJim Harris 	struct nvd_controller	*ctrlr = ctrlr_arg;
277bb0ec6b3SJim Harris 
278bb0ec6b3SJim Harris 	ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_NOWAIT);
279bb0ec6b3SJim Harris 
280bb0ec6b3SJim Harris 	disk = disk_alloc();
281bb0ec6b3SJim Harris 	disk->d_strategy = nvd_strategy;
282bb0ec6b3SJim Harris 	disk->d_ioctl = nvd_ioctl;
283bb0ec6b3SJim Harris 	disk->d_name = "nvd";
284bb0ec6b3SJim Harris 	disk->d_drv1 = ndisk;
285bb0ec6b3SJim Harris 
286bb0ec6b3SJim Harris 	disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns);
287bb0ec6b3SJim Harris 	disk->d_sectorsize = nvme_ns_get_sector_size(ns);
288bb0ec6b3SJim Harris 	disk->d_mediasize = (off_t)nvme_ns_get_size(ns);
289bb0ec6b3SJim Harris 
290ec84ecbbSJim Harris 	if (TAILQ_EMPTY(&disk_head))
291bb0ec6b3SJim Harris 		disk->d_unit = 0;
292bb0ec6b3SJim Harris 	else
293ec84ecbbSJim Harris 		disk->d_unit =
294ec84ecbbSJim Harris 		    TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1;
295bb0ec6b3SJim Harris 
296bb0ec6b3SJim Harris 	disk->d_flags = 0;
297bb0ec6b3SJim Harris 
298bb0ec6b3SJim Harris 	if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED)
299bb0ec6b3SJim Harris 		disk->d_flags |= DISKFLAG_CANDELETE;
300bb0ec6b3SJim Harris 
301bb0ec6b3SJim Harris 	if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED)
302bb0ec6b3SJim Harris 		disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
303bb0ec6b3SJim Harris 
304bb0ec6b3SJim Harris 	strlcpy(disk->d_ident, nvme_ns_get_serial_number(ns),
305bb0ec6b3SJim Harris 	    sizeof(disk->d_ident));
306bb0ec6b3SJim Harris 
307bb0ec6b3SJim Harris #if __FreeBSD_version >= 900034
308bb0ec6b3SJim Harris 	strlcpy(disk->d_descr, nvme_ns_get_model_number(ns),
309bb0ec6b3SJim Harris 	    sizeof(disk->d_descr));
310bb0ec6b3SJim Harris #endif
311bb0ec6b3SJim Harris 
312bb0ec6b3SJim Harris 	disk_create(disk, DISK_VERSION);
313bb0ec6b3SJim Harris 
314bb0ec6b3SJim Harris 	ndisk->ns = ns;
315bb0ec6b3SJim Harris 	ndisk->disk = disk;
316bb0ec6b3SJim Harris 	ndisk->cur_depth = 0;
317bb0ec6b3SJim Harris 
318bb0ec6b3SJim Harris 	mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF);
319bb0ec6b3SJim Harris 	bioq_init(&ndisk->bioq);
320bb0ec6b3SJim Harris 
321bb0ec6b3SJim Harris 	TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk);
322bb0ec6b3SJim Harris 	ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK,
323bb0ec6b3SJim Harris 	    taskqueue_thread_enqueue, &ndisk->tq);
324bb0ec6b3SJim Harris 	taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq");
325bb0ec6b3SJim Harris 
326ec84ecbbSJim Harris 	TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq);
327ec84ecbbSJim Harris 	TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq);
328038a5ee4SJim Harris 
329038a5ee4SJim Harris 	return (NULL);
330bb0ec6b3SJim Harris }
331bb0ec6b3SJim Harris 
332bb0ec6b3SJim Harris static void
333bb0ec6b3SJim Harris destroy_geom_disk(struct nvd_disk *ndisk)
334bb0ec6b3SJim Harris {
335bb0ec6b3SJim Harris 	struct bio *bp;
336bb0ec6b3SJim Harris 
337bb0ec6b3SJim Harris 	taskqueue_free(ndisk->tq);
338bb0ec6b3SJim Harris 	disk_destroy(ndisk->disk);
339bb0ec6b3SJim Harris 
340bb0ec6b3SJim Harris 	mtx_lock(&ndisk->bioqlock);
341bb0ec6b3SJim Harris 	for (;;) {
342bb0ec6b3SJim Harris 		bp = bioq_takefirst(&ndisk->bioq);
343bb0ec6b3SJim Harris 		if (bp == NULL)
344bb0ec6b3SJim Harris 			break;
345bb0ec6b3SJim Harris 		bp->bio_error = EIO;
346bb0ec6b3SJim Harris 		bp->bio_flags |= BIO_ERROR;
347bb0ec6b3SJim Harris 		bp->bio_resid = bp->bio_bcount;
348bb0ec6b3SJim Harris 
349bb0ec6b3SJim Harris 		biodone(bp);
350bb0ec6b3SJim Harris 	}
351bb0ec6b3SJim Harris 	mtx_unlock(&ndisk->bioqlock);
352bb0ec6b3SJim Harris 
353bb0ec6b3SJim Harris 	mtx_destroy(&ndisk->bioqlock);
354bb0ec6b3SJim Harris }
355*232e2edbSJim Harris 
356*232e2edbSJim Harris static void
357*232e2edbSJim Harris nvd_controller_fail(void *ctrlr_arg)
358*232e2edbSJim Harris {
359*232e2edbSJim Harris 	struct nvd_controller	*ctrlr = ctrlr_arg;
360*232e2edbSJim Harris 	struct nvd_disk		*disk;
361*232e2edbSJim Harris 
362*232e2edbSJim Harris 	while (!TAILQ_EMPTY(&ctrlr->disk_head)) {
363*232e2edbSJim Harris 		disk = TAILQ_FIRST(&ctrlr->disk_head);
364*232e2edbSJim Harris 		TAILQ_REMOVE(&disk_head, disk, global_tailq);
365*232e2edbSJim Harris 		TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq);
366*232e2edbSJim Harris 		destroy_geom_disk(disk);
367*232e2edbSJim Harris 		free(disk, M_NVD);
368*232e2edbSJim Harris 	}
369*232e2edbSJim Harris 
370*232e2edbSJim Harris 	TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
371*232e2edbSJim Harris 	free(ctrlr, M_NVD);
372*232e2edbSJim Harris }
373*232e2edbSJim Harris 
374