1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (C) 2012-2016 Intel Corporation
5 * All rights reserved.
6 * Copyright (C) 2018-2020 Alexander Motin <mav@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/param.h>
31 #include <sys/bio.h>
32 #include <sys/devicestat.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/module.h>
36 #include <sys/queue.h>
37 #include <sys/sysctl.h>
38 #include <sys/systm.h>
39 #include <sys/taskqueue.h>
40 #include <machine/atomic.h>
41
42 #include <geom/geom.h>
43 #include <geom/geom_disk.h>
44
45 #include <dev/nvme/nvme.h>
46 #include <dev/nvme/nvme_private.h>
47
48 #include <dev/pci/pcivar.h>
49
50 #define NVD_STR "nvd"
51
52 struct nvd_disk;
53 struct nvd_controller;
54
55 static disk_ioctl_t nvd_ioctl;
56 static disk_strategy_t nvd_strategy;
57 static dumper_t nvd_dump;
58 static disk_getattr_t nvd_getattr;
59
60 static void nvd_done(void *arg, const struct nvme_completion *cpl);
61 static void nvd_gone(struct nvd_disk *ndisk);
62
63 static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr);
64 static void *nvd_ns_changed(struct nvme_namespace *ns, void *ctrlr);
65
66 static void *nvd_new_controller(struct nvme_controller *ctrlr);
67 static void nvd_controller_fail(void *ctrlr);
68
69 static int nvd_load(void);
70 static void nvd_unload(void);
71
72 MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations");
73
74 struct nvme_consumer *consumer_handle;
75
76 struct nvd_disk {
77 struct nvd_controller *ctrlr;
78
79 struct bio_queue_head bioq;
80 struct task bioqtask;
81 struct mtx bioqlock;
82
83 struct disk *disk;
84 struct taskqueue *tq;
85 struct nvme_namespace *ns;
86
87 uint32_t cur_depth;
88 #define NVD_ODEPTH (1 << 30)
89 uint32_t ordered_in_flight;
90 u_int unit;
91
92 TAILQ_ENTRY(nvd_disk) global_tailq;
93 TAILQ_ENTRY(nvd_disk) ctrlr_tailq;
94 };
95
96 struct nvd_controller {
97 struct nvme_controller *ctrlr;
98 TAILQ_ENTRY(nvd_controller) tailq;
99 TAILQ_HEAD(, nvd_disk) disk_head;
100 };
101
102 static struct mtx nvd_lock;
103 static TAILQ_HEAD(, nvd_controller) ctrlr_head;
104 static TAILQ_HEAD(disk_list, nvd_disk) disk_head;
105
106 static SYSCTL_NODE(_hw, OID_AUTO, nvd, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
107 "nvd driver parameters");
108 /*
109 * The NVMe specification does not define a maximum or optimal delete size, so
110 * technically max delete size is min(full size of the namespace, 2^32 - 1
111 * LBAs). A single delete for a multi-TB NVMe namespace though may take much
112 * longer to complete than the nvme(4) I/O timeout period. So choose a sensible
113 * default here that is still suitably large to minimize the number of overall
114 * delete operations.
115 */
116 static uint64_t nvd_delete_max = (1024 * 1024 * 1024); /* 1GB */
117 SYSCTL_UQUAD(_hw_nvd, OID_AUTO, delete_max, CTLFLAG_RDTUN, &nvd_delete_max, 0,
118 "nvd maximum BIO_DELETE size in bytes");
119
nvd_modevent(module_t mod,int type,void * arg)120 static int nvd_modevent(module_t mod, int type, void *arg)
121 {
122 int error = 0;
123
124 switch (type) {
125 case MOD_LOAD:
126 error = nvd_load();
127 break;
128 case MOD_UNLOAD:
129 nvd_unload();
130 break;
131 default:
132 break;
133 }
134
135 return (error);
136 }
137
138 moduledata_t nvd_mod = {
139 NVD_STR,
140 (modeventhand_t)nvd_modevent,
141 0
142 };
143
144 DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
145 MODULE_VERSION(nvd, 1);
146 MODULE_DEPEND(nvd, nvme, 1, 1, 1);
147
148 static int
nvd_load(void)149 nvd_load(void)
150 {
151 if (!nvme_use_nvd)
152 return 0;
153
154 mtx_init(&nvd_lock, "nvd_lock", NULL, MTX_DEF);
155 TAILQ_INIT(&ctrlr_head);
156 TAILQ_INIT(&disk_head);
157
158 consumer_handle = nvme_register_consumer(nvd_ns_changed,
159 nvd_new_controller, NULL, nvd_controller_fail);
160
161 return (consumer_handle != NULL ? 0 : -1);
162 }
163
164 static void
nvd_unload(void)165 nvd_unload(void)
166 {
167 struct nvd_controller *ctrlr;
168 struct nvd_disk *ndisk;
169
170 if (!nvme_use_nvd)
171 return;
172
173 mtx_lock(&nvd_lock);
174 while ((ctrlr = TAILQ_FIRST(&ctrlr_head)) != NULL) {
175 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
176 TAILQ_FOREACH(ndisk, &ctrlr->disk_head, ctrlr_tailq)
177 nvd_gone(ndisk);
178 while (!TAILQ_EMPTY(&ctrlr->disk_head))
179 msleep(&ctrlr->disk_head, &nvd_lock, 0, "nvd_unload",0);
180 free(ctrlr, M_NVD);
181 }
182 mtx_unlock(&nvd_lock);
183
184 nvme_unregister_consumer(consumer_handle);
185
186 mtx_destroy(&nvd_lock);
187 }
188
189 static void
nvd_bio_submit(struct nvd_disk * ndisk,struct bio * bp)190 nvd_bio_submit(struct nvd_disk *ndisk, struct bio *bp)
191 {
192 int err;
193
194 bp->bio_driver1 = NULL;
195 if (__predict_false(bp->bio_flags & BIO_ORDERED))
196 atomic_add_int(&ndisk->cur_depth, NVD_ODEPTH);
197 else
198 atomic_add_int(&ndisk->cur_depth, 1);
199 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done);
200 if (err) {
201 if (__predict_false(bp->bio_flags & BIO_ORDERED)) {
202 atomic_add_int(&ndisk->cur_depth, -NVD_ODEPTH);
203 atomic_add_int(&ndisk->ordered_in_flight, -1);
204 wakeup(&ndisk->cur_depth);
205 } else {
206 if (atomic_fetchadd_int(&ndisk->cur_depth, -1) == 1 &&
207 __predict_false(ndisk->ordered_in_flight != 0))
208 wakeup(&ndisk->cur_depth);
209 }
210 bp->bio_error = err;
211 bp->bio_flags |= BIO_ERROR;
212 bp->bio_resid = bp->bio_bcount;
213 biodone(bp);
214 }
215 }
216
217 static void
nvd_strategy(struct bio * bp)218 nvd_strategy(struct bio *bp)
219 {
220 struct nvd_disk *ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
221
222 /*
223 * bio with BIO_ORDERED flag must be executed after all previous
224 * bios in the queue, and before any successive bios.
225 */
226 if (__predict_false(bp->bio_flags & BIO_ORDERED)) {
227 if (atomic_fetchadd_int(&ndisk->ordered_in_flight, 1) == 0 &&
228 ndisk->cur_depth == 0 && bioq_first(&ndisk->bioq) == NULL) {
229 nvd_bio_submit(ndisk, bp);
230 return;
231 }
232 } else if (__predict_true(ndisk->ordered_in_flight == 0)) {
233 nvd_bio_submit(ndisk, bp);
234 return;
235 }
236
237 /*
238 * There are ordered bios in flight, so we need to submit
239 * bios through the task queue to enforce ordering.
240 */
241 mtx_lock(&ndisk->bioqlock);
242 bioq_insert_tail(&ndisk->bioq, bp);
243 mtx_unlock(&ndisk->bioqlock);
244 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask);
245 }
246
247 static void
nvd_gone(struct nvd_disk * ndisk)248 nvd_gone(struct nvd_disk *ndisk)
249 {
250 struct bio *bp;
251
252 printf(NVD_STR"%u: detached\n", ndisk->unit);
253 mtx_lock(&ndisk->bioqlock);
254 disk_gone(ndisk->disk);
255 while ((bp = bioq_takefirst(&ndisk->bioq)) != NULL) {
256 if (__predict_false(bp->bio_flags & BIO_ORDERED))
257 atomic_add_int(&ndisk->ordered_in_flight, -1);
258 bp->bio_error = ENXIO;
259 bp->bio_flags |= BIO_ERROR;
260 bp->bio_resid = bp->bio_bcount;
261 biodone(bp);
262 }
263 mtx_unlock(&ndisk->bioqlock);
264 }
265
266 static void
nvd_gonecb(struct disk * dp)267 nvd_gonecb(struct disk *dp)
268 {
269 struct nvd_disk *ndisk = (struct nvd_disk *)dp->d_drv1;
270
271 disk_destroy(ndisk->disk);
272 mtx_lock(&nvd_lock);
273 TAILQ_REMOVE(&disk_head, ndisk, global_tailq);
274 TAILQ_REMOVE(&ndisk->ctrlr->disk_head, ndisk, ctrlr_tailq);
275 if (TAILQ_EMPTY(&ndisk->ctrlr->disk_head))
276 wakeup(&ndisk->ctrlr->disk_head);
277 mtx_unlock(&nvd_lock);
278 taskqueue_free(ndisk->tq);
279 mtx_destroy(&ndisk->bioqlock);
280 free(ndisk, M_NVD);
281 }
282
283 static int
nvd_ioctl(struct disk * dp,u_long cmd,void * data,int fflag,struct thread * td)284 nvd_ioctl(struct disk *dp, u_long cmd, void *data, int fflag,
285 struct thread *td)
286 {
287 struct nvd_disk *ndisk = dp->d_drv1;
288
289 return (nvme_ns_ioctl_process(ndisk->ns, cmd, data, fflag, td));
290 }
291
292 static int
nvd_dump(void * arg,void * virt,off_t offset,size_t len)293 nvd_dump(void *arg, void *virt, off_t offset, size_t len)
294 {
295 struct disk *dp = arg;
296 struct nvd_disk *ndisk = dp->d_drv1;
297
298 return (nvme_ns_dump(ndisk->ns, virt, offset, len));
299 }
300
301 static int
nvd_getattr(struct bio * bp)302 nvd_getattr(struct bio *bp)
303 {
304 struct nvd_disk *ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1;
305 const struct nvme_namespace_data *nsdata;
306 u_int i;
307
308 if (!strcmp("GEOM::lunid", bp->bio_attribute)) {
309 nsdata = nvme_ns_get_data(ndisk->ns);
310
311 /* Try to return NGUID as lunid. */
312 for (i = 0; i < sizeof(nsdata->nguid); i++) {
313 if (nsdata->nguid[i] != 0)
314 break;
315 }
316 if (i < sizeof(nsdata->nguid)) {
317 if (bp->bio_length < sizeof(nsdata->nguid) * 2 + 1)
318 return (EFAULT);
319 for (i = 0; i < sizeof(nsdata->nguid); i++) {
320 sprintf(&bp->bio_data[i * 2], "%02x",
321 nsdata->nguid[i]);
322 }
323 bp->bio_completed = bp->bio_length;
324 return (0);
325 }
326
327 /* Try to return EUI64 as lunid. */
328 for (i = 0; i < sizeof(nsdata->eui64); i++) {
329 if (nsdata->eui64[i] != 0)
330 break;
331 }
332 if (i < sizeof(nsdata->eui64)) {
333 if (bp->bio_length < sizeof(nsdata->eui64) * 2 + 1)
334 return (EFAULT);
335 for (i = 0; i < sizeof(nsdata->eui64); i++) {
336 sprintf(&bp->bio_data[i * 2], "%02x",
337 nsdata->eui64[i]);
338 }
339 bp->bio_completed = bp->bio_length;
340 return (0);
341 }
342 }
343 return (-1);
344 }
345
346 static void
nvd_done(void * arg,const struct nvme_completion * cpl)347 nvd_done(void *arg, const struct nvme_completion *cpl)
348 {
349 struct bio *bp = (struct bio *)arg;
350 struct nvd_disk *ndisk = bp->bio_disk->d_drv1;
351
352 if (__predict_false(bp->bio_flags & BIO_ORDERED)) {
353 atomic_add_int(&ndisk->cur_depth, -NVD_ODEPTH);
354 atomic_add_int(&ndisk->ordered_in_flight, -1);
355 wakeup(&ndisk->cur_depth);
356 } else {
357 if (atomic_fetchadd_int(&ndisk->cur_depth, -1) == 1 &&
358 __predict_false(ndisk->ordered_in_flight != 0))
359 wakeup(&ndisk->cur_depth);
360 }
361
362 biodone(bp);
363 }
364
365 static void
nvd_bioq_process(void * arg,int pending)366 nvd_bioq_process(void *arg, int pending)
367 {
368 struct nvd_disk *ndisk = arg;
369 struct bio *bp;
370
371 for (;;) {
372 mtx_lock(&ndisk->bioqlock);
373 bp = bioq_takefirst(&ndisk->bioq);
374 mtx_unlock(&ndisk->bioqlock);
375 if (bp == NULL)
376 break;
377
378 if (__predict_false(bp->bio_flags & BIO_ORDERED)) {
379 /*
380 * bio with BIO_ORDERED flag set must be executed
381 * after all previous bios.
382 */
383 while (ndisk->cur_depth > 0)
384 tsleep(&ndisk->cur_depth, 0, "nvdorb", 1);
385 } else {
386 /*
387 * bio with BIO_ORDERED flag set must be completed
388 * before proceeding with additional bios.
389 */
390 while (ndisk->cur_depth >= NVD_ODEPTH)
391 tsleep(&ndisk->cur_depth, 0, "nvdora", 1);
392 }
393
394 nvd_bio_submit(ndisk, bp);
395 }
396 }
397
398 static void *
nvd_new_controller(struct nvme_controller * ctrlr)399 nvd_new_controller(struct nvme_controller *ctrlr)
400 {
401 struct nvd_controller *nvd_ctrlr;
402
403 nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD,
404 M_ZERO | M_WAITOK);
405
406 nvd_ctrlr->ctrlr = ctrlr;
407 TAILQ_INIT(&nvd_ctrlr->disk_head);
408 mtx_lock(&nvd_lock);
409 TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq);
410 mtx_unlock(&nvd_lock);
411
412 return (nvd_ctrlr);
413 }
414
415 static void *
nvd_new_disk(struct nvme_namespace * ns,void * ctrlr_arg)416 nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg)
417 {
418 uint8_t descr[NVME_MODEL_NUMBER_LENGTH+1];
419 struct nvd_disk *ndisk, *tnd;
420 struct disk *disk;
421 struct nvd_controller *ctrlr = ctrlr_arg;
422 device_t dev = ctrlr->ctrlr->dev;
423 int unit;
424
425 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK);
426 ndisk->ctrlr = ctrlr;
427 ndisk->ns = ns;
428 ndisk->cur_depth = 0;
429 ndisk->ordered_in_flight = 0;
430 mtx_init(&ndisk->bioqlock, "nvd bioq lock", NULL, MTX_DEF);
431 bioq_init(&ndisk->bioq);
432 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk);
433
434 mtx_lock(&nvd_lock);
435 unit = 0;
436 TAILQ_FOREACH(tnd, &disk_head, global_tailq) {
437 if (tnd->unit > unit)
438 break;
439 unit = tnd->unit + 1;
440 }
441 ndisk->unit = unit;
442 if (tnd != NULL)
443 TAILQ_INSERT_BEFORE(tnd, ndisk, global_tailq);
444 else
445 TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq);
446 TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq);
447 mtx_unlock(&nvd_lock);
448
449 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK,
450 taskqueue_thread_enqueue, &ndisk->tq);
451 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq");
452
453 disk = ndisk->disk = disk_alloc();
454 disk->d_strategy = nvd_strategy;
455 disk->d_ioctl = nvd_ioctl;
456 disk->d_dump = nvd_dump;
457 disk->d_getattr = nvd_getattr;
458 disk->d_gone = nvd_gonecb;
459 disk->d_name = NVD_STR;
460 disk->d_unit = ndisk->unit;
461 disk->d_drv1 = ndisk;
462
463 disk->d_sectorsize = nvme_ns_get_sector_size(ns);
464 disk->d_mediasize = (off_t)nvme_ns_get_size(ns);
465 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns);
466 disk->d_delmaxsize = (off_t)nvme_ns_get_size(ns);
467 if (disk->d_delmaxsize > nvd_delete_max)
468 disk->d_delmaxsize = nvd_delete_max;
469 disk->d_stripesize = nvme_ns_get_stripesize(ns);
470 disk->d_flags = DISKFLAG_UNMAPPED_BIO | DISKFLAG_DIRECT_COMPLETION;
471 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED)
472 disk->d_flags |= DISKFLAG_CANDELETE;
473 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED)
474 disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
475 disk->d_devstat = devstat_new_entry(disk->d_name, disk->d_unit,
476 disk->d_sectorsize, DEVSTAT_ALL_SUPPORTED,
477 DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_NVME,
478 DEVSTAT_PRIORITY_DISK);
479
480 /*
481 * d_ident and d_descr are both far bigger than the length of either
482 * the serial or model number strings.
483 */
484 nvme_strvis(disk->d_ident, nvme_ns_get_serial_number(ns),
485 sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH);
486 nvme_strvis(descr, nvme_ns_get_model_number(ns), sizeof(descr),
487 NVME_MODEL_NUMBER_LENGTH);
488 strlcpy(disk->d_descr, descr, sizeof(descr));
489
490 /*
491 * For devices that are reported as children of the AHCI controller,
492 * which has no access to the config space for this controller, report
493 * the AHCI controller's data.
494 */
495 if (ctrlr->ctrlr->quirks & QUIRK_AHCI)
496 dev = device_get_parent(dev);
497 disk->d_hba_vendor = pci_get_vendor(dev);
498 disk->d_hba_device = pci_get_device(dev);
499 disk->d_hba_subvendor = pci_get_subvendor(dev);
500 disk->d_hba_subdevice = pci_get_subdevice(dev);
501 disk->d_rotation_rate = DISK_RR_NON_ROTATING;
502 strlcpy(disk->d_attachment, device_get_nameunit(dev),
503 sizeof(disk->d_attachment));
504
505 disk_create(disk, DISK_VERSION);
506
507 printf(NVD_STR"%u: <%s> NVMe namespace\n", disk->d_unit, descr);
508 printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit,
509 (uintmax_t)disk->d_mediasize / (1024*1024),
510 (uintmax_t)disk->d_mediasize / disk->d_sectorsize,
511 disk->d_sectorsize);
512
513 return (ndisk);
514 }
515
516 static void
nvd_resize(struct nvd_disk * ndisk)517 nvd_resize(struct nvd_disk *ndisk)
518 {
519 struct disk *disk = ndisk->disk;
520 struct nvme_namespace *ns = ndisk->ns;
521
522 disk->d_sectorsize = nvme_ns_get_sector_size(ns);
523 disk->d_mediasize = (off_t)nvme_ns_get_size(ns);
524 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns);
525 disk->d_delmaxsize = (off_t)nvme_ns_get_size(ns);
526 if (disk->d_delmaxsize > nvd_delete_max)
527 disk->d_delmaxsize = nvd_delete_max;
528
529 disk_resize(disk, M_NOWAIT);
530
531 printf(NVD_STR"%u: NVMe namespace resized\n", ndisk->unit);
532 printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit,
533 (uintmax_t)disk->d_mediasize / (1024*1024),
534 (uintmax_t)disk->d_mediasize / disk->d_sectorsize,
535 disk->d_sectorsize);
536 }
537
538 static void *
nvd_ns_changed(struct nvme_namespace * ns,void * ctrlr_arg)539 nvd_ns_changed(struct nvme_namespace *ns, void *ctrlr_arg)
540 {
541 struct nvd_disk *ndisk;
542 struct nvd_controller *ctrlr = ctrlr_arg;
543
544 if ((ns->flags & NVME_NS_CHANGED) == 0)
545 return (nvd_new_disk(ns, ctrlr_arg));
546
547 mtx_lock(&nvd_lock);
548 TAILQ_FOREACH(ndisk, &ctrlr->disk_head, ctrlr_tailq) {
549 if (ndisk->ns->id != ns->id)
550 continue;
551 nvd_resize(ndisk);
552 break;
553 }
554 mtx_unlock(&nvd_lock);
555 return (ctrlr_arg);
556 }
557
558 static void
nvd_controller_fail(void * ctrlr_arg)559 nvd_controller_fail(void *ctrlr_arg)
560 {
561 struct nvd_controller *ctrlr = ctrlr_arg;
562 struct nvd_disk *ndisk;
563
564 mtx_lock(&nvd_lock);
565 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
566 TAILQ_FOREACH(ndisk, &ctrlr->disk_head, ctrlr_tailq)
567 nvd_gone(ndisk);
568 while (!TAILQ_EMPTY(&ctrlr->disk_head))
569 msleep(&ctrlr->disk_head, &nvd_lock, 0, "nvd_fail", 0);
570 mtx_unlock(&nvd_lock);
571 free(ctrlr, M_NVD);
572 }
573