1bb0ec6b3SJim Harris /*- 2bb0ec6b3SJim Harris * Copyright (C) 2012 Intel Corporation 3bb0ec6b3SJim Harris * All rights reserved. 4bb0ec6b3SJim Harris * 5bb0ec6b3SJim Harris * Redistribution and use in source and binary forms, with or without 6bb0ec6b3SJim Harris * modification, are permitted provided that the following conditions 7bb0ec6b3SJim Harris * are met: 8bb0ec6b3SJim Harris * 1. Redistributions of source code must retain the above copyright 9bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer. 10bb0ec6b3SJim Harris * 2. Redistributions in binary form must reproduce the above copyright 11bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer in the 12bb0ec6b3SJim Harris * documentation and/or other materials provided with the distribution. 13bb0ec6b3SJim Harris * 14bb0ec6b3SJim Harris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15bb0ec6b3SJim Harris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16bb0ec6b3SJim Harris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17bb0ec6b3SJim Harris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18bb0ec6b3SJim Harris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19bb0ec6b3SJim Harris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20bb0ec6b3SJim Harris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21bb0ec6b3SJim Harris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22bb0ec6b3SJim Harris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23bb0ec6b3SJim Harris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24bb0ec6b3SJim Harris * SUCH DAMAGE. 25bb0ec6b3SJim Harris */ 26bb0ec6b3SJim Harris 27bb0ec6b3SJim Harris #include <sys/cdefs.h> 28bb0ec6b3SJim Harris __FBSDID("$FreeBSD$"); 29bb0ec6b3SJim Harris 30bb0ec6b3SJim Harris #include <sys/param.h> 31bb0ec6b3SJim Harris #include <sys/bio.h> 32bb0ec6b3SJim Harris #include <sys/kernel.h> 33bb0ec6b3SJim Harris #include <sys/malloc.h> 34bb0ec6b3SJim Harris #include <sys/module.h> 35bb0ec6b3SJim Harris #include <sys/systm.h> 36bb0ec6b3SJim Harris #include <sys/taskqueue.h> 37bb0ec6b3SJim Harris 38bb0ec6b3SJim Harris #include <geom/geom.h> 39bb0ec6b3SJim Harris #include <geom/geom_disk.h> 40bb0ec6b3SJim Harris 41bb0ec6b3SJim Harris #include <dev/nvme/nvme.h> 42bb0ec6b3SJim Harris 43bb0ec6b3SJim Harris struct nvd_disk; 44bb0ec6b3SJim Harris 45bb0ec6b3SJim Harris static disk_ioctl_t nvd_ioctl; 46bb0ec6b3SJim Harris static disk_strategy_t nvd_strategy; 47bb0ec6b3SJim Harris 48ec84ecbbSJim Harris static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr); 49bb0ec6b3SJim Harris static void destroy_geom_disk(struct nvd_disk *ndisk); 50bb0ec6b3SJim Harris 51ec84ecbbSJim Harris static void *nvd_new_controller(struct nvme_controller *ctrlr); 52*232e2edbSJim Harris static void nvd_controller_fail(void *ctrlr); 53ec84ecbbSJim Harris 54bb0ec6b3SJim Harris static int nvd_load(void); 55bb0ec6b3SJim Harris static void nvd_unload(void); 56bb0ec6b3SJim Harris 57bb0ec6b3SJim Harris MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 58bb0ec6b3SJim Harris 59bb0ec6b3SJim Harris struct nvme_consumer *consumer_handle; 60bb0ec6b3SJim Harris 61bb0ec6b3SJim Harris struct nvd_disk { 62bb0ec6b3SJim Harris 63bb0ec6b3SJim Harris struct bio_queue_head bioq; 64bb0ec6b3SJim Harris struct task bioqtask; 65bb0ec6b3SJim Harris struct mtx bioqlock; 66bb0ec6b3SJim Harris 67bb0ec6b3SJim Harris struct disk *disk; 68bb0ec6b3SJim Harris struct taskqueue *tq; 69bb0ec6b3SJim Harris struct nvme_namespace *ns; 70bb0ec6b3SJim Harris 71bb0ec6b3SJim Harris uint32_t cur_depth; 72bb0ec6b3SJim Harris 73ec84ecbbSJim Harris TAILQ_ENTRY(nvd_disk) global_tailq; 74ec84ecbbSJim Harris TAILQ_ENTRY(nvd_disk) ctrlr_tailq; 75bb0ec6b3SJim Harris }; 76bb0ec6b3SJim Harris 77ec84ecbbSJim Harris struct nvd_controller { 78ec84ecbbSJim Harris 79ec84ecbbSJim Harris TAILQ_ENTRY(nvd_controller) tailq; 80ec84ecbbSJim Harris TAILQ_HEAD(, nvd_disk) disk_head; 81ec84ecbbSJim Harris }; 82ec84ecbbSJim Harris 83ec84ecbbSJim Harris static TAILQ_HEAD(, nvd_controller) ctrlr_head; 84ec84ecbbSJim Harris static TAILQ_HEAD(disk_list, nvd_disk) disk_head; 85bb0ec6b3SJim Harris 86bb0ec6b3SJim Harris static int nvd_modevent(module_t mod, int type, void *arg) 87bb0ec6b3SJim Harris { 88bb0ec6b3SJim Harris int error = 0; 89bb0ec6b3SJim Harris 90bb0ec6b3SJim Harris switch (type) { 91bb0ec6b3SJim Harris case MOD_LOAD: 92bb0ec6b3SJim Harris error = nvd_load(); 93bb0ec6b3SJim Harris break; 94bb0ec6b3SJim Harris case MOD_UNLOAD: 95bb0ec6b3SJim Harris nvd_unload(); 96bb0ec6b3SJim Harris break; 97bb0ec6b3SJim Harris default: 98bb0ec6b3SJim Harris break; 99bb0ec6b3SJim Harris } 100bb0ec6b3SJim Harris 101bb0ec6b3SJim Harris return (error); 102bb0ec6b3SJim Harris } 103bb0ec6b3SJim Harris 104bb0ec6b3SJim Harris moduledata_t nvd_mod = { 105bb0ec6b3SJim Harris "nvd", 106bb0ec6b3SJim Harris (modeventhand_t)nvd_modevent, 1079823d527SKevin Lo 0 108bb0ec6b3SJim Harris }; 109bb0ec6b3SJim Harris 110bb0ec6b3SJim Harris DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 111bb0ec6b3SJim Harris MODULE_VERSION(nvd, 1); 112bb0ec6b3SJim Harris MODULE_DEPEND(nvd, nvme, 1, 1, 1); 113bb0ec6b3SJim Harris 114bb0ec6b3SJim Harris static int 115bb0ec6b3SJim Harris nvd_load() 116bb0ec6b3SJim Harris { 117bb0ec6b3SJim Harris 118ec84ecbbSJim Harris TAILQ_INIT(&ctrlr_head); 119ec84ecbbSJim Harris TAILQ_INIT(&disk_head); 120ec84ecbbSJim Harris 121ec84ecbbSJim Harris consumer_handle = nvme_register_consumer(nvd_new_disk, 122*232e2edbSJim Harris nvd_new_controller, NULL, nvd_controller_fail); 123bb0ec6b3SJim Harris 124bb0ec6b3SJim Harris return (consumer_handle != NULL ? 0 : -1); 125bb0ec6b3SJim Harris } 126bb0ec6b3SJim Harris 127bb0ec6b3SJim Harris static void 128bb0ec6b3SJim Harris nvd_unload() 129bb0ec6b3SJim Harris { 130ec84ecbbSJim Harris struct nvd_controller *ctrlr; 131ec84ecbbSJim Harris struct nvd_disk *disk; 132bb0ec6b3SJim Harris 133ec84ecbbSJim Harris while (!TAILQ_EMPTY(&ctrlr_head)) { 134ec84ecbbSJim Harris ctrlr = TAILQ_FIRST(&ctrlr_head); 135ec84ecbbSJim Harris TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 136ec84ecbbSJim Harris free(ctrlr, M_NVD); 137ec84ecbbSJim Harris } 138ec84ecbbSJim Harris 139ec84ecbbSJim Harris while (!TAILQ_EMPTY(&disk_head)) { 140ec84ecbbSJim Harris disk = TAILQ_FIRST(&disk_head); 141ec84ecbbSJim Harris TAILQ_REMOVE(&disk_head, disk, global_tailq); 142ec84ecbbSJim Harris destroy_geom_disk(disk); 143ec84ecbbSJim Harris free(disk, M_NVD); 144bb0ec6b3SJim Harris } 145bb0ec6b3SJim Harris 146bb0ec6b3SJim Harris nvme_unregister_consumer(consumer_handle); 147bb0ec6b3SJim Harris } 148bb0ec6b3SJim Harris 149bb0ec6b3SJim Harris static void 150bb0ec6b3SJim Harris nvd_strategy(struct bio *bp) 151bb0ec6b3SJim Harris { 152bb0ec6b3SJim Harris struct nvd_disk *ndisk; 153bb0ec6b3SJim Harris 154bb0ec6b3SJim Harris ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 155bb0ec6b3SJim Harris 156bb0ec6b3SJim Harris mtx_lock(&ndisk->bioqlock); 157bb0ec6b3SJim Harris bioq_insert_tail(&ndisk->bioq, bp); 158bb0ec6b3SJim Harris mtx_unlock(&ndisk->bioqlock); 159bb0ec6b3SJim Harris taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 160bb0ec6b3SJim Harris } 161bb0ec6b3SJim Harris 162bb0ec6b3SJim Harris static int 163bb0ec6b3SJim Harris nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 164bb0ec6b3SJim Harris struct thread *td) 165bb0ec6b3SJim Harris { 166bb0ec6b3SJim Harris int ret = 0; 167bb0ec6b3SJim Harris 168bb0ec6b3SJim Harris switch (cmd) { 169bb0ec6b3SJim Harris default: 170bb0ec6b3SJim Harris ret = EIO; 171bb0ec6b3SJim Harris } 172bb0ec6b3SJim Harris 173bb0ec6b3SJim Harris return (ret); 174bb0ec6b3SJim Harris } 175bb0ec6b3SJim Harris 176bb0ec6b3SJim Harris static void 177cf81529cSJim Harris nvd_done(void *arg, const struct nvme_completion *cpl) 178bb0ec6b3SJim Harris { 179bb0ec6b3SJim Harris struct bio *bp; 180bb0ec6b3SJim Harris struct nvd_disk *ndisk; 181bb0ec6b3SJim Harris 182bb0ec6b3SJim Harris bp = (struct bio *)arg; 183bb0ec6b3SJim Harris 184bb0ec6b3SJim Harris ndisk = bp->bio_disk->d_drv1; 185bb0ec6b3SJim Harris 1860f71ecf7SJim Harris atomic_add_int(&ndisk->cur_depth, -1); 187bb0ec6b3SJim Harris 188bb0ec6b3SJim Harris /* 189bb0ec6b3SJim Harris * TODO: add more extensive translation of NVMe status codes 190bb0ec6b3SJim Harris * to different bio error codes (i.e. EIO, EINVAL, etc.) 191bb0ec6b3SJim Harris */ 192cf81529cSJim Harris if (nvme_completion_is_error(cpl)) { 193bb0ec6b3SJim Harris bp->bio_error = EIO; 194bb0ec6b3SJim Harris bp->bio_flags |= BIO_ERROR; 195bb0ec6b3SJim Harris bp->bio_resid = bp->bio_bcount; 196bb0ec6b3SJim Harris } else 197bb0ec6b3SJim Harris bp->bio_resid = 0; 198bb0ec6b3SJim Harris 199bb0ec6b3SJim Harris biodone(bp); 200bb0ec6b3SJim Harris } 201bb0ec6b3SJim Harris 202bb0ec6b3SJim Harris static void 203bb0ec6b3SJim Harris nvd_bioq_process(void *arg, int pending) 204bb0ec6b3SJim Harris { 205bb0ec6b3SJim Harris struct nvd_disk *ndisk = arg; 206bb0ec6b3SJim Harris struct bio *bp; 207bb0ec6b3SJim Harris int err; 208bb0ec6b3SJim Harris 209bb0ec6b3SJim Harris for (;;) { 210bb0ec6b3SJim Harris mtx_lock(&ndisk->bioqlock); 211bb0ec6b3SJim Harris bp = bioq_takefirst(&ndisk->bioq); 212bb0ec6b3SJim Harris mtx_unlock(&ndisk->bioqlock); 213bb0ec6b3SJim Harris if (bp == NULL) 214bb0ec6b3SJim Harris break; 215bb0ec6b3SJim Harris 216bb0ec6b3SJim Harris #ifdef BIO_ORDERED 217bb0ec6b3SJim Harris /* 218bb0ec6b3SJim Harris * BIO_ORDERED flag dictates that all outstanding bios 219bb0ec6b3SJim Harris * must be completed before processing the bio with 220bb0ec6b3SJim Harris * BIO_ORDERED flag set. 221bb0ec6b3SJim Harris */ 222bb0ec6b3SJim Harris if (bp->bio_flags & BIO_ORDERED) { 223bb0ec6b3SJim Harris while (ndisk->cur_depth > 0) { 224bb0ec6b3SJim Harris pause("nvd flush", 1); 225bb0ec6b3SJim Harris } 226bb0ec6b3SJim Harris } 227bb0ec6b3SJim Harris #endif 228bb0ec6b3SJim Harris 229bb0ec6b3SJim Harris bp->bio_driver1 = NULL; 2300f71ecf7SJim Harris atomic_add_int(&ndisk->cur_depth, 1); 231bb0ec6b3SJim Harris 232bb0ec6b3SJim Harris err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 233bb0ec6b3SJim Harris 234bb0ec6b3SJim Harris if (err) { 2350f71ecf7SJim Harris atomic_add_int(&ndisk->cur_depth, -1); 2369eb93f29SJim Harris bp->bio_error = err; 237bb0ec6b3SJim Harris bp->bio_flags |= BIO_ERROR; 238bb0ec6b3SJim Harris bp->bio_resid = bp->bio_bcount; 239bb0ec6b3SJim Harris biodone(bp); 240bb0ec6b3SJim Harris } 241bb0ec6b3SJim Harris 242bb0ec6b3SJim Harris #ifdef BIO_ORDERED 243bb0ec6b3SJim Harris /* 244bb0ec6b3SJim Harris * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 245bb0ec6b3SJim Harris * flag set must be completed before proceeding with 246bb0ec6b3SJim Harris * additional bios. 247bb0ec6b3SJim Harris */ 248bb0ec6b3SJim Harris if (bp->bio_flags & BIO_ORDERED) { 249bb0ec6b3SJim Harris while (ndisk->cur_depth > 0) { 250bb0ec6b3SJim Harris pause("nvd flush", 1); 251bb0ec6b3SJim Harris } 252bb0ec6b3SJim Harris } 253bb0ec6b3SJim Harris #endif 254bb0ec6b3SJim Harris } 255bb0ec6b3SJim Harris } 256bb0ec6b3SJim Harris 257038a5ee4SJim Harris static void * 258ec84ecbbSJim Harris nvd_new_controller(struct nvme_controller *ctrlr) 259ec84ecbbSJim Harris { 260ec84ecbbSJim Harris struct nvd_controller *nvd_ctrlr; 261ec84ecbbSJim Harris 262ec84ecbbSJim Harris nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD, 263ec84ecbbSJim Harris M_ZERO | M_NOWAIT); 264ec84ecbbSJim Harris 265ec84ecbbSJim Harris TAILQ_INIT(&nvd_ctrlr->disk_head); 266ec84ecbbSJim Harris TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq); 267ec84ecbbSJim Harris 268ec84ecbbSJim Harris return (nvd_ctrlr); 269ec84ecbbSJim Harris } 270ec84ecbbSJim Harris 271ec84ecbbSJim Harris static void * 272ec84ecbbSJim Harris nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg) 273bb0ec6b3SJim Harris { 274bb0ec6b3SJim Harris struct nvd_disk *ndisk; 275bb0ec6b3SJim Harris struct disk *disk; 276ec84ecbbSJim Harris struct nvd_controller *ctrlr = ctrlr_arg; 277bb0ec6b3SJim Harris 278bb0ec6b3SJim Harris ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_NOWAIT); 279bb0ec6b3SJim Harris 280bb0ec6b3SJim Harris disk = disk_alloc(); 281bb0ec6b3SJim Harris disk->d_strategy = nvd_strategy; 282bb0ec6b3SJim Harris disk->d_ioctl = nvd_ioctl; 283bb0ec6b3SJim Harris disk->d_name = "nvd"; 284bb0ec6b3SJim Harris disk->d_drv1 = ndisk; 285bb0ec6b3SJim Harris 286bb0ec6b3SJim Harris disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 287bb0ec6b3SJim Harris disk->d_sectorsize = nvme_ns_get_sector_size(ns); 288bb0ec6b3SJim Harris disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 289bb0ec6b3SJim Harris 290ec84ecbbSJim Harris if (TAILQ_EMPTY(&disk_head)) 291bb0ec6b3SJim Harris disk->d_unit = 0; 292bb0ec6b3SJim Harris else 293ec84ecbbSJim Harris disk->d_unit = 294ec84ecbbSJim Harris TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1; 295bb0ec6b3SJim Harris 296bb0ec6b3SJim Harris disk->d_flags = 0; 297bb0ec6b3SJim Harris 298bb0ec6b3SJim Harris if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 299bb0ec6b3SJim Harris disk->d_flags |= DISKFLAG_CANDELETE; 300bb0ec6b3SJim Harris 301bb0ec6b3SJim Harris if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 302bb0ec6b3SJim Harris disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 303bb0ec6b3SJim Harris 304bb0ec6b3SJim Harris strlcpy(disk->d_ident, nvme_ns_get_serial_number(ns), 305bb0ec6b3SJim Harris sizeof(disk->d_ident)); 306bb0ec6b3SJim Harris 307bb0ec6b3SJim Harris #if __FreeBSD_version >= 900034 308bb0ec6b3SJim Harris strlcpy(disk->d_descr, nvme_ns_get_model_number(ns), 309bb0ec6b3SJim Harris sizeof(disk->d_descr)); 310bb0ec6b3SJim Harris #endif 311bb0ec6b3SJim Harris 312bb0ec6b3SJim Harris disk_create(disk, DISK_VERSION); 313bb0ec6b3SJim Harris 314bb0ec6b3SJim Harris ndisk->ns = ns; 315bb0ec6b3SJim Harris ndisk->disk = disk; 316bb0ec6b3SJim Harris ndisk->cur_depth = 0; 317bb0ec6b3SJim Harris 318bb0ec6b3SJim Harris mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 319bb0ec6b3SJim Harris bioq_init(&ndisk->bioq); 320bb0ec6b3SJim Harris 321bb0ec6b3SJim Harris TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 322bb0ec6b3SJim Harris ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 323bb0ec6b3SJim Harris taskqueue_thread_enqueue, &ndisk->tq); 324bb0ec6b3SJim Harris taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 325bb0ec6b3SJim Harris 326ec84ecbbSJim Harris TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq); 327ec84ecbbSJim Harris TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq); 328038a5ee4SJim Harris 329038a5ee4SJim Harris return (NULL); 330bb0ec6b3SJim Harris } 331bb0ec6b3SJim Harris 332bb0ec6b3SJim Harris static void 333bb0ec6b3SJim Harris destroy_geom_disk(struct nvd_disk *ndisk) 334bb0ec6b3SJim Harris { 335bb0ec6b3SJim Harris struct bio *bp; 336bb0ec6b3SJim Harris 337bb0ec6b3SJim Harris taskqueue_free(ndisk->tq); 338bb0ec6b3SJim Harris disk_destroy(ndisk->disk); 339bb0ec6b3SJim Harris 340bb0ec6b3SJim Harris mtx_lock(&ndisk->bioqlock); 341bb0ec6b3SJim Harris for (;;) { 342bb0ec6b3SJim Harris bp = bioq_takefirst(&ndisk->bioq); 343bb0ec6b3SJim Harris if (bp == NULL) 344bb0ec6b3SJim Harris break; 345bb0ec6b3SJim Harris bp->bio_error = EIO; 346bb0ec6b3SJim Harris bp->bio_flags |= BIO_ERROR; 347bb0ec6b3SJim Harris bp->bio_resid = bp->bio_bcount; 348bb0ec6b3SJim Harris 349bb0ec6b3SJim Harris biodone(bp); 350bb0ec6b3SJim Harris } 351bb0ec6b3SJim Harris mtx_unlock(&ndisk->bioqlock); 352bb0ec6b3SJim Harris 353bb0ec6b3SJim Harris mtx_destroy(&ndisk->bioqlock); 354bb0ec6b3SJim Harris } 355*232e2edbSJim Harris 356*232e2edbSJim Harris static void 357*232e2edbSJim Harris nvd_controller_fail(void *ctrlr_arg) 358*232e2edbSJim Harris { 359*232e2edbSJim Harris struct nvd_controller *ctrlr = ctrlr_arg; 360*232e2edbSJim Harris struct nvd_disk *disk; 361*232e2edbSJim Harris 362*232e2edbSJim Harris while (!TAILQ_EMPTY(&ctrlr->disk_head)) { 363*232e2edbSJim Harris disk = TAILQ_FIRST(&ctrlr->disk_head); 364*232e2edbSJim Harris TAILQ_REMOVE(&disk_head, disk, global_tailq); 365*232e2edbSJim Harris TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq); 366*232e2edbSJim Harris destroy_geom_disk(disk); 367*232e2edbSJim Harris free(disk, M_NVD); 368*232e2edbSJim Harris } 369*232e2edbSJim Harris 370*232e2edbSJim Harris TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 371*232e2edbSJim Harris free(ctrlr, M_NVD); 372*232e2edbSJim Harris } 373*232e2edbSJim Harris 374