1 /*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bio.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/module.h> 35 #include <sys/systm.h> 36 #include <sys/taskqueue.h> 37 38 #include <geom/geom.h> 39 #include <geom/geom_disk.h> 40 41 #include <dev/nvme/nvme.h> 42 43 struct nvd_disk; 44 45 static disk_ioctl_t nvd_ioctl; 46 static disk_strategy_t nvd_strategy; 47 48 static void create_geom_disk(void *, struct nvme_namespace *ns); 49 static void destroy_geom_disk(struct nvd_disk *ndisk); 50 51 static int nvd_load(void); 52 static void nvd_unload(void); 53 54 MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 55 56 struct nvme_consumer *consumer_handle; 57 58 struct nvd_disk { 59 60 struct bio_queue_head bioq; 61 struct task bioqtask; 62 struct mtx bioqlock; 63 64 struct disk *disk; 65 struct taskqueue *tq; 66 struct nvme_namespace *ns; 67 68 uint32_t cur_depth; 69 70 TAILQ_ENTRY(nvd_disk) tailq; 71 }; 72 73 TAILQ_HEAD(, nvd_disk) nvd_head; 74 75 static int nvd_modevent(module_t mod, int type, void *arg) 76 { 77 int error = 0; 78 79 switch (type) { 80 case MOD_LOAD: 81 error = nvd_load(); 82 break; 83 case MOD_UNLOAD: 84 nvd_unload(); 85 break; 86 default: 87 break; 88 } 89 90 return (error); 91 } 92 93 moduledata_t nvd_mod = { 94 "nvd", 95 (modeventhand_t)nvd_modevent, 96 0 97 }; 98 99 DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 100 MODULE_VERSION(nvd, 1); 101 MODULE_DEPEND(nvd, nvme, 1, 1, 1); 102 103 static int 104 nvd_load() 105 { 106 107 TAILQ_INIT(&nvd_head); 108 consumer_handle = nvme_register_consumer(create_geom_disk, NULL); 109 110 return (consumer_handle != NULL ? 0 : -1); 111 } 112 113 static void 114 nvd_unload() 115 { 116 struct nvd_disk *nvd; 117 118 while (!TAILQ_EMPTY(&nvd_head)) { 119 nvd = TAILQ_FIRST(&nvd_head); 120 TAILQ_REMOVE(&nvd_head, nvd, tailq); 121 destroy_geom_disk(nvd); 122 free(nvd, M_NVD); 123 } 124 125 nvme_unregister_consumer(consumer_handle); 126 } 127 128 static void 129 nvd_strategy(struct bio *bp) 130 { 131 struct nvd_disk *ndisk; 132 133 ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 134 135 mtx_lock(&ndisk->bioqlock); 136 bioq_insert_tail(&ndisk->bioq, bp); 137 mtx_unlock(&ndisk->bioqlock); 138 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 139 } 140 141 static int 142 nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 143 struct thread *td) 144 { 145 int ret = 0; 146 147 switch (cmd) { 148 default: 149 ret = EIO; 150 } 151 152 return (ret); 153 } 154 155 static void 156 nvd_done(void *arg, const struct nvme_completion *status) 157 { 158 struct bio *bp; 159 struct nvd_disk *ndisk; 160 161 bp = (struct bio *)arg; 162 163 ndisk = bp->bio_disk->d_drv1; 164 165 atomic_add_int(&ndisk->cur_depth, -1); 166 167 /* 168 * TODO: add more extensive translation of NVMe status codes 169 * to different bio error codes (i.e. EIO, EINVAL, etc.) 170 */ 171 if (status->sf_sc || status->sf_sct) { 172 bp->bio_error = EIO; 173 bp->bio_flags |= BIO_ERROR; 174 bp->bio_resid = bp->bio_bcount; 175 } else 176 bp->bio_resid = 0; 177 178 biodone(bp); 179 } 180 181 static void 182 nvd_bioq_process(void *arg, int pending) 183 { 184 struct nvd_disk *ndisk = arg; 185 struct bio *bp; 186 int err; 187 188 for (;;) { 189 mtx_lock(&ndisk->bioqlock); 190 bp = bioq_takefirst(&ndisk->bioq); 191 mtx_unlock(&ndisk->bioqlock); 192 if (bp == NULL) 193 break; 194 195 #ifdef BIO_ORDERED 196 /* 197 * BIO_ORDERED flag dictates that all outstanding bios 198 * must be completed before processing the bio with 199 * BIO_ORDERED flag set. 200 */ 201 if (bp->bio_flags & BIO_ORDERED) { 202 while (ndisk->cur_depth > 0) { 203 pause("nvd flush", 1); 204 } 205 } 206 #endif 207 208 bp->bio_driver1 = NULL; 209 atomic_add_int(&ndisk->cur_depth, 1); 210 211 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 212 213 if (err) { 214 atomic_add_int(&ndisk->cur_depth, -1); 215 bp->bio_error = err; 216 bp->bio_flags |= BIO_ERROR; 217 bp->bio_resid = bp->bio_bcount; 218 biodone(bp); 219 } 220 221 #ifdef BIO_ORDERED 222 /* 223 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 224 * flag set must be completed before proceeding with 225 * additional bios. 226 */ 227 if (bp->bio_flags & BIO_ORDERED) { 228 while (ndisk->cur_depth > 0) { 229 pause("nvd flush", 1); 230 } 231 } 232 #endif 233 } 234 } 235 236 static void 237 create_geom_disk(void *arg, struct nvme_namespace *ns) 238 { 239 struct nvd_disk *ndisk; 240 struct disk *disk; 241 242 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_NOWAIT); 243 244 disk = disk_alloc(); 245 disk->d_strategy = nvd_strategy; 246 disk->d_ioctl = nvd_ioctl; 247 disk->d_name = "nvd"; 248 disk->d_drv1 = ndisk; 249 250 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 251 disk->d_sectorsize = nvme_ns_get_sector_size(ns); 252 disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 253 254 if (TAILQ_EMPTY(&nvd_head)) 255 disk->d_unit = 0; 256 else 257 disk->d_unit = TAILQ_FIRST(&nvd_head)->disk->d_unit + 1; 258 259 disk->d_flags = 0; 260 261 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 262 disk->d_flags |= DISKFLAG_CANDELETE; 263 264 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 265 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 266 267 strlcpy(disk->d_ident, nvme_ns_get_serial_number(ns), 268 sizeof(disk->d_ident)); 269 270 #if __FreeBSD_version >= 900034 271 strlcpy(disk->d_descr, nvme_ns_get_model_number(ns), 272 sizeof(disk->d_descr)); 273 #endif 274 275 disk_create(disk, DISK_VERSION); 276 277 ndisk->ns = ns; 278 ndisk->disk = disk; 279 ndisk->cur_depth = 0; 280 281 mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 282 bioq_init(&ndisk->bioq); 283 284 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 285 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 286 taskqueue_thread_enqueue, &ndisk->tq); 287 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 288 289 TAILQ_INSERT_HEAD(&nvd_head, ndisk, tailq); 290 } 291 292 static void 293 destroy_geom_disk(struct nvd_disk *ndisk) 294 { 295 struct bio *bp; 296 297 taskqueue_free(ndisk->tq); 298 disk_destroy(ndisk->disk); 299 300 mtx_lock(&ndisk->bioqlock); 301 for (;;) { 302 bp = bioq_takefirst(&ndisk->bioq); 303 if (bp == NULL) 304 break; 305 bp->bio_error = EIO; 306 bp->bio_flags |= BIO_ERROR; 307 bp->bio_resid = bp->bio_bcount; 308 309 biodone(bp); 310 } 311 mtx_unlock(&ndisk->bioqlock); 312 313 mtx_destroy(&ndisk->bioqlock); 314 } 315