1 /*- 2 * Copyright (C) 2012-2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bio.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/module.h> 35 #include <sys/systm.h> 36 #include <sys/taskqueue.h> 37 38 #include <geom/geom.h> 39 #include <geom/geom_disk.h> 40 41 #include <dev/nvme/nvme.h> 42 43 #define NVD_STR "nvd" 44 45 struct nvd_disk; 46 47 static disk_ioctl_t nvd_ioctl; 48 static disk_strategy_t nvd_strategy; 49 50 static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr); 51 static void destroy_geom_disk(struct nvd_disk *ndisk); 52 53 static void *nvd_new_controller(struct nvme_controller *ctrlr); 54 static void nvd_controller_fail(void *ctrlr); 55 56 static int nvd_load(void); 57 static void nvd_unload(void); 58 59 MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 60 61 struct nvme_consumer *consumer_handle; 62 63 struct nvd_disk { 64 65 struct bio_queue_head bioq; 66 struct task bioqtask; 67 struct mtx bioqlock; 68 69 struct disk *disk; 70 struct taskqueue *tq; 71 struct nvme_namespace *ns; 72 73 uint32_t cur_depth; 74 75 TAILQ_ENTRY(nvd_disk) global_tailq; 76 TAILQ_ENTRY(nvd_disk) ctrlr_tailq; 77 }; 78 79 struct nvd_controller { 80 81 TAILQ_ENTRY(nvd_controller) tailq; 82 TAILQ_HEAD(, nvd_disk) disk_head; 83 }; 84 85 static TAILQ_HEAD(, nvd_controller) ctrlr_head; 86 static TAILQ_HEAD(disk_list, nvd_disk) disk_head; 87 88 static int nvd_modevent(module_t mod, int type, void *arg) 89 { 90 int error = 0; 91 92 switch (type) { 93 case MOD_LOAD: 94 error = nvd_load(); 95 break; 96 case MOD_UNLOAD: 97 nvd_unload(); 98 break; 99 default: 100 break; 101 } 102 103 return (error); 104 } 105 106 moduledata_t nvd_mod = { 107 NVD_STR, 108 (modeventhand_t)nvd_modevent, 109 0 110 }; 111 112 DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 113 MODULE_VERSION(nvd, 1); 114 MODULE_DEPEND(nvd, nvme, 1, 1, 1); 115 116 static int 117 nvd_load() 118 { 119 120 TAILQ_INIT(&ctrlr_head); 121 TAILQ_INIT(&disk_head); 122 123 consumer_handle = nvme_register_consumer(nvd_new_disk, 124 nvd_new_controller, NULL, nvd_controller_fail); 125 126 return (consumer_handle != NULL ? 0 : -1); 127 } 128 129 static void 130 nvd_unload() 131 { 132 struct nvd_controller *ctrlr; 133 struct nvd_disk *disk; 134 135 while (!TAILQ_EMPTY(&ctrlr_head)) { 136 ctrlr = TAILQ_FIRST(&ctrlr_head); 137 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 138 free(ctrlr, M_NVD); 139 } 140 141 while (!TAILQ_EMPTY(&disk_head)) { 142 disk = TAILQ_FIRST(&disk_head); 143 TAILQ_REMOVE(&disk_head, disk, global_tailq); 144 destroy_geom_disk(disk); 145 free(disk, M_NVD); 146 } 147 148 nvme_unregister_consumer(consumer_handle); 149 } 150 151 static void 152 nvd_strategy(struct bio *bp) 153 { 154 struct nvd_disk *ndisk; 155 156 ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 157 158 mtx_lock(&ndisk->bioqlock); 159 bioq_insert_tail(&ndisk->bioq, bp); 160 mtx_unlock(&ndisk->bioqlock); 161 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 162 } 163 164 static int 165 nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 166 struct thread *td) 167 { 168 int ret = 0; 169 170 switch (cmd) { 171 default: 172 ret = EIO; 173 } 174 175 return (ret); 176 } 177 178 static void 179 nvd_done(void *arg, const struct nvme_completion *cpl) 180 { 181 struct bio *bp; 182 struct nvd_disk *ndisk; 183 184 bp = (struct bio *)arg; 185 186 ndisk = bp->bio_disk->d_drv1; 187 188 atomic_add_int(&ndisk->cur_depth, -1); 189 190 /* 191 * TODO: add more extensive translation of NVMe status codes 192 * to different bio error codes (i.e. EIO, EINVAL, etc.) 193 */ 194 if (nvme_completion_is_error(cpl)) { 195 bp->bio_error = EIO; 196 bp->bio_flags |= BIO_ERROR; 197 bp->bio_resid = bp->bio_bcount; 198 } else 199 bp->bio_resid = 0; 200 201 biodone(bp); 202 } 203 204 static void 205 nvd_bioq_process(void *arg, int pending) 206 { 207 struct nvd_disk *ndisk = arg; 208 struct bio *bp; 209 int err; 210 211 for (;;) { 212 mtx_lock(&ndisk->bioqlock); 213 bp = bioq_takefirst(&ndisk->bioq); 214 mtx_unlock(&ndisk->bioqlock); 215 if (bp == NULL) 216 break; 217 218 #ifdef BIO_ORDERED 219 /* 220 * BIO_ORDERED flag dictates that all outstanding bios 221 * must be completed before processing the bio with 222 * BIO_ORDERED flag set. 223 */ 224 if (bp->bio_flags & BIO_ORDERED) { 225 while (ndisk->cur_depth > 0) { 226 pause("nvd flush", 1); 227 } 228 } 229 #endif 230 231 bp->bio_driver1 = NULL; 232 atomic_add_int(&ndisk->cur_depth, 1); 233 234 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 235 236 if (err) { 237 atomic_add_int(&ndisk->cur_depth, -1); 238 bp->bio_error = err; 239 bp->bio_flags |= BIO_ERROR; 240 bp->bio_resid = bp->bio_bcount; 241 biodone(bp); 242 } 243 244 #ifdef BIO_ORDERED 245 /* 246 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 247 * flag set must be completed before proceeding with 248 * additional bios. 249 */ 250 if (bp->bio_flags & BIO_ORDERED) { 251 while (ndisk->cur_depth > 0) { 252 pause("nvd flush", 1); 253 } 254 } 255 #endif 256 } 257 } 258 259 static void * 260 nvd_new_controller(struct nvme_controller *ctrlr) 261 { 262 struct nvd_controller *nvd_ctrlr; 263 264 nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD, 265 M_ZERO | M_WAITOK); 266 267 TAILQ_INIT(&nvd_ctrlr->disk_head); 268 TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq); 269 270 return (nvd_ctrlr); 271 } 272 273 static void * 274 nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg) 275 { 276 uint8_t descr[NVME_MODEL_NUMBER_LENGTH+1]; 277 struct nvd_disk *ndisk; 278 struct disk *disk; 279 struct nvd_controller *ctrlr = ctrlr_arg; 280 281 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK); 282 283 disk = disk_alloc(); 284 disk->d_strategy = nvd_strategy; 285 disk->d_ioctl = nvd_ioctl; 286 disk->d_name = NVD_STR; 287 disk->d_drv1 = ndisk; 288 289 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 290 disk->d_sectorsize = nvme_ns_get_sector_size(ns); 291 disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 292 293 if (TAILQ_EMPTY(&disk_head)) 294 disk->d_unit = 0; 295 else 296 disk->d_unit = 297 TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1; 298 299 disk->d_flags = 0; 300 301 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 302 disk->d_flags |= DISKFLAG_CANDELETE; 303 304 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 305 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 306 307 /* ifdef used here to ease porting to stable branches at a later point. */ 308 #ifdef DISKFLAG_UNMAPPED_BIO 309 disk->d_flags |= DISKFLAG_UNMAPPED_BIO; 310 #endif 311 312 /* 313 * d_ident and d_descr are both far bigger than the length of either 314 * the serial or model number strings. 315 */ 316 nvme_strvis(disk->d_ident, nvme_ns_get_serial_number(ns), 317 sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH); 318 319 nvme_strvis(descr, nvme_ns_get_model_number(ns), sizeof(descr), 320 NVME_MODEL_NUMBER_LENGTH); 321 322 #if __FreeBSD_version >= 900034 323 strlcpy(disk->d_descr, descr, sizeof(descr)); 324 #endif 325 326 ndisk->ns = ns; 327 ndisk->disk = disk; 328 ndisk->cur_depth = 0; 329 330 mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 331 bioq_init(&ndisk->bioq); 332 333 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 334 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 335 taskqueue_thread_enqueue, &ndisk->tq); 336 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 337 338 TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq); 339 TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq); 340 341 disk_create(disk, DISK_VERSION); 342 343 printf(NVD_STR"%u: <%s> NVMe namespace\n", disk->d_unit, descr); 344 printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit, 345 (uintmax_t)disk->d_mediasize / (1024*1024), 346 (uintmax_t)disk->d_mediasize / disk->d_sectorsize, 347 disk->d_sectorsize); 348 349 return (NULL); 350 } 351 352 static void 353 destroy_geom_disk(struct nvd_disk *ndisk) 354 { 355 struct bio *bp; 356 struct disk *disk; 357 uint32_t unit; 358 int cnt = 0; 359 360 disk = ndisk->disk; 361 unit = disk->d_unit; 362 taskqueue_free(ndisk->tq); 363 364 disk_destroy(ndisk->disk); 365 366 mtx_lock(&ndisk->bioqlock); 367 for (;;) { 368 bp = bioq_takefirst(&ndisk->bioq); 369 if (bp == NULL) 370 break; 371 bp->bio_error = EIO; 372 bp->bio_flags |= BIO_ERROR; 373 bp->bio_resid = bp->bio_bcount; 374 cnt++; 375 biodone(bp); 376 } 377 378 printf(NVD_STR"%u: lost device - %d outstanding\n", unit, cnt); 379 printf(NVD_STR"%u: removing device entry\n", unit); 380 381 mtx_unlock(&ndisk->bioqlock); 382 383 mtx_destroy(&ndisk->bioqlock); 384 } 385 386 static void 387 nvd_controller_fail(void *ctrlr_arg) 388 { 389 struct nvd_controller *ctrlr = ctrlr_arg; 390 struct nvd_disk *disk; 391 392 while (!TAILQ_EMPTY(&ctrlr->disk_head)) { 393 disk = TAILQ_FIRST(&ctrlr->disk_head); 394 TAILQ_REMOVE(&disk_head, disk, global_tailq); 395 TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq); 396 destroy_geom_disk(disk); 397 free(disk, M_NVD); 398 } 399 400 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 401 free(ctrlr, M_NVD); 402 } 403 404