1 /*- 2 * Copyright (C) 2012-2013 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bio.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/module.h> 35 #include <sys/systm.h> 36 #include <sys/taskqueue.h> 37 38 #include <geom/geom.h> 39 #include <geom/geom_disk.h> 40 41 #include <dev/nvme/nvme.h> 42 43 #define NVD_STR "nvd" 44 45 struct nvd_disk; 46 47 static disk_ioctl_t nvd_ioctl; 48 static disk_strategy_t nvd_strategy; 49 50 static void nvd_done(void *arg, const struct nvme_completion *cpl); 51 52 static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr); 53 static void destroy_geom_disk(struct nvd_disk *ndisk); 54 55 static void *nvd_new_controller(struct nvme_controller *ctrlr); 56 static void nvd_controller_fail(void *ctrlr); 57 58 static int nvd_load(void); 59 static void nvd_unload(void); 60 61 MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 62 63 struct nvme_consumer *consumer_handle; 64 65 struct nvd_disk { 66 67 struct bio_queue_head bioq; 68 struct task bioqtask; 69 struct mtx bioqlock; 70 71 struct disk *disk; 72 struct taskqueue *tq; 73 struct nvme_namespace *ns; 74 75 uint32_t cur_depth; 76 uint32_t ordered_in_flight; 77 78 TAILQ_ENTRY(nvd_disk) global_tailq; 79 TAILQ_ENTRY(nvd_disk) ctrlr_tailq; 80 }; 81 82 struct nvd_controller { 83 84 TAILQ_ENTRY(nvd_controller) tailq; 85 TAILQ_HEAD(, nvd_disk) disk_head; 86 }; 87 88 static TAILQ_HEAD(, nvd_controller) ctrlr_head; 89 static TAILQ_HEAD(disk_list, nvd_disk) disk_head; 90 91 static int nvd_modevent(module_t mod, int type, void *arg) 92 { 93 int error = 0; 94 95 switch (type) { 96 case MOD_LOAD: 97 error = nvd_load(); 98 break; 99 case MOD_UNLOAD: 100 nvd_unload(); 101 break; 102 default: 103 break; 104 } 105 106 return (error); 107 } 108 109 moduledata_t nvd_mod = { 110 NVD_STR, 111 (modeventhand_t)nvd_modevent, 112 0 113 }; 114 115 DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 116 MODULE_VERSION(nvd, 1); 117 MODULE_DEPEND(nvd, nvme, 1, 1, 1); 118 119 static int 120 nvd_load() 121 { 122 123 TAILQ_INIT(&ctrlr_head); 124 TAILQ_INIT(&disk_head); 125 126 consumer_handle = nvme_register_consumer(nvd_new_disk, 127 nvd_new_controller, NULL, nvd_controller_fail); 128 129 return (consumer_handle != NULL ? 0 : -1); 130 } 131 132 static void 133 nvd_unload() 134 { 135 struct nvd_controller *ctrlr; 136 struct nvd_disk *disk; 137 138 while (!TAILQ_EMPTY(&ctrlr_head)) { 139 ctrlr = TAILQ_FIRST(&ctrlr_head); 140 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 141 free(ctrlr, M_NVD); 142 } 143 144 while (!TAILQ_EMPTY(&disk_head)) { 145 disk = TAILQ_FIRST(&disk_head); 146 TAILQ_REMOVE(&disk_head, disk, global_tailq); 147 destroy_geom_disk(disk); 148 free(disk, M_NVD); 149 } 150 151 nvme_unregister_consumer(consumer_handle); 152 } 153 154 static int 155 nvd_bio_submit(struct nvd_disk *ndisk, struct bio *bp) 156 { 157 int err; 158 159 bp->bio_driver1 = NULL; 160 atomic_add_int(&ndisk->cur_depth, 1); 161 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 162 if (err) { 163 atomic_add_int(&ndisk->cur_depth, -1); 164 if (__predict_false(bp->bio_flags & BIO_ORDERED)) 165 atomic_add_int(&ndisk->ordered_in_flight, -1); 166 bp->bio_error = err; 167 bp->bio_flags |= BIO_ERROR; 168 bp->bio_resid = bp->bio_bcount; 169 biodone(bp); 170 return (-1); 171 } 172 173 return (0); 174 } 175 176 static void 177 nvd_strategy(struct bio *bp) 178 { 179 struct nvd_disk *ndisk; 180 181 ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 182 183 if (__predict_false(bp->bio_flags & BIO_ORDERED)) 184 atomic_add_int(&ndisk->ordered_in_flight, 1); 185 186 if (__predict_true(ndisk->ordered_in_flight == 0)) { 187 nvd_bio_submit(ndisk, bp); 188 return; 189 } 190 191 /* 192 * There are ordered bios in flight, so we need to submit 193 * bios through the task queue to enforce ordering. 194 */ 195 mtx_lock(&ndisk->bioqlock); 196 bioq_insert_tail(&ndisk->bioq, bp); 197 mtx_unlock(&ndisk->bioqlock); 198 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 199 } 200 201 static int 202 nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 203 struct thread *td) 204 { 205 int ret = 0; 206 207 switch (cmd) { 208 default: 209 ret = EIO; 210 } 211 212 return (ret); 213 } 214 215 static void 216 nvd_done(void *arg, const struct nvme_completion *cpl) 217 { 218 struct bio *bp; 219 struct nvd_disk *ndisk; 220 221 bp = (struct bio *)arg; 222 223 ndisk = bp->bio_disk->d_drv1; 224 225 atomic_add_int(&ndisk->cur_depth, -1); 226 if (__predict_false(bp->bio_flags & BIO_ORDERED)) 227 atomic_add_int(&ndisk->ordered_in_flight, -1); 228 229 biodone(bp); 230 } 231 232 static void 233 nvd_bioq_process(void *arg, int pending) 234 { 235 struct nvd_disk *ndisk = arg; 236 struct bio *bp; 237 238 for (;;) { 239 mtx_lock(&ndisk->bioqlock); 240 bp = bioq_takefirst(&ndisk->bioq); 241 mtx_unlock(&ndisk->bioqlock); 242 if (bp == NULL) 243 break; 244 245 if (nvd_bio_submit(ndisk, bp) != 0) { 246 continue; 247 } 248 249 #ifdef BIO_ORDERED 250 /* 251 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 252 * flag set must be completed before proceeding with 253 * additional bios. 254 */ 255 if (bp->bio_flags & BIO_ORDERED) { 256 while (ndisk->cur_depth > 0) { 257 pause("nvd flush", 1); 258 } 259 } 260 #endif 261 } 262 } 263 264 static void * 265 nvd_new_controller(struct nvme_controller *ctrlr) 266 { 267 struct nvd_controller *nvd_ctrlr; 268 269 nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD, 270 M_ZERO | M_WAITOK); 271 272 TAILQ_INIT(&nvd_ctrlr->disk_head); 273 TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq); 274 275 return (nvd_ctrlr); 276 } 277 278 static void * 279 nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg) 280 { 281 uint8_t descr[NVME_MODEL_NUMBER_LENGTH+1]; 282 struct nvd_disk *ndisk; 283 struct disk *disk; 284 struct nvd_controller *ctrlr = ctrlr_arg; 285 286 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK); 287 288 disk = disk_alloc(); 289 disk->d_strategy = nvd_strategy; 290 disk->d_ioctl = nvd_ioctl; 291 disk->d_name = NVD_STR; 292 disk->d_drv1 = ndisk; 293 294 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 295 disk->d_sectorsize = nvme_ns_get_sector_size(ns); 296 disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 297 disk->d_delmaxsize = (off_t)nvme_ns_get_size(ns); 298 disk->d_stripesize = nvme_ns_get_optimal_sector_size(ns); 299 300 if (TAILQ_EMPTY(&disk_head)) 301 disk->d_unit = 0; 302 else 303 disk->d_unit = 304 TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1; 305 306 disk->d_flags = DISKFLAG_DIRECT_COMPLETION; 307 308 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 309 disk->d_flags |= DISKFLAG_CANDELETE; 310 311 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 312 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 313 314 /* ifdef used here to ease porting to stable branches at a later point. */ 315 #ifdef DISKFLAG_UNMAPPED_BIO 316 disk->d_flags |= DISKFLAG_UNMAPPED_BIO; 317 #endif 318 319 /* 320 * d_ident and d_descr are both far bigger than the length of either 321 * the serial or model number strings. 322 */ 323 nvme_strvis(disk->d_ident, nvme_ns_get_serial_number(ns), 324 sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH); 325 326 nvme_strvis(descr, nvme_ns_get_model_number(ns), sizeof(descr), 327 NVME_MODEL_NUMBER_LENGTH); 328 329 #if __FreeBSD_version >= 900034 330 strlcpy(disk->d_descr, descr, sizeof(descr)); 331 #endif 332 333 ndisk->ns = ns; 334 ndisk->disk = disk; 335 ndisk->cur_depth = 0; 336 ndisk->ordered_in_flight = 0; 337 338 mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 339 bioq_init(&ndisk->bioq); 340 341 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 342 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 343 taskqueue_thread_enqueue, &ndisk->tq); 344 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 345 346 TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq); 347 TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq); 348 349 disk_create(disk, DISK_VERSION); 350 351 printf(NVD_STR"%u: <%s> NVMe namespace\n", disk->d_unit, descr); 352 printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit, 353 (uintmax_t)disk->d_mediasize / (1024*1024), 354 (uintmax_t)disk->d_mediasize / disk->d_sectorsize, 355 disk->d_sectorsize); 356 357 return (NULL); 358 } 359 360 static void 361 destroy_geom_disk(struct nvd_disk *ndisk) 362 { 363 struct bio *bp; 364 struct disk *disk; 365 uint32_t unit; 366 int cnt = 0; 367 368 disk = ndisk->disk; 369 unit = disk->d_unit; 370 taskqueue_free(ndisk->tq); 371 372 disk_destroy(ndisk->disk); 373 374 mtx_lock(&ndisk->bioqlock); 375 for (;;) { 376 bp = bioq_takefirst(&ndisk->bioq); 377 if (bp == NULL) 378 break; 379 bp->bio_error = EIO; 380 bp->bio_flags |= BIO_ERROR; 381 bp->bio_resid = bp->bio_bcount; 382 cnt++; 383 biodone(bp); 384 } 385 386 printf(NVD_STR"%u: lost device - %d outstanding\n", unit, cnt); 387 printf(NVD_STR"%u: removing device entry\n", unit); 388 389 mtx_unlock(&ndisk->bioqlock); 390 391 mtx_destroy(&ndisk->bioqlock); 392 } 393 394 static void 395 nvd_controller_fail(void *ctrlr_arg) 396 { 397 struct nvd_controller *ctrlr = ctrlr_arg; 398 struct nvd_disk *disk; 399 400 while (!TAILQ_EMPTY(&ctrlr->disk_head)) { 401 disk = TAILQ_FIRST(&ctrlr->disk_head); 402 TAILQ_REMOVE(&disk_head, disk, global_tailq); 403 TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq); 404 destroy_geom_disk(disk); 405 free(disk, M_NVD); 406 } 407 408 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 409 free(ctrlr, M_NVD); 410 } 411 412