1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2012-2016 Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bio.h> 34 #include <sys/kernel.h> 35 #include <sys/malloc.h> 36 #include <sys/module.h> 37 #include <sys/sysctl.h> 38 #include <sys/systm.h> 39 #include <sys/taskqueue.h> 40 41 #include <geom/geom.h> 42 #include <geom/geom_disk.h> 43 44 #include <dev/nvme/nvme.h> 45 46 #define NVD_STR "nvd" 47 48 struct nvd_disk; 49 50 static disk_ioctl_t nvd_ioctl; 51 static disk_strategy_t nvd_strategy; 52 static dumper_t nvd_dump; 53 54 static void nvd_done(void *arg, const struct nvme_completion *cpl); 55 56 static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr); 57 static void destroy_geom_disk(struct nvd_disk *ndisk); 58 59 static void *nvd_new_controller(struct nvme_controller *ctrlr); 60 static void nvd_controller_fail(void *ctrlr); 61 62 static int nvd_load(void); 63 static void nvd_unload(void); 64 65 MALLOC_DEFINE(M_NVD, "nvd", "nvd(4) allocations"); 66 67 struct nvme_consumer *consumer_handle; 68 69 struct nvd_disk { 70 71 struct bio_queue_head bioq; 72 struct task bioqtask; 73 struct mtx bioqlock; 74 75 struct disk *disk; 76 struct taskqueue *tq; 77 struct nvme_namespace *ns; 78 79 uint32_t cur_depth; 80 uint32_t ordered_in_flight; 81 82 TAILQ_ENTRY(nvd_disk) global_tailq; 83 TAILQ_ENTRY(nvd_disk) ctrlr_tailq; 84 }; 85 86 struct nvd_controller { 87 88 TAILQ_ENTRY(nvd_controller) tailq; 89 TAILQ_HEAD(, nvd_disk) disk_head; 90 }; 91 92 static TAILQ_HEAD(, nvd_controller) ctrlr_head; 93 static TAILQ_HEAD(disk_list, nvd_disk) disk_head; 94 95 static SYSCTL_NODE(_hw, OID_AUTO, nvd, CTLFLAG_RD, 0, "nvd driver parameters"); 96 /* 97 * The NVMe specification does not define a maximum or optimal delete size, so 98 * technically max delete size is min(full size of the namespace, 2^32 - 1 99 * LBAs). A single delete for a multi-TB NVMe namespace though may take much 100 * longer to complete than the nvme(4) I/O timeout period. So choose a sensible 101 * default here that is still suitably large to minimize the number of overall 102 * delete operations. 103 */ 104 static uint64_t nvd_delete_max = (1024 * 1024 * 1024); /* 1GB */ 105 SYSCTL_UQUAD(_hw_nvd, OID_AUTO, delete_max, CTLFLAG_RDTUN, &nvd_delete_max, 0, 106 "nvd maximum BIO_DELETE size in bytes"); 107 108 static int nvd_modevent(module_t mod, int type, void *arg) 109 { 110 int error = 0; 111 112 switch (type) { 113 case MOD_LOAD: 114 error = nvd_load(); 115 break; 116 case MOD_UNLOAD: 117 nvd_unload(); 118 break; 119 default: 120 break; 121 } 122 123 return (error); 124 } 125 126 moduledata_t nvd_mod = { 127 NVD_STR, 128 (modeventhand_t)nvd_modevent, 129 0 130 }; 131 132 DECLARE_MODULE(nvd, nvd_mod, SI_SUB_DRIVERS, SI_ORDER_ANY); 133 MODULE_VERSION(nvd, 1); 134 MODULE_DEPEND(nvd, nvme, 1, 1, 1); 135 136 static int 137 nvd_load() 138 { 139 if (!nvme_use_nvd) 140 return 0; 141 142 TAILQ_INIT(&ctrlr_head); 143 TAILQ_INIT(&disk_head); 144 145 consumer_handle = nvme_register_consumer(nvd_new_disk, 146 nvd_new_controller, NULL, nvd_controller_fail); 147 148 return (consumer_handle != NULL ? 0 : -1); 149 } 150 151 static void 152 nvd_unload() 153 { 154 struct nvd_controller *ctrlr; 155 struct nvd_disk *disk; 156 157 if (!nvme_use_nvd) 158 return; 159 160 while (!TAILQ_EMPTY(&ctrlr_head)) { 161 ctrlr = TAILQ_FIRST(&ctrlr_head); 162 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 163 free(ctrlr, M_NVD); 164 } 165 166 while (!TAILQ_EMPTY(&disk_head)) { 167 disk = TAILQ_FIRST(&disk_head); 168 TAILQ_REMOVE(&disk_head, disk, global_tailq); 169 destroy_geom_disk(disk); 170 free(disk, M_NVD); 171 } 172 173 nvme_unregister_consumer(consumer_handle); 174 } 175 176 static int 177 nvd_bio_submit(struct nvd_disk *ndisk, struct bio *bp) 178 { 179 int err; 180 181 bp->bio_driver1 = NULL; 182 atomic_add_int(&ndisk->cur_depth, 1); 183 err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); 184 if (err) { 185 atomic_add_int(&ndisk->cur_depth, -1); 186 if (__predict_false(bp->bio_flags & BIO_ORDERED)) 187 atomic_add_int(&ndisk->ordered_in_flight, -1); 188 bp->bio_error = err; 189 bp->bio_flags |= BIO_ERROR; 190 bp->bio_resid = bp->bio_bcount; 191 biodone(bp); 192 return (-1); 193 } 194 195 return (0); 196 } 197 198 static void 199 nvd_strategy(struct bio *bp) 200 { 201 struct nvd_disk *ndisk; 202 203 ndisk = (struct nvd_disk *)bp->bio_disk->d_drv1; 204 205 if (__predict_false(bp->bio_flags & BIO_ORDERED)) 206 atomic_add_int(&ndisk->ordered_in_flight, 1); 207 208 if (__predict_true(ndisk->ordered_in_flight == 0)) { 209 nvd_bio_submit(ndisk, bp); 210 return; 211 } 212 213 /* 214 * There are ordered bios in flight, so we need to submit 215 * bios through the task queue to enforce ordering. 216 */ 217 mtx_lock(&ndisk->bioqlock); 218 bioq_insert_tail(&ndisk->bioq, bp); 219 mtx_unlock(&ndisk->bioqlock); 220 taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); 221 } 222 223 static int 224 nvd_ioctl(struct disk *ndisk, u_long cmd, void *data, int fflag, 225 struct thread *td) 226 { 227 int ret = 0; 228 229 switch (cmd) { 230 default: 231 ret = EIO; 232 } 233 234 return (ret); 235 } 236 237 static int 238 nvd_dump(void *arg, void *virt, vm_offset_t phys, off_t offset, size_t len) 239 { 240 struct nvd_disk *ndisk; 241 struct disk *dp; 242 243 dp = arg; 244 ndisk = dp->d_drv1; 245 246 return (nvme_ns_dump(ndisk->ns, virt, offset, len)); 247 } 248 249 static void 250 nvd_done(void *arg, const struct nvme_completion *cpl) 251 { 252 struct bio *bp; 253 struct nvd_disk *ndisk; 254 255 bp = (struct bio *)arg; 256 257 ndisk = bp->bio_disk->d_drv1; 258 259 atomic_add_int(&ndisk->cur_depth, -1); 260 if (__predict_false(bp->bio_flags & BIO_ORDERED)) 261 atomic_add_int(&ndisk->ordered_in_flight, -1); 262 263 biodone(bp); 264 } 265 266 static void 267 nvd_bioq_process(void *arg, int pending) 268 { 269 struct nvd_disk *ndisk = arg; 270 struct bio *bp; 271 272 for (;;) { 273 mtx_lock(&ndisk->bioqlock); 274 bp = bioq_takefirst(&ndisk->bioq); 275 mtx_unlock(&ndisk->bioqlock); 276 if (bp == NULL) 277 break; 278 279 if (nvd_bio_submit(ndisk, bp) != 0) { 280 continue; 281 } 282 283 #ifdef BIO_ORDERED 284 /* 285 * BIO_ORDERED flag dictates that the bio with BIO_ORDERED 286 * flag set must be completed before proceeding with 287 * additional bios. 288 */ 289 if (bp->bio_flags & BIO_ORDERED) { 290 while (ndisk->cur_depth > 0) { 291 pause("nvd flush", 1); 292 } 293 } 294 #endif 295 } 296 } 297 298 static void * 299 nvd_new_controller(struct nvme_controller *ctrlr) 300 { 301 struct nvd_controller *nvd_ctrlr; 302 303 nvd_ctrlr = malloc(sizeof(struct nvd_controller), M_NVD, 304 M_ZERO | M_WAITOK); 305 306 TAILQ_INIT(&nvd_ctrlr->disk_head); 307 TAILQ_INSERT_TAIL(&ctrlr_head, nvd_ctrlr, tailq); 308 309 return (nvd_ctrlr); 310 } 311 312 static void * 313 nvd_new_disk(struct nvme_namespace *ns, void *ctrlr_arg) 314 { 315 uint8_t descr[NVME_MODEL_NUMBER_LENGTH+1]; 316 struct nvd_disk *ndisk; 317 struct disk *disk; 318 struct nvd_controller *ctrlr = ctrlr_arg; 319 320 ndisk = malloc(sizeof(struct nvd_disk), M_NVD, M_ZERO | M_WAITOK); 321 322 disk = disk_alloc(); 323 disk->d_strategy = nvd_strategy; 324 disk->d_ioctl = nvd_ioctl; 325 disk->d_dump = nvd_dump; 326 disk->d_name = NVD_STR; 327 disk->d_drv1 = ndisk; 328 329 disk->d_maxsize = nvme_ns_get_max_io_xfer_size(ns); 330 disk->d_sectorsize = nvme_ns_get_sector_size(ns); 331 disk->d_mediasize = (off_t)nvme_ns_get_size(ns); 332 disk->d_delmaxsize = (off_t)nvme_ns_get_size(ns); 333 if (disk->d_delmaxsize > nvd_delete_max) 334 disk->d_delmaxsize = nvd_delete_max; 335 disk->d_stripesize = nvme_ns_get_stripesize(ns); 336 337 if (TAILQ_EMPTY(&disk_head)) 338 disk->d_unit = 0; 339 else 340 disk->d_unit = 341 TAILQ_LAST(&disk_head, disk_list)->disk->d_unit + 1; 342 343 disk->d_flags = DISKFLAG_DIRECT_COMPLETION; 344 345 if (nvme_ns_get_flags(ns) & NVME_NS_DEALLOCATE_SUPPORTED) 346 disk->d_flags |= DISKFLAG_CANDELETE; 347 348 if (nvme_ns_get_flags(ns) & NVME_NS_FLUSH_SUPPORTED) 349 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; 350 351 /* ifdef used here to ease porting to stable branches at a later point. */ 352 #ifdef DISKFLAG_UNMAPPED_BIO 353 disk->d_flags |= DISKFLAG_UNMAPPED_BIO; 354 #endif 355 356 /* 357 * d_ident and d_descr are both far bigger than the length of either 358 * the serial or model number strings. 359 */ 360 nvme_strvis(disk->d_ident, nvme_ns_get_serial_number(ns), 361 sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH); 362 nvme_strvis(descr, nvme_ns_get_model_number(ns), sizeof(descr), 363 NVME_MODEL_NUMBER_LENGTH); 364 strlcpy(disk->d_descr, descr, sizeof(descr)); 365 366 disk->d_rotation_rate = DISK_RR_NON_ROTATING; 367 368 ndisk->ns = ns; 369 ndisk->disk = disk; 370 ndisk->cur_depth = 0; 371 ndisk->ordered_in_flight = 0; 372 373 mtx_init(&ndisk->bioqlock, "NVD bioq lock", NULL, MTX_DEF); 374 bioq_init(&ndisk->bioq); 375 376 TASK_INIT(&ndisk->bioqtask, 0, nvd_bioq_process, ndisk); 377 ndisk->tq = taskqueue_create("nvd_taskq", M_WAITOK, 378 taskqueue_thread_enqueue, &ndisk->tq); 379 taskqueue_start_threads(&ndisk->tq, 1, PI_DISK, "nvd taskq"); 380 381 TAILQ_INSERT_TAIL(&disk_head, ndisk, global_tailq); 382 TAILQ_INSERT_TAIL(&ctrlr->disk_head, ndisk, ctrlr_tailq); 383 384 disk_create(disk, DISK_VERSION); 385 386 printf(NVD_STR"%u: <%s> NVMe namespace\n", disk->d_unit, descr); 387 printf(NVD_STR"%u: %juMB (%ju %u byte sectors)\n", disk->d_unit, 388 (uintmax_t)disk->d_mediasize / (1024*1024), 389 (uintmax_t)disk->d_mediasize / disk->d_sectorsize, 390 disk->d_sectorsize); 391 392 return (NULL); 393 } 394 395 static void 396 destroy_geom_disk(struct nvd_disk *ndisk) 397 { 398 struct bio *bp; 399 struct disk *disk; 400 uint32_t unit; 401 int cnt = 0; 402 403 disk = ndisk->disk; 404 unit = disk->d_unit; 405 taskqueue_free(ndisk->tq); 406 407 disk_destroy(ndisk->disk); 408 409 mtx_lock(&ndisk->bioqlock); 410 for (;;) { 411 bp = bioq_takefirst(&ndisk->bioq); 412 if (bp == NULL) 413 break; 414 bp->bio_error = EIO; 415 bp->bio_flags |= BIO_ERROR; 416 bp->bio_resid = bp->bio_bcount; 417 cnt++; 418 biodone(bp); 419 } 420 421 printf(NVD_STR"%u: lost device - %d outstanding\n", unit, cnt); 422 printf(NVD_STR"%u: removing device entry\n", unit); 423 424 mtx_unlock(&ndisk->bioqlock); 425 426 mtx_destroy(&ndisk->bioqlock); 427 } 428 429 static void 430 nvd_controller_fail(void *ctrlr_arg) 431 { 432 struct nvd_controller *ctrlr = ctrlr_arg; 433 struct nvd_disk *disk; 434 435 while (!TAILQ_EMPTY(&ctrlr->disk_head)) { 436 disk = TAILQ_FIRST(&ctrlr->disk_head); 437 TAILQ_REMOVE(&disk_head, disk, global_tailq); 438 TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq); 439 destroy_geom_disk(disk); 440 free(disk, M_NVD); 441 } 442 443 TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq); 444 free(ctrlr, M_NVD); 445 } 446 447