1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2012-2013 Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/bio.h> 34 #include <sys/bus.h> 35 #include <sys/conf.h> 36 #include <sys/disk.h> 37 #include <sys/fcntl.h> 38 #include <sys/ioccom.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/proc.h> 42 #include <sys/systm.h> 43 44 #include <dev/pci/pcivar.h> 45 46 #include <geom/geom.h> 47 48 #include "nvme_private.h" 49 50 static void nvme_bio_child_inbed(struct bio *parent, int bio_error); 51 static void nvme_bio_child_done(void *arg, 52 const struct nvme_completion *cpl); 53 static uint32_t nvme_get_num_segments(uint64_t addr, uint64_t size, 54 uint32_t alignment); 55 static void nvme_free_child_bios(int num_bios, 56 struct bio **child_bios); 57 static struct bio ** nvme_allocate_child_bios(int num_bios); 58 static struct bio ** nvme_construct_child_bios(struct bio *bp, 59 uint32_t alignment, 60 int *num_bios); 61 static int nvme_ns_split_bio(struct nvme_namespace *ns, 62 struct bio *bp, 63 uint32_t alignment); 64 65 static int 66 nvme_ns_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag, 67 struct thread *td) 68 { 69 struct nvme_namespace *ns; 70 struct nvme_controller *ctrlr; 71 struct nvme_pt_command *pt; 72 73 ns = cdev->si_drv1; 74 ctrlr = ns->ctrlr; 75 76 switch (cmd) { 77 case NVME_IO_TEST: 78 case NVME_BIO_TEST: 79 nvme_ns_test(ns, cmd, arg); 80 break; 81 case NVME_PASSTHROUGH_CMD: 82 pt = (struct nvme_pt_command *)arg; 83 return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, ns->id, 84 1 /* is_user_buffer */, 0 /* is_admin_cmd */)); 85 case DIOCGMEDIASIZE: 86 *(off_t *)arg = (off_t)nvme_ns_get_size(ns); 87 break; 88 case DIOCGSECTORSIZE: 89 *(u_int *)arg = nvme_ns_get_sector_size(ns); 90 break; 91 default: 92 return (ENOTTY); 93 } 94 95 return (0); 96 } 97 98 static int 99 nvme_ns_open(struct cdev *dev __unused, int flags, int fmt __unused, 100 struct thread *td) 101 { 102 int error = 0; 103 104 if (flags & FWRITE) 105 error = securelevel_gt(td->td_ucred, 0); 106 107 return (error); 108 } 109 110 static int 111 nvme_ns_close(struct cdev *dev __unused, int flags, int fmt __unused, 112 struct thread *td) 113 { 114 115 return (0); 116 } 117 118 static void 119 nvme_ns_strategy_done(void *arg, const struct nvme_completion *cpl) 120 { 121 struct bio *bp = arg; 122 123 /* 124 * TODO: add more extensive translation of NVMe status codes 125 * to different bio error codes (i.e. EIO, EINVAL, etc.) 126 */ 127 if (nvme_completion_is_error(cpl)) { 128 bp->bio_error = EIO; 129 bp->bio_flags |= BIO_ERROR; 130 bp->bio_resid = bp->bio_bcount; 131 } else 132 bp->bio_resid = 0; 133 134 biodone(bp); 135 } 136 137 static void 138 nvme_ns_strategy(struct bio *bp) 139 { 140 struct nvme_namespace *ns; 141 int err; 142 143 ns = bp->bio_dev->si_drv1; 144 err = nvme_ns_bio_process(ns, bp, nvme_ns_strategy_done); 145 146 if (err) { 147 bp->bio_error = err; 148 bp->bio_flags |= BIO_ERROR; 149 bp->bio_resid = bp->bio_bcount; 150 biodone(bp); 151 } 152 153 } 154 155 static struct cdevsw nvme_ns_cdevsw = { 156 .d_version = D_VERSION, 157 .d_flags = D_DISK, 158 .d_read = physread, 159 .d_write = physwrite, 160 .d_open = nvme_ns_open, 161 .d_close = nvme_ns_close, 162 .d_strategy = nvme_ns_strategy, 163 .d_ioctl = nvme_ns_ioctl 164 }; 165 166 uint32_t 167 nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns) 168 { 169 return ns->ctrlr->max_xfer_size; 170 } 171 172 uint32_t 173 nvme_ns_get_sector_size(struct nvme_namespace *ns) 174 { 175 uint8_t flbas_fmt, lbads; 176 177 flbas_fmt = (ns->data.flbas >> NVME_NS_DATA_FLBAS_FORMAT_SHIFT) & 178 NVME_NS_DATA_FLBAS_FORMAT_MASK; 179 lbads = (ns->data.lbaf[flbas_fmt] >> NVME_NS_DATA_LBAF_LBADS_SHIFT) & 180 NVME_NS_DATA_LBAF_LBADS_MASK; 181 182 return (1 << lbads); 183 } 184 185 uint64_t 186 nvme_ns_get_num_sectors(struct nvme_namespace *ns) 187 { 188 return (ns->data.nsze); 189 } 190 191 uint64_t 192 nvme_ns_get_size(struct nvme_namespace *ns) 193 { 194 return (nvme_ns_get_num_sectors(ns) * nvme_ns_get_sector_size(ns)); 195 } 196 197 uint32_t 198 nvme_ns_get_flags(struct nvme_namespace *ns) 199 { 200 return (ns->flags); 201 } 202 203 const char * 204 nvme_ns_get_serial_number(struct nvme_namespace *ns) 205 { 206 return ((const char *)ns->ctrlr->cdata.sn); 207 } 208 209 const char * 210 nvme_ns_get_model_number(struct nvme_namespace *ns) 211 { 212 return ((const char *)ns->ctrlr->cdata.mn); 213 } 214 215 const struct nvme_namespace_data * 216 nvme_ns_get_data(struct nvme_namespace *ns) 217 { 218 219 return (&ns->data); 220 } 221 222 uint32_t 223 nvme_ns_get_stripesize(struct nvme_namespace *ns) 224 { 225 226 return (ns->stripesize); 227 } 228 229 static void 230 nvme_ns_bio_done(void *arg, const struct nvme_completion *status) 231 { 232 struct bio *bp = arg; 233 nvme_cb_fn_t bp_cb_fn; 234 235 bp_cb_fn = bp->bio_driver1; 236 237 if (bp->bio_driver2) 238 free(bp->bio_driver2, M_NVME); 239 240 if (nvme_completion_is_error(status)) { 241 bp->bio_flags |= BIO_ERROR; 242 if (bp->bio_error == 0) 243 bp->bio_error = EIO; 244 } 245 246 if ((bp->bio_flags & BIO_ERROR) == 0) 247 bp->bio_resid = 0; 248 else 249 bp->bio_resid = bp->bio_bcount; 250 251 bp_cb_fn(bp, status); 252 } 253 254 static void 255 nvme_bio_child_inbed(struct bio *parent, int bio_error) 256 { 257 struct nvme_completion parent_cpl; 258 int children, inbed; 259 260 if (bio_error != 0) { 261 parent->bio_flags |= BIO_ERROR; 262 parent->bio_error = bio_error; 263 } 264 265 /* 266 * atomic_fetchadd will return value before adding 1, so we still 267 * must add 1 to get the updated inbed number. Save bio_children 268 * before incrementing to guard against race conditions when 269 * two children bios complete on different queues. 270 */ 271 children = atomic_load_acq_int(&parent->bio_children); 272 inbed = atomic_fetchadd_int(&parent->bio_inbed, 1) + 1; 273 if (inbed == children) { 274 bzero(&parent_cpl, sizeof(parent_cpl)); 275 if (parent->bio_flags & BIO_ERROR) { 276 parent_cpl.status &= ~(NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT); 277 parent_cpl.status |= (NVME_SC_DATA_TRANSFER_ERROR) << NVME_STATUS_SC_SHIFT; 278 } 279 nvme_ns_bio_done(parent, &parent_cpl); 280 } 281 } 282 283 static void 284 nvme_bio_child_done(void *arg, const struct nvme_completion *cpl) 285 { 286 struct bio *child = arg; 287 struct bio *parent; 288 int bio_error; 289 290 parent = child->bio_parent; 291 g_destroy_bio(child); 292 bio_error = nvme_completion_is_error(cpl) ? EIO : 0; 293 nvme_bio_child_inbed(parent, bio_error); 294 } 295 296 static uint32_t 297 nvme_get_num_segments(uint64_t addr, uint64_t size, uint32_t align) 298 { 299 uint32_t num_segs, offset, remainder; 300 301 if (align == 0) 302 return (1); 303 304 KASSERT((align & (align - 1)) == 0, ("alignment not power of 2\n")); 305 306 num_segs = size / align; 307 remainder = size & (align - 1); 308 offset = addr & (align - 1); 309 if (remainder > 0 || offset > 0) 310 num_segs += 1 + (remainder + offset - 1) / align; 311 return (num_segs); 312 } 313 314 static void 315 nvme_free_child_bios(int num_bios, struct bio **child_bios) 316 { 317 int i; 318 319 for (i = 0; i < num_bios; i++) { 320 if (child_bios[i] != NULL) 321 g_destroy_bio(child_bios[i]); 322 } 323 324 free(child_bios, M_NVME); 325 } 326 327 static struct bio ** 328 nvme_allocate_child_bios(int num_bios) 329 { 330 struct bio **child_bios; 331 int err = 0, i; 332 333 child_bios = malloc(num_bios * sizeof(struct bio *), M_NVME, M_NOWAIT); 334 if (child_bios == NULL) 335 return (NULL); 336 337 for (i = 0; i < num_bios; i++) { 338 child_bios[i] = g_new_bio(); 339 if (child_bios[i] == NULL) 340 err = ENOMEM; 341 } 342 343 if (err == ENOMEM) { 344 nvme_free_child_bios(num_bios, child_bios); 345 return (NULL); 346 } 347 348 return (child_bios); 349 } 350 351 static struct bio ** 352 nvme_construct_child_bios(struct bio *bp, uint32_t alignment, int *num_bios) 353 { 354 struct bio **child_bios; 355 struct bio *child; 356 uint64_t cur_offset; 357 caddr_t data; 358 uint32_t rem_bcount; 359 int i; 360 #ifdef NVME_UNMAPPED_BIO_SUPPORT 361 struct vm_page **ma; 362 uint32_t ma_offset; 363 #endif 364 365 *num_bios = nvme_get_num_segments(bp->bio_offset, bp->bio_bcount, 366 alignment); 367 child_bios = nvme_allocate_child_bios(*num_bios); 368 if (child_bios == NULL) 369 return (NULL); 370 371 bp->bio_children = *num_bios; 372 bp->bio_inbed = 0; 373 cur_offset = bp->bio_offset; 374 rem_bcount = bp->bio_bcount; 375 data = bp->bio_data; 376 #ifdef NVME_UNMAPPED_BIO_SUPPORT 377 ma_offset = bp->bio_ma_offset; 378 ma = bp->bio_ma; 379 #endif 380 381 for (i = 0; i < *num_bios; i++) { 382 child = child_bios[i]; 383 child->bio_parent = bp; 384 child->bio_cmd = bp->bio_cmd; 385 child->bio_offset = cur_offset; 386 child->bio_bcount = min(rem_bcount, 387 alignment - (cur_offset & (alignment - 1))); 388 child->bio_flags = bp->bio_flags; 389 #ifdef NVME_UNMAPPED_BIO_SUPPORT 390 if (bp->bio_flags & BIO_UNMAPPED) { 391 child->bio_ma_offset = ma_offset; 392 child->bio_ma = ma; 393 child->bio_ma_n = 394 nvme_get_num_segments(child->bio_ma_offset, 395 child->bio_bcount, PAGE_SIZE); 396 ma_offset = (ma_offset + child->bio_bcount) & 397 PAGE_MASK; 398 ma += child->bio_ma_n; 399 if (ma_offset != 0) 400 ma -= 1; 401 } else 402 #endif 403 { 404 child->bio_data = data; 405 data += child->bio_bcount; 406 } 407 cur_offset += child->bio_bcount; 408 rem_bcount -= child->bio_bcount; 409 } 410 411 return (child_bios); 412 } 413 414 static int 415 nvme_ns_split_bio(struct nvme_namespace *ns, struct bio *bp, 416 uint32_t alignment) 417 { 418 struct bio *child; 419 struct bio **child_bios; 420 int err, i, num_bios; 421 422 child_bios = nvme_construct_child_bios(bp, alignment, &num_bios); 423 if (child_bios == NULL) 424 return (ENOMEM); 425 426 for (i = 0; i < num_bios; i++) { 427 child = child_bios[i]; 428 err = nvme_ns_bio_process(ns, child, nvme_bio_child_done); 429 if (err != 0) { 430 nvme_bio_child_inbed(bp, err); 431 g_destroy_bio(child); 432 } 433 } 434 435 free(child_bios, M_NVME); 436 return (0); 437 } 438 439 int 440 nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp, 441 nvme_cb_fn_t cb_fn) 442 { 443 struct nvme_dsm_range *dsm_range; 444 uint32_t num_bios; 445 int err; 446 447 bp->bio_driver1 = cb_fn; 448 449 if (ns->stripesize > 0 && 450 (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) { 451 num_bios = nvme_get_num_segments(bp->bio_offset, 452 bp->bio_bcount, ns->stripesize); 453 if (num_bios > 1) 454 return (nvme_ns_split_bio(ns, bp, ns->stripesize)); 455 } 456 457 switch (bp->bio_cmd) { 458 case BIO_READ: 459 err = nvme_ns_cmd_read_bio(ns, bp, nvme_ns_bio_done, bp); 460 break; 461 case BIO_WRITE: 462 err = nvme_ns_cmd_write_bio(ns, bp, nvme_ns_bio_done, bp); 463 break; 464 case BIO_FLUSH: 465 err = nvme_ns_cmd_flush(ns, nvme_ns_bio_done, bp); 466 break; 467 case BIO_DELETE: 468 dsm_range = 469 malloc(sizeof(struct nvme_dsm_range), M_NVME, 470 M_ZERO | M_WAITOK); 471 if (!dsm_range) { 472 err = ENOMEM; 473 break; 474 } 475 dsm_range->length = 476 htole32(bp->bio_bcount/nvme_ns_get_sector_size(ns)); 477 dsm_range->starting_lba = 478 htole64(bp->bio_offset/nvme_ns_get_sector_size(ns)); 479 bp->bio_driver2 = dsm_range; 480 err = nvme_ns_cmd_deallocate(ns, dsm_range, 1, 481 nvme_ns_bio_done, bp); 482 if (err != 0) 483 free(dsm_range, M_NVME); 484 break; 485 default: 486 err = EIO; 487 break; 488 } 489 490 return (err); 491 } 492 493 int 494 nvme_ns_construct(struct nvme_namespace *ns, uint32_t id, 495 struct nvme_controller *ctrlr) 496 { 497 struct nvme_completion_poll_status status; 498 int unit; 499 uint16_t oncs; 500 uint8_t dsm; 501 uint8_t flbas_fmt; 502 uint8_t vwc_present; 503 504 ns->ctrlr = ctrlr; 505 ns->id = id; 506 ns->stripesize = 0; 507 508 if (pci_get_devid(ctrlr->dev) == 0x09538086 && ctrlr->cdata.vs[3] != 0) 509 ns->stripesize = 510 (1 << ctrlr->cdata.vs[3]) * ctrlr->min_page_size; 511 512 /* 513 * Namespaces are reconstructed after a controller reset, so check 514 * to make sure we only call mtx_init once on each mtx. 515 * 516 * TODO: Move this somewhere where it gets called at controller 517 * construction time, which is not invoked as part of each 518 * controller reset. 519 */ 520 if (!mtx_initialized(&ns->lock)) 521 mtx_init(&ns->lock, "nvme ns lock", NULL, MTX_DEF); 522 523 status.done = FALSE; 524 nvme_ctrlr_cmd_identify_namespace(ctrlr, id, &ns->data, 525 nvme_completion_poll_cb, &status); 526 while (status.done == FALSE) 527 DELAY(5); 528 if (nvme_completion_is_error(&status.cpl)) { 529 nvme_printf(ctrlr, "nvme_identify_namespace failed\n"); 530 return (ENXIO); 531 } 532 533 /* Convert data to host endian */ 534 nvme_namespace_data_swapbytes(&ns->data); 535 536 /* 537 * If the size of is zero, chances are this isn't a valid 538 * namespace (eg one that's not been configured yet). The 539 * standard says the entire id will be zeros, so this is a 540 * cheap way to test for that. 541 */ 542 if (ns->data.nsze == 0) 543 return (ENXIO); 544 545 flbas_fmt = (ns->data.flbas >> NVME_NS_DATA_FLBAS_FORMAT_SHIFT) & 546 NVME_NS_DATA_FLBAS_FORMAT_MASK; 547 /* 548 * Note: format is a 0-based value, so > is appropriate here, 549 * not >=. 550 */ 551 if (flbas_fmt > ns->data.nlbaf) { 552 printf("lba format %d exceeds number supported (%d)\n", 553 flbas_fmt, ns->data.nlbaf + 1); 554 return (ENXIO); 555 } 556 557 oncs = ctrlr->cdata.oncs; 558 dsm = (oncs >> NVME_CTRLR_DATA_ONCS_DSM_SHIFT) & NVME_CTRLR_DATA_ONCS_DSM_MASK; 559 if (dsm) 560 ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED; 561 562 vwc_present = (ctrlr->cdata.vwc >> NVME_CTRLR_DATA_VWC_PRESENT_SHIFT) & 563 NVME_CTRLR_DATA_VWC_PRESENT_MASK; 564 if (vwc_present) 565 ns->flags |= NVME_NS_FLUSH_SUPPORTED; 566 567 /* 568 * cdev may have already been created, if we are reconstructing the 569 * namespace after a controller-level reset. 570 */ 571 if (ns->cdev != NULL) 572 return (0); 573 574 /* 575 * Namespace IDs start at 1, so we need to subtract 1 to create a 576 * correct unit number. 577 */ 578 unit = device_get_unit(ctrlr->dev) * NVME_MAX_NAMESPACES + ns->id - 1; 579 580 ns->cdev = make_dev_credf(0, &nvme_ns_cdevsw, unit, 581 NULL, UID_ROOT, GID_WHEEL, 0600, "nvme%dns%d", 582 device_get_unit(ctrlr->dev), ns->id); 583 #ifdef NVME_UNMAPPED_BIO_SUPPORT 584 ns->cdev->si_flags |= SI_UNMAPPED; 585 #endif 586 587 if (ns->cdev != NULL) 588 ns->cdev->si_drv1 = ns; 589 590 return (0); 591 } 592 593 void nvme_ns_destruct(struct nvme_namespace *ns) 594 { 595 596 if (ns->cdev != NULL) 597 destroy_dev(ns->cdev); 598 } 599