1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2012-2013 Intel Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/bio.h> 31 #include <sys/bus.h> 32 #include <sys/conf.h> 33 #include <sys/disk.h> 34 #include <sys/fcntl.h> 35 #include <sys/ioccom.h> 36 #include <sys/malloc.h> 37 #include <sys/module.h> 38 #include <sys/proc.h> 39 #include <sys/systm.h> 40 41 #include <dev/pci/pcivar.h> 42 43 #include <geom/geom.h> 44 45 #include "nvme_private.h" 46 #include "nvme_linux.h" 47 48 static void nvme_bio_child_inbed(struct bio *parent, int bio_error); 49 static void nvme_bio_child_done(void *arg, 50 const struct nvme_completion *cpl); 51 static uint32_t nvme_get_num_segments(uint64_t addr, uint64_t size, 52 uint32_t alignment); 53 static void nvme_free_child_bios(int num_bios, 54 struct bio **child_bios); 55 static struct bio ** nvme_allocate_child_bios(int num_bios); 56 static struct bio ** nvme_construct_child_bios(struct bio *bp, 57 uint32_t alignment, 58 int *num_bios); 59 static int nvme_ns_split_bio(struct nvme_namespace *ns, 60 struct bio *bp, 61 uint32_t alignment); 62 63 static int 64 nvme_ns_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag, 65 struct thread *td) 66 { 67 struct nvme_namespace *ns; 68 struct nvme_controller *ctrlr; 69 struct nvme_pt_command *pt; 70 71 ns = cdev->si_drv1; 72 ctrlr = ns->ctrlr; 73 74 switch (cmd) { 75 case NVME_IO_TEST: 76 case NVME_BIO_TEST: 77 nvme_ns_test(ns, cmd, arg); 78 break; 79 case NVME_PASSTHROUGH_CMD: 80 pt = (struct nvme_pt_command *)arg; 81 return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, ns->id, 82 1 /* is_user_buffer */, 0 /* is_admin_cmd */)); 83 case NVME_GET_NSID: 84 { 85 struct nvme_get_nsid *gnsid = (struct nvme_get_nsid *)arg; 86 strlcpy(gnsid->cdev, device_get_nameunit(ctrlr->dev), 87 sizeof(gnsid->cdev)); 88 gnsid->nsid = ns->id; 89 break; 90 } 91 case DIOCGIDENT: { 92 uint8_t *sn = arg; 93 nvme_ctrlr_get_ident(ctrlr, sn); 94 break; 95 } 96 case DIOCGMEDIASIZE: 97 *(off_t *)arg = (off_t)nvme_ns_get_size(ns); 98 break; 99 case DIOCGSECTORSIZE: 100 *(u_int *)arg = nvme_ns_get_sector_size(ns); 101 break; 102 /* Linux Compatible (see nvme_linux.h) */ 103 case NVME_IOCTL_ID: 104 td->td_retval[0] = ns->id; 105 return (0); 106 107 case NVME_IOCTL_ADMIN_CMD: 108 case NVME_IOCTL_IO_CMD: { 109 struct nvme_passthru_cmd *npc = (struct nvme_passthru_cmd *)arg; 110 111 return (nvme_ctrlr_linux_passthru_cmd(ctrlr, npc, ns->id, true, 112 cmd == NVME_IOCTL_ADMIN_CMD)); 113 } 114 default: 115 return (ENOTTY); 116 } 117 118 return (0); 119 } 120 121 static int 122 nvme_ns_open(struct cdev *dev __unused, int flags, int fmt __unused, 123 struct thread *td) 124 { 125 int error = 0; 126 127 if (flags & FWRITE) 128 error = securelevel_gt(td->td_ucred, 0); 129 130 return (error); 131 } 132 133 static int 134 nvme_ns_close(struct cdev *dev __unused, int flags, int fmt __unused, 135 struct thread *td) 136 { 137 return (0); 138 } 139 140 static void 141 nvme_ns_strategy_done(void *arg, const struct nvme_completion *cpl) 142 { 143 struct bio *bp = arg; 144 145 if (nvme_completion_is_error(cpl)) { 146 bp->bio_error = EIO; 147 bp->bio_flags |= BIO_ERROR; 148 bp->bio_resid = bp->bio_bcount; 149 } else 150 bp->bio_resid = 0; 151 152 biodone(bp); 153 } 154 155 static void 156 nvme_ns_strategy(struct bio *bp) 157 { 158 struct nvme_namespace *ns; 159 int err; 160 161 ns = bp->bio_dev->si_drv1; 162 err = nvme_ns_bio_process(ns, bp, nvme_ns_strategy_done); 163 164 if (err) { 165 bp->bio_error = err; 166 bp->bio_flags |= BIO_ERROR; 167 bp->bio_resid = bp->bio_bcount; 168 biodone(bp); 169 } 170 171 } 172 173 static struct cdevsw nvme_ns_cdevsw = { 174 .d_version = D_VERSION, 175 .d_flags = D_DISK, 176 .d_read = physread, 177 .d_write = physwrite, 178 .d_open = nvme_ns_open, 179 .d_close = nvme_ns_close, 180 .d_strategy = nvme_ns_strategy, 181 .d_ioctl = nvme_ns_ioctl 182 }; 183 184 uint32_t 185 nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns) 186 { 187 return ns->ctrlr->max_xfer_size; 188 } 189 190 uint32_t 191 nvme_ns_get_sector_size(struct nvme_namespace *ns) 192 { 193 uint8_t flbas_fmt, lbads; 194 195 flbas_fmt = NVMEV(NVME_NS_DATA_FLBAS_FORMAT, ns->data.flbas); 196 lbads = NVMEV(NVME_NS_DATA_LBAF_LBADS, ns->data.lbaf[flbas_fmt]); 197 198 return (1 << lbads); 199 } 200 201 uint64_t 202 nvme_ns_get_num_sectors(struct nvme_namespace *ns) 203 { 204 return (ns->data.nsze); 205 } 206 207 uint64_t 208 nvme_ns_get_size(struct nvme_namespace *ns) 209 { 210 return (nvme_ns_get_num_sectors(ns) * nvme_ns_get_sector_size(ns)); 211 } 212 213 uint32_t 214 nvme_ns_get_flags(struct nvme_namespace *ns) 215 { 216 return (ns->flags); 217 } 218 219 const char * 220 nvme_ns_get_serial_number(struct nvme_namespace *ns) 221 { 222 return ((const char *)ns->ctrlr->cdata.sn); 223 } 224 225 const char * 226 nvme_ns_get_model_number(struct nvme_namespace *ns) 227 { 228 return ((const char *)ns->ctrlr->cdata.mn); 229 } 230 231 const struct nvme_namespace_data * 232 nvme_ns_get_data(struct nvme_namespace *ns) 233 { 234 return (&ns->data); 235 } 236 237 uint32_t 238 nvme_ns_get_stripesize(struct nvme_namespace *ns) 239 { 240 uint32_t ss; 241 242 if (NVMEV(NVME_NS_DATA_NSFEAT_NPVALID, ns->data.nsfeat) != 0) { 243 ss = nvme_ns_get_sector_size(ns); 244 if (ns->data.npwa != 0) 245 return ((ns->data.npwa + 1) * ss); 246 else if (ns->data.npwg != 0) 247 return ((ns->data.npwg + 1) * ss); 248 } 249 return (ns->boundary); 250 } 251 252 static void 253 nvme_ns_bio_done(void *arg, const struct nvme_completion *status) 254 { 255 struct bio *bp = arg; 256 nvme_cb_fn_t bp_cb_fn; 257 258 bp_cb_fn = bp->bio_driver1; 259 260 if (bp->bio_driver2) 261 free(bp->bio_driver2, M_NVME); 262 263 if (nvme_completion_is_error(status)) { 264 bp->bio_flags |= BIO_ERROR; 265 if (bp->bio_error == 0) 266 bp->bio_error = EIO; 267 } 268 269 if ((bp->bio_flags & BIO_ERROR) == 0) 270 bp->bio_resid = 0; 271 else 272 bp->bio_resid = bp->bio_bcount; 273 274 bp_cb_fn(bp, status); 275 } 276 277 static void 278 nvme_bio_child_inbed(struct bio *parent, int bio_error) 279 { 280 struct nvme_completion parent_cpl; 281 int children, inbed; 282 283 if (bio_error != 0) { 284 parent->bio_flags |= BIO_ERROR; 285 parent->bio_error = bio_error; 286 } 287 288 /* 289 * atomic_fetchadd will return value before adding 1, so we still 290 * must add 1 to get the updated inbed number. Save bio_children 291 * before incrementing to guard against race conditions when 292 * two children bios complete on different queues. 293 */ 294 children = atomic_load_acq_int(&parent->bio_children); 295 inbed = atomic_fetchadd_int(&parent->bio_inbed, 1) + 1; 296 if (inbed == children) { 297 bzero(&parent_cpl, sizeof(parent_cpl)); 298 if (parent->bio_flags & BIO_ERROR) { 299 parent_cpl.status &= ~NVMEM(NVME_STATUS_SC); 300 parent_cpl.status |= NVMEF(NVME_STATUS_SC, 301 NVME_SC_DATA_TRANSFER_ERROR); 302 } 303 nvme_ns_bio_done(parent, &parent_cpl); 304 } 305 } 306 307 static void 308 nvme_bio_child_done(void *arg, const struct nvme_completion *cpl) 309 { 310 struct bio *child = arg; 311 struct bio *parent; 312 int bio_error; 313 314 parent = child->bio_parent; 315 g_destroy_bio(child); 316 bio_error = nvme_completion_is_error(cpl) ? EIO : 0; 317 nvme_bio_child_inbed(parent, bio_error); 318 } 319 320 static uint32_t 321 nvme_get_num_segments(uint64_t addr, uint64_t size, uint32_t align) 322 { 323 uint32_t num_segs, offset, remainder; 324 325 if (align == 0) 326 return (1); 327 328 KASSERT((align & (align - 1)) == 0, ("alignment not power of 2\n")); 329 330 num_segs = size / align; 331 remainder = size & (align - 1); 332 offset = addr & (align - 1); 333 if (remainder > 0 || offset > 0) 334 num_segs += 1 + (remainder + offset - 1) / align; 335 return (num_segs); 336 } 337 338 static void 339 nvme_free_child_bios(int num_bios, struct bio **child_bios) 340 { 341 int i; 342 343 for (i = 0; i < num_bios; i++) { 344 if (child_bios[i] != NULL) 345 g_destroy_bio(child_bios[i]); 346 } 347 348 free(child_bios, M_NVME); 349 } 350 351 static struct bio ** 352 nvme_allocate_child_bios(int num_bios) 353 { 354 struct bio **child_bios; 355 int err = 0, i; 356 357 child_bios = malloc(num_bios * sizeof(struct bio *), M_NVME, M_NOWAIT); 358 if (child_bios == NULL) 359 return (NULL); 360 361 for (i = 0; i < num_bios; i++) { 362 child_bios[i] = g_new_bio(); 363 if (child_bios[i] == NULL) 364 err = ENOMEM; 365 } 366 367 if (err == ENOMEM) { 368 nvme_free_child_bios(num_bios, child_bios); 369 return (NULL); 370 } 371 372 return (child_bios); 373 } 374 375 static struct bio ** 376 nvme_construct_child_bios(struct bio *bp, uint32_t alignment, int *num_bios) 377 { 378 struct bio **child_bios; 379 struct bio *child; 380 uint64_t cur_offset; 381 caddr_t data; 382 uint32_t rem_bcount; 383 int i; 384 struct vm_page **ma; 385 uint32_t ma_offset; 386 387 *num_bios = nvme_get_num_segments(bp->bio_offset, bp->bio_bcount, 388 alignment); 389 child_bios = nvme_allocate_child_bios(*num_bios); 390 if (child_bios == NULL) 391 return (NULL); 392 393 bp->bio_children = *num_bios; 394 bp->bio_inbed = 0; 395 cur_offset = bp->bio_offset; 396 rem_bcount = bp->bio_bcount; 397 data = bp->bio_data; 398 ma_offset = bp->bio_ma_offset; 399 ma = bp->bio_ma; 400 401 for (i = 0; i < *num_bios; i++) { 402 child = child_bios[i]; 403 child->bio_parent = bp; 404 child->bio_cmd = bp->bio_cmd; 405 child->bio_offset = cur_offset; 406 child->bio_bcount = min(rem_bcount, 407 alignment - (cur_offset & (alignment - 1))); 408 child->bio_flags = bp->bio_flags; 409 if (bp->bio_flags & BIO_UNMAPPED) { 410 child->bio_ma_offset = ma_offset; 411 child->bio_ma = ma; 412 child->bio_ma_n = 413 nvme_get_num_segments(child->bio_ma_offset, 414 child->bio_bcount, PAGE_SIZE); 415 ma_offset = (ma_offset + child->bio_bcount) & 416 PAGE_MASK; 417 ma += child->bio_ma_n; 418 if (ma_offset != 0) 419 ma -= 1; 420 } else { 421 child->bio_data = data; 422 data += child->bio_bcount; 423 } 424 cur_offset += child->bio_bcount; 425 rem_bcount -= child->bio_bcount; 426 } 427 428 return (child_bios); 429 } 430 431 static int 432 nvme_ns_split_bio(struct nvme_namespace *ns, struct bio *bp, 433 uint32_t alignment) 434 { 435 struct bio *child; 436 struct bio **child_bios; 437 int err, i, num_bios; 438 439 child_bios = nvme_construct_child_bios(bp, alignment, &num_bios); 440 if (child_bios == NULL) 441 return (ENOMEM); 442 443 counter_u64_add(ns->ctrlr->alignment_splits, 1); 444 for (i = 0; i < num_bios; i++) { 445 child = child_bios[i]; 446 err = nvme_ns_bio_process(ns, child, nvme_bio_child_done); 447 if (err != 0) { 448 nvme_bio_child_inbed(bp, err); 449 g_destroy_bio(child); 450 } 451 } 452 453 free(child_bios, M_NVME); 454 return (0); 455 } 456 457 int 458 nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp, 459 nvme_cb_fn_t cb_fn) 460 { 461 struct nvme_dsm_range *dsm_range; 462 uint32_t num_bios; 463 int err; 464 465 bp->bio_driver1 = cb_fn; 466 467 if (ns->boundary > 0 && 468 (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) { 469 num_bios = nvme_get_num_segments(bp->bio_offset, 470 bp->bio_bcount, ns->boundary); 471 if (num_bios > 1) 472 return (nvme_ns_split_bio(ns, bp, ns->boundary)); 473 } 474 475 switch (bp->bio_cmd) { 476 case BIO_READ: 477 err = nvme_ns_cmd_read_bio(ns, bp, nvme_ns_bio_done, bp); 478 break; 479 case BIO_WRITE: 480 err = nvme_ns_cmd_write_bio(ns, bp, nvme_ns_bio_done, bp); 481 break; 482 case BIO_FLUSH: 483 err = nvme_ns_cmd_flush(ns, nvme_ns_bio_done, bp); 484 break; 485 case BIO_DELETE: 486 dsm_range = 487 malloc(sizeof(struct nvme_dsm_range), M_NVME, 488 M_ZERO | M_NOWAIT); 489 if (!dsm_range) { 490 err = ENOMEM; 491 break; 492 } 493 dsm_range->length = 494 htole32(bp->bio_bcount/nvme_ns_get_sector_size(ns)); 495 dsm_range->starting_lba = 496 htole64(bp->bio_offset/nvme_ns_get_sector_size(ns)); 497 bp->bio_driver2 = dsm_range; 498 err = nvme_ns_cmd_deallocate(ns, dsm_range, 1, 499 nvme_ns_bio_done, bp); 500 if (err != 0) 501 free(dsm_range, M_NVME); 502 break; 503 default: 504 err = EOPNOTSUPP; 505 break; 506 } 507 508 return (err); 509 } 510 511 int 512 nvme_ns_ioctl_process(struct nvme_namespace *ns, u_long cmd, caddr_t arg, 513 int flag, struct thread *td) 514 { 515 return (nvme_ns_ioctl(ns->cdev, cmd, arg, flag, td)); 516 } 517 518 int 519 nvme_ns_construct(struct nvme_namespace *ns, uint32_t id, 520 struct nvme_controller *ctrlr) 521 { 522 struct make_dev_args md_args; 523 struct nvme_completion_poll_status status; 524 int res; 525 int unit; 526 uint8_t flbas_fmt; 527 uint8_t vwc_present; 528 529 ns->ctrlr = ctrlr; 530 ns->id = id; 531 532 /* 533 * Namespaces are reconstructed after a controller reset, so check 534 * to make sure we only call mtx_init once on each mtx. 535 * 536 * TODO: Move this somewhere where it gets called at controller 537 * construction time, which is not invoked as part of each 538 * controller reset. 539 */ 540 if (!mtx_initialized(&ns->lock)) 541 mtx_init(&ns->lock, "nvme ns lock", NULL, MTX_DEF); 542 543 status.done = 0; 544 nvme_ctrlr_cmd_identify_namespace(ctrlr, id, &ns->data, 545 nvme_completion_poll_cb, &status); 546 nvme_completion_poll(&status); 547 if (nvme_completion_is_error(&status.cpl)) { 548 nvme_printf(ctrlr, "nvme_identify_namespace failed\n"); 549 return (ENXIO); 550 } 551 552 /* Convert data to host endian */ 553 nvme_namespace_data_swapbytes(&ns->data); 554 555 /* 556 * If the size of is zero, chances are this isn't a valid 557 * namespace (eg one that's not been configured yet). The 558 * standard says the entire id will be zeros, so this is a 559 * cheap way to test for that. 560 */ 561 if (ns->data.nsze == 0) 562 return (ENXIO); 563 564 flbas_fmt = NVMEV(NVME_NS_DATA_FLBAS_FORMAT, ns->data.flbas); 565 566 /* 567 * Note: format is a 0-based value, so > is appropriate here, 568 * not >=. 569 */ 570 if (flbas_fmt > ns->data.nlbaf) { 571 nvme_printf(ctrlr, 572 "lba format %d exceeds number supported (%d)\n", 573 flbas_fmt, ns->data.nlbaf + 1); 574 return (ENXIO); 575 } 576 577 /* 578 * Older Intel devices (like the PC35xxx and P45xx series) advertise in 579 * vendor specific space an alignment that improves performance. If 580 * present use for the stripe size. NVMe 1.3 standardized this as 581 * NOIOB, and newer Intel drives use that. 582 */ 583 if ((ctrlr->quirks & QUIRK_INTEL_ALIGNMENT) != 0) { 584 if (ctrlr->cdata.vs[3] != 0) 585 ns->boundary = 586 1 << (ctrlr->cdata.vs[3] + NVME_MPS_SHIFT + 587 NVME_CAP_HI_MPSMIN(ctrlr->cap_hi)); 588 else 589 ns->boundary = 0; 590 } else { 591 ns->boundary = ns->data.noiob * nvme_ns_get_sector_size(ns); 592 } 593 594 if (nvme_ctrlr_has_dataset_mgmt(&ctrlr->cdata)) 595 ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED; 596 597 vwc_present = NVMEV(NVME_CTRLR_DATA_VWC_PRESENT, ctrlr->cdata.vwc); 598 if (vwc_present) 599 ns->flags |= NVME_NS_FLUSH_SUPPORTED; 600 601 /* 602 * cdev may have already been created, if we are reconstructing the 603 * namespace after a controller-level reset. 604 */ 605 if (ns->cdev != NULL) 606 return (0); 607 608 /* 609 * Namespace IDs start at 1, so we need to subtract 1 to create a 610 * correct unit number. 611 */ 612 unit = device_get_unit(ctrlr->dev) * NVME_MAX_NAMESPACES + ns->id - 1; 613 614 make_dev_args_init(&md_args); 615 md_args.mda_devsw = &nvme_ns_cdevsw; 616 md_args.mda_unit = unit; 617 md_args.mda_mode = 0600; 618 md_args.mda_si_drv1 = ns; 619 res = make_dev_s(&md_args, &ns->cdev, "%sn%d", 620 device_get_nameunit(ctrlr->dev), ns->id); 621 if (res != 0) 622 return (ENXIO); 623 ns->cdev->si_drv2 = make_dev_alias(ns->cdev, "%sns%d", 624 device_get_nameunit(ctrlr->dev), ns->id); 625 ns->cdev->si_flags |= SI_UNMAPPED; 626 627 return (0); 628 } 629 630 void 631 nvme_ns_destruct(struct nvme_namespace *ns) 632 { 633 if (ns->cdev != NULL) { 634 if (ns->cdev->si_drv2 != NULL) 635 destroy_dev(ns->cdev->si_drv2); 636 destroy_dev(ns->cdev); 637 } 638 } 639